feat: wait for the local EBS provider before recovery relaunches (#1665)

The startup readiness wait and the recovery retry now also require this
node's EBS provider to answer an empty HoldsUnsealedWrites request on its
per-node subject, which no other node's provider can answer.
checkViperblockReady becomes checkJetStreamReady, since it only checks
JetStream. vm.Deps gains EBSProviderReady, read by RetryRecoveryFailed
alone; BackingStoreReady and the restore orphan check are unchanged.

The provider contract states that an empty unsealed-writes request answers
none and that per-node subjects are served after startup recovery, and a
conformance case holds every provider to the empty request.

.github/scripts/suite-sets.sh

@@ -40,7 +40,11 @@
 # received. It is last because it SIGSTOPs a cluster-wide
 # service: anything sharing the environment would see the same outage and report
 # a failure that says nothing about itself.
-E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota route53 storagefault"
+# uefistore is single-only because it runs on the node it tests: it restarts
+# that node's daemon under a running guest and reads the variable store with
+# the node's own Predastore credentials. Guests outlive a daemon restart, so
+# suites sharing the node see a brief control-plane gap, not an outage.
+E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota route53 uefistore storagefault"
 
 # Suites runnable against a multi-node environment.
 #
@@ -195,6 +199,10 @@ e2e_suite_timeout() {
     # took 24-29m, and an Oracle host ran out at 30m inside the last test's
     # cleanup. The default left it no headroom at all.
     single) echo "40m" ;;
+    # uefistore's prune test has to outwait the 10-minute prune age and a GC
+    # sweep after its last write-back, on top of three stop/starts and a boot.
+    # A pass is estimated at 40 minutes, unmeasured; the rest is the watch's cap.
+    uefistore) echo "60m" ;;
     *) echo "30m" ;;
   esac
 }

.github/workflows/e2e.yml

@@ -478,7 +478,9 @@ jobs:
                printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \
                date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
                TMO=30m; for kv in ${SUITE_TMOS}; do case \$kv in \$SUITE:*) TMO=\${kv#*:} ;; esac; done; \
-               GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+               NODE_SSH=; if [ \"\$SUITE\" = eks ]; then \
+                 NODE_SSH=\"SPINIFEX_NODES=${{ steps.vm.outputs.wan_ip }} SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2\"; fi; \
+               env \$NODE_SSH GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
                  ARTIFACT_DIR='${ARTIFACT_DIR}' \
                  ./\$SUITE.test -test.v -test.timeout=\$TMO ${TEST_RUN_FLAG} \
                  2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \

contracts/cluster/v1/heartbeat.go

@@ -1,6 +1,9 @@
 package clusterv1
 
-import "time"
+import (
+	"encoding/json"
+	"time"
+)
 
 // ClusterStateBucket is the KV bucket holding each daemon's heartbeat alongside
 // other cluster-wide state.
@@ -11,6 +14,27 @@ func HeartbeatKey(node string) string {
 	return "heartbeat." + node
 }
 
+// LivenessWindow is how recently a node must have heartbeated for a reader
+// deciding on a takeover to count it live. It is six heartbeats, compared
+// across node clocks, where NTP skew is small against it.
+const LivenessWindow = 60 * time.Second
+
+// LiveAt reports whether a heartbeat written at written still vouches for its
+// node at now: it does while it is under LivenessWindow old.
+func LiveAt(written, now time.Time) bool {
+	return now.Sub(written) < LivenessWindow
+}
+
+// HeartbeatWritten dates a stored heartbeat value. One that cannot be decoded
+// or dated vouches for nothing, which is an answer rather than a fault.
+func HeartbeatWritten(data []byte) (time.Time, bool) {
+	var beat HeartbeatTimestamp
+	if json.Unmarshal(data, &beat) != nil {
+		return time.Time{}, false
+	}
+	return beat.Time()
+}
+
 // HeartbeatTimestamp is the read side of a heartbeat for node liveness. It
 // decodes only the timestamp, so a reader does not depend on the capacity
 // figures the daemon writes beside it.

contracts/cluster/v1/heartbeat_test.go

@@ -33,3 +33,23 @@ func TestHeartbeatTimestamp_UndatedBeatVouchesForNothing(t *testing.T) {
 		assert.False(t, ok, "timestamp %q", ts)
 	}
 }
+
+// A beat is live strictly under the window, so one exactly the window old is
+// the first a takeover may treat as stopped.
+func TestLiveAt_IsStrictlyUnderTheWindow(t *testing.T) {
+	now := time.Date(2026, 11, 5, 12, 0, 0, 0, time.UTC)
+	assert.True(t, clusterv1.LiveAt(now.Add(-clusterv1.LivenessWindow+time.Nanosecond), now))
+	assert.False(t, clusterv1.LiveAt(now.Add(-clusterv1.LivenessWindow), now))
+	assert.True(t, clusterv1.LiveAt(now.Add(time.Second), now), "a beat from a clock running ahead is live")
+}
+
+func TestHeartbeatWritten_DatesAStoredBeatAndRefusesAnUndecodableOne(t *testing.T) {
+	written, ok := clusterv1.HeartbeatWritten([]byte(`{"node":"node-a","timestamp":"2026-10-30T01:02:03Z"}`))
+	require.True(t, ok)
+	assert.True(t, written.Equal(time.Date(2026, 10, 30, 1, 2, 3, 0, time.UTC)))
+
+	for _, raw := range []string{"not json", `{"node":"node-a"}`} {
+		_, ok := clusterv1.HeartbeatWritten([]byte(raw))
+		assert.False(t, ok, raw)
+	}
+}

contracts/viperblockd/legacy/v1/README.md

@@ -24,13 +24,10 @@ changes require a new contract version or an explicit migration plan.
 
 ## State reload route
 
-`SyncSubject` is `ebs.sync`. After a successful EC2 `ModifyVolume`, the daemon
-uses request/reply to ask a `spinifex-workers` queue-group member to reload the
-volume's state from its backend. The selected Viperblockd worker responds with
-`EBSSyncResponse`; it returns an error when it does not own a live mount of the
-volume. The daemon records that error as a warning after the API response has
-already been returned. This is deployed legacy behaviour, not a guarantee that
-every modification has reached its owning mount.
+`SyncSubject` is `ebs.sync`. Every `viperblockd` worker subscribes to it in the `spinifex-workers` queue group, and the selected worker reloads the volume's state from its backend and responds with `EBSSyncResponse`.
+It returns an error when it does not own a live mount of the volume.
+No control-plane code sends this request: `ExpandVolume` persists the new size before it returns and refuses a mounted volume, so every later mount loads the new size at open.
+The subscriber stays until the legacy subjects are retired.
 
 The route carries `EBSSyncRequest` and `EBSSyncResponse`. Their JSON shape and
 the queue-group route are compatibility behaviour.

docs/DESIGN.md

@@ -111,9 +111,9 @@ Daemons (`spinifex/runtime/daemon/daemon.go`) subscribe to NATS topics and handl
 When a daemon handles `RunInstances`:
 
 1. **Resource Check**: Validates CPU/memory availability via `ResourceManager`
-2. **Volume Generation**: Creates boot and EFI volumes via Viperblock
-   A UEFI instance whose record carries a variable store pointer has no EFI volume: its `vars.fd` is fetched from the object under `uefi/` in the configured Predastore bucket that the pointer names, recorded in the `spinifex-uefi-stores` JetStream bucket (or reused when `seeded.json` on this node already names the same object) after a firmware architecture and VARS size check, and a record with both or neither is refused.
-   RunInstances still creates the EFI volume, nothing writes `vars.fd` back to the object store yet, and a terminated instance's store is removed by a cluster-wide reaper.
+2. **Volume Generation**: Creates the boot volume through the EBS provider and, for a UEFI guest, seeds its variable store as an object under `uefi/<instance-id>/` with a store record in `spinifex-uefi-stores` that points at it
+   At launch a UEFI instance's `vars.fd` is fetched from the object its store pointer names (or reused when `seeded.json` on this node already names the same object) after a firmware architecture and VARS size check, and a UEFI instance with no live store record naming a pointer, or one that still carries an EFI volume request, is refused.
+   The store is written back to the object store on write and at stop, and a terminated instance's store is removed by a cluster-wide reaper.
 3. **Volume Mount**: Publishes each volume through the EBS provider contract (`PublishVolume` on `ebs.provider.v1.{node}.mount`), receives its export
 4. **QEMU Launch**: Builds and executes QEMU command with NBD-backed disks
 5. **QMP Monitoring**: Establishes QEMU Machine Protocol connection for VM management

docs/admin/uefi-variable-store/README.md

@@ -0,0 +1,148 @@
+---
+title: "UEFI Variable Store"
+seoTitle: "UEFI Variable Store Durability and Recovery — Spinifex Docs"
+description: "Where a UEFI instance's variable store lives, how much a node loss can lose, how owed write-backs recover, and which firmware differences refuse a launch."
+category: "Admin"
+tags:
+  - uefi
+  - firmware
+  - instances
+  - storage
+resources:
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "Host and Guest Lifecycle"
+    url: "/docs/host-lifecycle"
+---
+
+# UEFI Variable Store
+
+> Where a UEFI instance keeps its firmware variables, how durable they are, and what an operator sees when something goes wrong.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Where the Store Lives](#where-the-store-lives)
+- [Durability](#durability)
+- [Owed Write-Backs](#owed-write-backs)
+- [Starting on Another Node](#starting-on-another-node)
+- [Firmware Compatibility](#firmware-compatibility)
+- [Instances Refused at Launch](#instances-refused-at-launch)
+- [Terminate](#terminate)
+- [Inspecting Stores](#inspecting-stores)
+- [Metrics](#metrics)
+
+---
+
+## Overview
+
+An instance whose boot mode is `uefi` or `uefi-preferred` boots through UEFI firmware, which keeps its variables (`BootOrder`, `BootNext`, secure-boot state) in a VARS region. Spinifex gives each such instance its own variable store, so those variables survive a reboot, a stop and start, and a start on a different node. A direct-boot microvm boots without UEFI and has no store.
+
+RunInstances seeds the store from the launching node's VARS template. From then on the node running the instance writes the guest's changes back to the platform object store, and every later start boots from the last change that landed there.
+
+## Where the Store Lives
+
+A store has three parts.
+
+| Part | Where | What it holds |
+|---|---|---|
+| Store objects | The platform object store, in the bucket set as the node's Predastore bucket, under `uefi/<instance-id>/` | The VARS region, gzip-compressed, one object per write. Each write goes to a fresh key, so no write replaces an object a record points at. Each object's metadata names the instance, the writer node, the source (`template` or `write-back`) and the firmware it was written under. |
+| Store record | KV bucket `spinifex-uefi-stores`, keyed by instance ID | The instance and account, the state (`live` or `deleting`), the claim generation that fences writes, and the pointer: the one object the instance boots from, with the firmware it was written under, the writer node and the write time. |
+| Local copy | `<data-dir>/uefi/<instance-id>/` on the node running the instance (`/var/lib/spinifex/uefi/<instance-id>/` by default) | `vars.fd`, the file QEMU opens as pflash unit 1, and `seeded.json`, which records the key the copy came from or last landed at, the claim generation, the firmware, and whether a write-back is owed. |
+
+The record outlives the instance record, so the store's deletion obligation is not lost when a terminated instance record expires. The bucket is never recreated empty if it goes missing, because an empty bucket would report every store as having no obligations.
+
+Only one node may write a store at a time. That node holds the store's claim, `uefi.<instance-id>` in the cluster's sync bucket, and a write moves the pointer only while the record still carries that claim's generation. A node that takes the claim stamps the record with its own generation before it boots, so a write from the previous holder is refused rather than landing over newer variables.
+
+## Durability
+
+The running node watches `vars.fd` for writes. Once the guest changes a variable, the change is written back 2 seconds after the last write, or 30 seconds after the first write in the window, whichever is sooner. A guest that writes continuously therefore still lands a write-back every 30 seconds. A node that cannot watch the file polls it every 10 seconds instead, and logs `Cannot watch the UEFI variable store; polling it for changes instead`.
+
+A write-back puts the region at a fresh key (bounded at 10 seconds) and then moves the store record's pointer to it (bounded at 10 seconds). A write-back that fails is retried 30 seconds later, and the changes stay owed until one lands.
+
+When an instance stops, the node writes the store back once QEMU has exited and before the instance's volumes unmount, then releases the claim.
+
+**What a node loss can lose:** the variable changes made since the last write-back that landed. While the object store is reachable that is at most the 30-second window plus the write in flight. While write-backs are failing, it is every change since the last one that landed, however long the failures last. The node's local copy still holds those changes, and they are recovered if the node comes back before another node takes the store over (see below).
+
+## Owed Write-Backs
+
+A write-back is owed whenever `vars.fd` may hold changes no object has. The node records that in `seeded.json` as `write_back_owed` when a window opens and clears it when the write lands. A stop whose write-back fails keeps the claim, marks the copy owed and logs `The stop could not write the UEFI variable store back; this node keeps its claim and retries`.
+
+Each node runs a `uefi-write-back` reaper on the garbage-collection sweep, every 2 minutes while KV is healthy. For each local copy whose instance is not running on the node, it:
+
+- writes an owed copy back and releases the claim once it lands (`trigger=reaper` on the write-back metric);
+- releases the claim over a clean copy once the stream leader confirms the record still points at the object the copy came from;
+- drops an owed copy whose claim another node now holds, or whose record a later claim has stamped, logging `vm/gc: dropping an owed UEFI variable store write-back; its changes are lost` and counting `mulga.vm.uefi_store.write_backs_lost`;
+- removes the copy of a store that is being deleted.
+
+The same reaper starts a writer for a running guest that has none, such as one whose reconnect after a daemon restart could not read the claim. A guest that outlives a daemon restart has its writer resumed with the window open, so any change the restart missed is written back.
+
+The reaper also deletes objects no pointer names under each prefix its write-backs touched, keeping the pointer's object and anything younger than 10 minutes.
+
+An operator does nothing to recover an owed write-back on a node that is up: the reaper retries until it lands. If it keeps failing, the node's log carries `Failed to write the UEFI variable store back` with the error, and the fix is whatever stops the node writing to the object store.
+
+## Starting on Another Node
+
+A start on a node that does not hold the claim takes it if it is free. If another node holds it, the start is refused while that node's heartbeat is fresh (under 60 seconds old) or cannot be read, because that node may still land newer variables. StartInstances returns `InsufficientInstanceCapacity` for that refusal, and the node logs `Refusing to start: another live node holds the UEFI variable store claim`. This is what an operator sees when a stopped instance's owed write-back has not landed yet: start it on the node that holds the claim, or let the reaper land the write-back and release the claim.
+
+Once the holder's heartbeat is 60 seconds old or absent, the starting node takes the claim over and logs `Took over the UEFI variable store claim from a node whose heartbeat has stopped; variable changes it never wrote back are lost`. If the old node comes back, its reaper finds the claim held elsewhere and drops its owed copy.
+
+A start on the node that holds the claim under its copy's generation boots that copy, because it may carry changes no object has.
+
+## Firmware Compatibility
+
+A store records the firmware it was written under: architecture, VARS size and the SHA-256 of the CODE blob. A node's firmware is the first installed pair from its firmware candidates (`/usr/share/OVMF/OVMF_CODE_4M.fd` and `OVMF_VARS_4M.fd` first on x86_64, `/usr/share/AAVMF/AAVMF_CODE.fd` and `AAVMF_VARS.fd` first on arm64).
+
+| Difference | Result |
+|---|---|
+| Architecture | The launch fails before the claim is taken, so the store is left untouched. |
+| VARS size | The launch fails before the claim is taken, so the store is left untouched. pflash needs a byte-exact region. |
+| CODE hash only | The instance boots. The node logs `UEFI variable store was written under a different CODE build; booting it` and counts `mulga.vm.uefi_store.firmware_mismatches` with `kind=code`. |
+
+A refused launch logs `UEFI variable store firmware does not match this node; refusing the launch` naming both firmwares, counts the mismatch with `kind=arch` or `kind=vars_size`, and records a `Server.InternalError` state reason naming both firmwares.
+
+Every node of an architecture should carry the same firmware package. A node with a different VARS size cannot boot any store written elsewhere, and a node with a different CODE build boots stores with a counted warning on every launch.
+
+## Instances Refused at Launch
+
+A UEFI instance is refused before anything about it changes when:
+
+- it still carries an EFI volume request (`<root>-efi`), which no node boots;
+- it has no store record, its record is `deleting`, or its record names no pointer;
+- its record cannot be read as valid.
+
+These starts fail with `Server.InternalError`, and the node logs `refusing to launch a UEFI instance without exactly one variable store` or `Refusing to launch a UEFI instance without a variable store` with the reason. A start whose store object is missing or does not decode to the node's VARS size also fails, and retrying does not help.
+
+## Terminate
+
+Terminate moves the store record to `deleting`, releases a claim the node holds, and removes the node's local copy. It does not delete objects. A cluster-wide `uefi-store` reaper deletes every object under the instance's prefix, then the claim, then the record, and only for an instance whose terminate is decided. It completes a terminate whose mark failed from the terminated instance record. A `deleting` record whose instance is not terminating is left for an operator and logged as `uefi-store: refusing to delete the store of an instance that is not terminating`.
+
+## Inspecting Stores
+
+`spx admin volumes orphans` reports UEFI variable stores nothing owns before it reports orphaned volumes:
+
+- `uefi/i-*/` prefixes with neither an instance record nor a store record;
+- `uefi/ami-*/` prefixes with no AMI document;
+- prefixes named for neither an instance nor an image;
+- store records that are not `deleting` and have no instance record.
+
+It only reads, and attaches to the buckets rather than creating them. It never deletes: an absent instance record is not proof of a terminate, so settling each one is an operator decision.
+
+To read one instance's store record:
+
+```bash
+nats kv get spinifex-uefi-stores i-0123456789abcdef0
+```
+
+## Metrics
+
+| Metric | Meaning |
+|---|---|
+| `mulga.vm.uefi_store.seeds` | UEFI launches by where the store came from: `source=local` (the node's copy) or `source=object`. |
+| `mulga.vm.uefi_store.write_backs` | Write-backs by `outcome` (`landed`, `failed`, `fenced`, `deleting`) and `trigger` (`change`, `stop`, `reaper`). |
+| `mulga.vm.uefi_store.write_back_skips` | Writers stopped without writing back, by the path that stopped them. |
+| `mulga.vm.uefi_store.write_backs_lost` | Owed write-backs dropped because another node took the claim over. |
+| `mulga.vm.uefi_store.pruned_objects` | Superseded store objects a prune deleted. |
+| `mulga.vm.uefi_store.firmware_mismatches` | Launches whose store was written under different firmware, by `kind`. Only `kind=code` boots. |
+| `mulga.vm.uefi_store.claim_refusals` | Starts refused because another node holds the claim and may be live, by `reason`. |
+| `mulga.vm.uefi_store.claim_takeovers` | Claims taken over from a node whose heartbeat had stopped. |

docs/testing/e2e-coverage.md

@@ -6,7 +6,7 @@ Suites live in `tests/e2e/<name>/` and build to `tests/e2e/_bin/<name>.test`. Ev
 
 | Set | Suites |
 | --- | --- |
-| `E2E_SUITES_SINGLE` | `single iam cert eks ecs storagegrowth partialblock rds quota route53 storagefault` |
+| `E2E_SUITES_SINGLE` | `single iam cert eks ecs storagegrowth partialblock rds quota route53 uefistore storagefault` |
 | `E2E_SUITES_MULTI` | `multinode lb cert quota route53 kvquorum lbrecovery instancerecovery storagefault` |
 | `E2E_SUITES_NIGHTLY_SINGLE` | `single cert iam` |
 | `E2E_SUITES_NIGHTLY_MULTI` | `multinode cert lb` |
@@ -144,6 +144,11 @@ The create/describe/connect path against a live cluster: a DB VM that actually b
 - **`storagegrowth`** — volume growth and image pull. Requires `external_mode=pool`; skips on a nat env, which has no EIP and therefore no guest SSH.
 - **`reboot`** — a slow guest waited for and keeping what it wrote, and a wedged guest forced off and relaunched once its budget expires. Proves a real reboot by requiring a *new* boot ID.
 - **`diskperf`** — guest disk performance gate, pinned fio profile on a dedicated blank volume. Also requires `external_mode=pool` for guest SSH.
+- **`uefistore`** — a real UEFI guest's variable writes reaching the variable store and surviving what it is built to survive. Single-node, and it runs on the node: it restarts that node's daemon and reads the store with the node's own Predastore credentials. Its three tests share one Ubuntu guest, which writes non-volatile variables (attributes 0x7) through efivarfs, and each landing is proved by fetching the object the store record points at and finding the variable's UTF-16 name and its run-unique value in the VARS region.
+  - `TestUEFIGuestVariableSurvivesStopStart`: the pointer moves to a fresh object holding the variable within 60 seconds (the documented bound is 30), the stop leaves the store holding it, and the guest reads it back after the start with a new boot ID.
+  - `TestUEFIWriterResumesAfterDaemonRestart`: a variable lands, `spinifex-daemon` restarts under the guest (whose boot ID must not change), a second variable lands, and both survive a stop and start.
+  - **`TestS7_UEFIStorePruneNeverDeletesThePointedObject` is the ADR-0003:S7 evidence for the prune.** It lands three versions of one variable and requires all three objects to remain, then polls the record and the prefix every 10 seconds until the prefix holds one object, failing if the pointed object or any object under 10 minutes old disappears. The prune must finish on an instance older than the 10-minute prune age, and the last version survives a final stop and start.
+  - Host reboot and node loss are not covered: both need host control or a second node that this suite does not have.
 
 ### Other
 

internal/testkit/recordingstore/recordingstore.go

@@ -1,8 +1,9 @@
 // Package recordingstore provides an objectstore.ObjectStore fake that records
-// every key it is asked to read and every prefix it is asked to list. It exists
-// so tests can assert an access pattern rather than only an answer: that an
-// account-scoped listing never touched another account's prefix, which an
-// in-memory filter over a whole-cluster read would satisfy just as well.
+// every key it is asked to read, head, write or delete and every prefix it is
+// asked to list. It exists so tests can assert an access pattern rather than
+// only an answer: that an account-scoped listing never touched another
+// account's prefix, which an in-memory filter over a whole-cluster read would
+// satisfy just as well.
 package recordingstore
 
 import (
@@ -16,14 +17,17 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 )
 
-// Store wraps a MemoryObjectStore, appending to Gets and ListPrefixes before
-// delegating. Both slices are safe to read once the call under test returns.
+// Store wraps a MemoryObjectStore, recording each call's key or prefix before
+// delegating. The recordings are safe to read once the call under test returns.
 type Store struct {
 	*objectstore.MemoryObjectStore
 
 	mu           sync.Mutex
 	gets         []string
 	listPrefixes []string
+	heads        []string
+	puts         []string
+	deletes      []string
 }
 
 // New returns a recording store backed by an empty in-memory store.
@@ -47,6 +51,27 @@ func (s *Store) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input)
 	return s.MemoryObjectStore.ListObjectsV2(ctx, input)
 }
 
+func (s *Store) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	s.record(&s.heads, aws.StringValue(input.Key))
+	return s.MemoryObjectStore.HeadObject(ctx, input)
+}
+
+func (s *Store) PutObject(ctx context.Context, input *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
+	s.record(&s.puts, aws.StringValue(input.Key))
+	return s.MemoryObjectStore.PutObject(ctx, input)
+}
+
+func (s *Store) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+	s.record(&s.deletes, aws.StringValue(input.Key))
+	return s.MemoryObjectStore.DeleteObject(ctx, input)
+}
+
+func (s *Store) record(into *[]string, key string) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	*into = append(*into, key)
+}
+
 // Gets returns every key read so far.
 func (s *Store) Gets() []string {
 	s.mu.Lock()
@@ -61,12 +86,33 @@ func (s *Store) ListPrefixes() []string {
 	return append([]string(nil), s.listPrefixes...)
 }
 
+// Heads returns every key headed so far.
+func (s *Store) Heads() []string {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return append([]string(nil), s.heads...)
+}
+
+// Puts returns every key written so far.
+func (s *Store) Puts() []string {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return append([]string(nil), s.puts...)
+}
+
+// Deletes returns every key deleted so far.
+func (s *Store) Deletes() []string {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return append([]string(nil), s.deletes...)
+}
+
 // Reset drops what has been recorded, so a test can seed fixtures and then
 // assert only on the call it is actually exercising.
 func (s *Store) Reset() {
 	s.mu.Lock()
 	defer s.mu.Unlock()
-	s.gets, s.listPrefixes = nil, nil
+	s.gets, s.listPrefixes, s.heads, s.puts, s.deletes = nil, nil, nil, nil, nil
 }
 
 // TouchedPrefix reports whether any recorded get or listing named a key under

spinifex/domains/admission/quota/quota.go

@@ -22,10 +22,10 @@ const ReconcileInterval = 30 * time.Second
 // dimension; the live-counted dimensions need no stored state.
 const KVBucketAccountUsage = "spinifex-account-usage"
 
-// KVBucketQuotaReconcile is the leader-lock bucket for the gateway quota
-// reconcile. It is deliberately distinct from vpcd's network-reconcile lock so
-// the two independent loops never share one mutex and block each other.
-const KVBucketQuotaReconcile = "spinifex-quota-reconcile"
+// KeyReconcile is the sync-bucket leader lease for the gateway quota reconcile.
+// It is deliberately distinct from vpcd's network-reconcile lease so the two
+// independent loops never share one mutex and block each other.
+const KeyReconcile = "quota.reconcile"
 
 // KVBucketAccountQuota holds the per-account limit overrides, keyed by
 // accountID. It is separate from the accounts bucket so a quota write never

spinifex/domains/dns/delegation.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	nsconfig "github.com/mulgadc/northstar/pkg/config"
-	"github.com/nats-io/nats.go"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 
 	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
 )
@@ -48,7 +48,7 @@ func (w *Writer) PlatformDelegation(name string) (servers []string, zone string,
 // Delegate writes, or with remove withdraws, an NS set in the base zone that
 // delegates each name to this cluster's nameservers. It takes the zone-writer
 // election first, so it never races the reconcile's own writes to the zone.
-func (w *Writer) Delegate(ctx context.Context, nc *nats.Conn, holder string, names []string, remove bool) (*ChangeResult, error) {
+func (w *Writer) Delegate(ctx context.Context, lease *kvsync.Lease, names []string, remove bool) (*ChangeResult, error) {
 	if !w.Enabled() {
 		return nil, errors.New("northstar is not configured on this node")
 	}
@@ -77,7 +77,7 @@ func (w *Writer) Delegate(ctx context.Context, nc *nats.Conn, holder string, nam
 	if len(batch.Changes) == 0 {
 		return &ChangeResult{}, nil
 	}
-	release, err := w.awaitWriterElection(ctx, nc, holder)
+	release, err := w.awaitWriterElection(ctx, lease)
 	if err != nil {
 		return nil, err
 	}
@@ -85,19 +85,19 @@ func (w *Writer) Delegate(ctx context.Context, nc *nats.Conn, holder string, nam
 	return w.ApplyBatch(batch)
 }
 
-func (w *Writer) awaitWriterElection(ctx context.Context, nc *nats.Conn, holder string) (func(), error) {
-	if nc == nil {
+func (w *Writer) awaitWriterElection(ctx context.Context, lease *kvsync.Lease) (func(), error) {
+	if lease == nil {
 		return func() {}, nil
 	}
 	tick := time.NewTicker(delegateLockPoll)
 	defer tick.Stop()
 	for {
-		if release, ok := reconcilelock.AcquireLeader(ctx, nc, KVBucketDNSReconcile, holder); ok {
+		if release, ok := reconcilelock.AcquireLeader(ctx, lease); ok {
 			return release, nil
 		}
 		select {
 		case <-ctx.Done():
-			return nil, fmt.Errorf("another node is writing the zone: %w", ctx.Err())
+			return nil, fmt.Errorf("another writer has the zone: %w", ctx.Err())
 		case <-tick.C:
 		}
 	}

spinifex/domains/dns/delegation_test.go

@@ -14,7 +14,7 @@ import (
 
 func TestDelegateWritesAndWithdrawsPlatformDelegations(t *testing.T) {
 	w, _ := newMultiNodeTestWriter(t)
-	_, err := w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net", "B.r53e2e.spx3.net."}, false)
+	_, err := w.Delegate(t.Context(), nil, []string{"a.r53e2e.spx3.net", "B.r53e2e.spx3.net."}, false)
 	require.NoError(t, err)
 
 	for _, name := range []string{"a.r53e2e.spx3.net.", "b.r53e2e.spx3.net"} {
@@ -29,7 +29,7 @@ func TestDelegateWritesAndWithdrawsPlatformDelegations(t *testing.T) {
 	assert.True(t, ok, "a name below the base domain is decided by the base zone")
 	assert.Empty(t, servers)
 
-	_, err = w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net"}, true)
+	_, err = w.Delegate(t.Context(), nil, []string{"a.r53e2e.spx3.net"}, true)
 	require.NoError(t, err)
 	servers, _, _, err = w.PlatformDelegation("a.r53e2e.spx3.net")
 	require.NoError(t, err)
@@ -43,11 +43,11 @@ func TestDelegateWritesAndWithdrawsPlatformDelegations(t *testing.T) {
 // servers written while the cluster published a larger set.
 func TestDelegateRemoveWithdrawsServersFromAnEarlierSet(t *testing.T) {
 	w, _ := newMultiNodeTestWriter(t)
-	_, err := w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net"}, false)
+	_, err := w.Delegate(t.Context(), nil, []string{"a.r53e2e.spx3.net"}, false)
 	require.NoError(t, err)
 
 	w.nameservers = w.nameservers[:1]
-	_, err = w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net"}, true)
+	_, err = w.Delegate(t.Context(), nil, []string{"a.r53e2e.spx3.net"}, true)
 	require.NoError(t, err)
 	servers, _, _, err := w.PlatformDelegation("a.r53e2e.spx3.net")
 	require.NoError(t, err)
@@ -66,7 +66,7 @@ func TestPlatformDelegationIgnoresNamesOutsideTheBaseDomain(t *testing.T) {
 func TestDelegateRefusesPlatformAndForeignNames(t *testing.T) {
 	w, _ := newMultiNodeTestWriter(t)
 	for _, name := range []string{"ns1.spx3.net", "x.elb.spx3.net", "compute.spx3.net", "example.com", "spx3.net"} {
-		_, err := w.Delegate(t.Context(), nil, "node1", []string{"ok.spx3.net", name}, false)
+		_, err := w.Delegate(t.Context(), nil, []string{"ok.spx3.net", name}, false)
 		require.Error(t, err, name)
 	}
 	servers, _, _, err := w.PlatformDelegation("ok.spx3.net")
@@ -76,22 +76,45 @@ func TestDelegateRefusesPlatformAndForeignNames(t *testing.T) {
 
 func TestDelegateWaitsForTheZoneWriterElection(t *testing.T) {
 	w, _ := newMultiNodeTestWriter(t)
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	prev := delegateLockPoll
 	delegateLockPoll = 10 * time.Millisecond
 	t.Cleanup(func() { delegateLockPoll = prev })
 
-	release, ok := reconcilelock.AcquireLeader(t.Context(), nc, KVBucketDNSReconcile, "node2")
+	release, ok := reconcilelock.AcquireLeader(t.Context(), dnsLease(js, "node2"))
 	require.True(t, ok)
 	ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
 	defer cancel()
-	_, err := w.Delegate(ctx, nc, "node1", []string{"a.spx3.net"}, false)
-	require.ErrorContains(t, err, "another node is writing the zone")
+	node1 := dnsLease(js, "node1")
+	_, err := w.Delegate(ctx, node1, []string{"a.spx3.net"}, false)
+	require.ErrorContains(t, err, "another writer has the zone")
 
 	release()
-	_, err = w.Delegate(t.Context(), nc, "node1", []string{"a.spx3.net"}, false)
+	_, err = w.Delegate(t.Context(), node1, []string{"a.spx3.net"}, false)
 	require.NoError(t, err)
 	servers, _, _, err := w.PlatformDelegation("a.spx3.net")
 	require.NoError(t, err)
 	assert.Len(t, servers, 2)
 }
+
+// The reconcile pass and a delegation on one node share the DNS lease, so a
+// delegation waits for this node's own pass as it would for another node's.
+func TestDelegateWaitsForThisNodesOwnReconcilePass(t *testing.T) {
+	w, _ := newMultiNodeTestWriter(t)
+	_, _, js := testutil.StartTestJetStream(t)
+	prev := delegateLockPoll
+	delegateLockPoll = 10 * time.Millisecond
+	t.Cleanup(func() { delegateLockPoll = prev })
+
+	shared := dnsLease(js, "node1")
+	release, ok := reconcilelock.AcquireLeader(t.Context(), shared)
+	require.True(t, ok, "the reconcile pass holds the lease")
+	ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
+	defer cancel()
+	_, err := w.Delegate(ctx, shared, []string{"a.spx3.net"}, false)
+	require.ErrorContains(t, err, "another writer has the zone")
+
+	release()
+	_, err = w.Delegate(t.Context(), shared, []string{"a.spx3.net"}, false)
+	require.NoError(t, err)
+}

spinifex/domains/dns/reconcile.go

@@ -11,7 +11,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/reconciler"
-	"github.com/nats-io/nats.go"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 )
 
 // DefaultReconcileInterval is how often the drift backstop rebuilds managed
@@ -33,6 +33,12 @@ const DefaultReconcileInterval = reconciler.DefaultResync
 // costs a cycle for nothing.
 const DefaultComputeTimeout = 2 * time.Minute
 
+// NewReconcileLease returns the zone writer's election lease, shared by the
+// drift reconcile and Delegate so the two never write the zone together.
+func NewReconcileLease(store kvsync.KVStore, holder string) *kvsync.Lease {
+	return reconcilelock.NewLeaderLease(store, KeyReconcile, holder)
+}
+
 // DesiredFunc returns the full desired managed record set built from the live
 // resource inventory across all tenants. The daemon supplies it by enumerating
 // instances, load balancers, and EKS clusters.
@@ -102,7 +108,8 @@ type Reconciler struct {
 	// lands in. Distinct from both baseDomain and internalDomain, so pruning it
 	// needs its own read the same way internalDomain does for EC2.
 	serviceZone string
-	nc          *nats.Conn
+	// lease elects the one node that writes the zones; nil skips the election.
+	lease *kvsync.Lease
 	// writer applies the converged batch. The reconcile owns the zone object on
 	// the node that won the election, so the batch never leaves this process.
 	writer   *Writer
@@ -111,7 +118,6 @@ type Reconciler struct {
 	// computeTimeout bounds how long one pass spends building the desired set.
 	// A field rather than a constant so a test can shrink it.
 	computeTimeout time.Duration
-	holder         string
 	sources        []reconciler.Source
 }
 
@@ -121,18 +127,15 @@ type Reconciler struct {
 // changes should wake the loop. They are optional: with none supplied the loop
 // falls back to the interval alone, which is the behaviour that predates the
 // watch.
-func NewReconciler(cfg *config.Config, cluster *config.ClusterConfig, nc *nats.Conn, writer *Writer, desired DesiredFunc, sources ...reconciler.Source) *Reconciler {
+func NewReconciler(cfg *config.Config, cluster *config.ClusterConfig, lease *kvsync.Lease, writer *Writer, desired DesiredFunc, sources ...reconciler.Source) *Reconciler {
 	r := &Reconciler{
-		nc:             nc,
+		lease:          lease,
 		writer:         writer,
 		desired:        desired,
 		interval:       DefaultReconcileInterval,
 		computeTimeout: DefaultComputeTimeout,
 		sources:        sources,
 	}
-	if cfg != nil {
-		r.holder = cfg.Node
-	}
 	zoneCfg, ok := zoneS3Config(cfg)
 	if !ok || desired == nil || !writer.Enabled() {
 		return r
@@ -166,25 +169,24 @@ func (r *Reconciler) Run(ctx context.Context) {
 		// No revisit deadline: the desired set is a function of the buckets
 		// being watched, so a change is the only reason to run early.
 		Reconcile: func(ctx context.Context) (time.Duration, error) {
-			r.reconcileOnce(ctx)
-			return 0, nil
+			return r.reconcileOnce(ctx), nil
 		},
 		Resync: r.interval,
 	})
 }
 
 // reconcileOnce computes the converging batch and applies it, on whichever node
-// wins this cycle's election. The read that finds the stale records and the
+// wins this cycle's election, and returns the election's revisit when it lost. The read that finds the stale records and the
 // write that removes them now happen in one process, so a record re-added
 // between them is no longer deleted by a batch that predates it.
-func (r *Reconciler) reconcileOnce(ctx context.Context) {
+func (r *Reconciler) reconcileOnce(ctx context.Context) time.Duration {
 	if !r.enabled {
-		return
+		return 0
 	}
-	if r.nc != nil {
-		release, elected := reconcilelock.AcquireLeader(ctx, r.nc, KVBucketDNSReconcile, r.holder)
+	if r.lease != nil {
+		release, retry, elected := reconcilelock.ElectPass(ctx, r.lease)
 		if !elected {
-			return
+			return retry
 		}
 		defer release()
 	}
@@ -193,18 +195,19 @@ func (r *Reconciler) reconcileOnce(ctx context.Context) {
 		// A corrupt zone is handled in readZone; anything reaching here left the
 		// desired state unapplied, so surface it rather than burying it in WARN.
 		slog.Error("dns reconcile: compute batch failed, retrying next cycle", "error", err)
-		return
+		return 0
 	}
 	if len(batch) == 0 {
-		return
+		return 0
 	}
 	slog.Debug("dns reconcile: converging", "changes", len(batch))
 	res, err := r.writer.ApplyBatch(&ChangeBatch{Changes: batch})
 	if err != nil {
 		slog.Warn("dns reconcile: apply failed, retrying next cycle", "changes", len(batch), "error", err)
-		return
+		return 0
 	}
 	slog.Debug("dns reconcile: converged", "changes", res.Applied, "zones", res.Zones)
+	return 0
 }
 
 // computeBatchBounded runs computeBatch with a deadline, so a pass that never

spinifex/domains/dns/reconcile_election_test.go

@@ -8,41 +8,50 @@ import (
 
 	"github.com/mulgadc/spinifex/internal/testkit"
 	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
-	"github.com/nats-io/nats.go"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-// electingReconcilerFor is reconcilerFor with a live connection, so a pass runs
-// the election instead of skipping it.
-func electingReconcilerFor(w *Writer, nc *nats.Conn, holder string, desired func() DesiredSet) *Reconciler {
+// electingReconcilerFor is reconcilerFor with a lease, so a pass runs the
+// election instead of skipping it.
+func electingReconcilerFor(w *Writer, lease *kvsync.Lease, desired func() DesiredSet) *Reconciler {
 	r := reconcilerFor(w, desired)
-	r.nc = nc
-	r.holder = holder
+	r.lease = lease
 	return r
 }
 
+// dnsLease returns holder's DNS lease on a store of its own, standing in for
+// a separate process.
+func dnsLease(js jetstream.JetStream, holder string) *kvsync.Lease {
+	return NewReconcileLease(kvstore.OpenSyncBucket(js), holder)
+}
+
 // The zone lock is gone, and what replaces it is that a zone has one writer:
 // the node holding the reconcile lease. A node that loses the election must
 // therefore not touch the object at all — if it did, two nodes could be
 // read-modify-writing the same zone with nothing serialising them.
 func TestReconcileOnce_ALoserOfTheElectionDoesNotWriteTheZone(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	w, objects := newTestWriter(t)
 	before := objects[testBase+".toml"]
 	require.NotEmpty(t, before)
 
 	// A peer holds the lease for the whole pass, so this node cannot win it.
-	release, elected := reconcilelock.AcquireLeader(t.Context(), nc, KVBucketDNSReconcile, "peer")
+	release, elected := reconcilelock.AcquireLeader(t.Context(), dnsLease(js, "peer"))
 	require.True(t, elected, "the peer has to hold the lease or this test proves nothing")
 
-	r := electingReconcilerFor(w, nc, "node1", func() DesiredSet {
+	r := electingReconcilerFor(w, dnsLease(js, "node1"), func() DesiredSet {
 		return DesiredSet{Changes: []Change{upsert("lb-1.elb."+testBase, "10.200.1.9")}}
 	})
-	r.reconcileOnce(t.Context())
+	revisit := r.reconcileOnce(t.Context())
 
 	assert.Equal(t, before, objects[testBase+".toml"],
 		"a node that did not win the election must leave the zone object exactly as it found it")
+	assert.Positive(t, revisit, "the loser asks to retry so a dead holder is taken over")
+	assert.LessOrEqual(t, revisit, reconcilelock.LeaderTTL/3, "the retry comes within a renewal interval")
 
 	release()
 }
@@ -50,10 +59,10 @@ func TestReconcileOnce_ALoserOfTheElectionDoesNotWriteTheZone(t *testing.T) {
 // The counterpart, so the test above is not passing because the pass was broken
 // for some other reason: with the lease free, the same reconciler writes.
 func TestReconcileOnce_TheWinnerOfTheElectionWritesTheZone(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	w, objects := newTestWriter(t)
 
-	r := electingReconcilerFor(w, nc, "node1", func() DesiredSet {
+	r := electingReconcilerFor(w, dnsLease(js, "node1"), func() DesiredSet {
 		return DesiredSet{Changes: []Change{upsert("lb-1.elb."+testBase, "10.200.1.9")}}
 	})
 	r.reconcileOnce(t.Context())

spinifex/domains/dns/reconcile_test.go

@@ -224,7 +224,7 @@ func TestReconciler_RunReturnsWhenDisabled(t *testing.T) {
 // path of Run: the startup pass must fire before any watch or resync activity,
 // and cancelling ctx must still let the loop return.
 func TestReconciler_RunPerformsStartupPassThenStopsOnCancel(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	endpoint, _ := fakeS3(t, "northstar")
 
 	var passes atomic.Int64
@@ -235,8 +235,7 @@ func TestReconciler_RunPerformsStartupPassThenStopsOnCancel(t *testing.T) {
 			Endpoint: endpoint, Bucket: "northstar", Region: "us-east-1",
 			AccessKey: "SYSTEM", SecretKey: "SYSTEMSECRET",
 		},
-		nc:     nc,
-		holder: "test-node",
+		lease: dnsLease(js, "test-node"),
 		desired: func() DesiredSet {
 			passes.Add(1)
 			return DesiredSet{}
@@ -268,7 +267,7 @@ func TestReconciler_RunPerformsStartupPassThenStopsOnCancel(t *testing.T) {
 // acquirable afterwards, not merely on the call returning, since it is the
 // leadership and not the pass that the rest of the cluster waits on.
 func TestReconcileOnce_AStuckPassSurrendersLeadership(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	endpoint, _ := fakeS3(t, "northstar")
 
 	blocked := make(chan struct{})
@@ -281,8 +280,7 @@ func TestReconcileOnce_AStuckPassSurrendersLeadership(t *testing.T) {
 			Endpoint: endpoint, Bucket: "northstar", Region: "us-east-1",
 			AccessKey: "SYSTEM", SecretKey: "SYSTEMSECRET",
 		},
-		nc:             nc,
-		holder:         "stuck-node",
+		lease:          dnsLease(js, "stuck-node"),
 		computeTimeout: 100 * time.Millisecond,
 		desired: func() DesiredSet {
 			<-blocked
@@ -302,7 +300,7 @@ func TestReconcileOnce_AStuckPassSurrendersLeadership(t *testing.T) {
 		t.Fatal("reconcileOnce never returned: a stuck pass still holds the lease forever")
 	}
 
-	release, elected := reconcilelock.AcquireLeader(t.Context(), nc, KVBucketDNSReconcile, "next-node")
+	release, elected := reconcilelock.AcquireLeader(t.Context(), dnsLease(js, "next-node"))
 	require.True(t, elected, "another node must be able to take the next cycle")
 	release()
 }

spinifex/domains/dns/types.go

@@ -11,9 +11,9 @@ const (
 	// PUT so every northstar instance reloads just that zone immediately, instead
 	// of waiting for the next S3 sync poll. No queue group: all servers consume.
 	SubjectZoneReload = "dns.zone.reload"
-	// KVBucketDNSReconcile elects the one node that writes the zones for a drift
-	// cycle. Its own bucket, so the election never blocks another reconcile loop.
-	KVBucketDNSReconcile = "spinifex-dns-reconcile"
+	// KeyReconcile elects the one node that writes the zones for a drift cycle.
+	// Its own key, so the election never blocks another reconcile loop.
+	KeyReconcile = "dns.reconcile"
 	// PrivateZone is the fixed AWS-parity private DNS zone (IMDS synthHostname).
 	PrivateZone = "compute.internal"
 	// DefaultTTL is applied when a change omits a TTL.

spinifex/domains/ec2/instance/provider_efi_volume_test.go

@@ -1,126 +0,0 @@
-package instance
-
-import (
-	"bytes"
-	"context"
-	"testing"
-
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// varsTemplateBytes stands in for a firmware VARS template. The real one is
-// 540,672 bytes, but nothing here depends on the size beyond it being exact.
-func varsTemplateBytes() []byte { return bytes.Repeat([]byte{0xAB}, 4096) }
-
-func providerEFIService(t *testing.T, provider ebsprovider.EBSProvider) *InstanceServiceImpl {
-	t.Helper()
-	return providerRootVolumeService(t, provider, rootVolumeAMILoader(), objectstore.NewMemoryObjectStore())
-}
-
-// TestCreateEFIVolume_Provider_SeedsFromTemplate is the core of the EFI
-// decouple: the variable store is allocated through the provider with the
-// template as seed bytes, so no control-plane engine is built for it.
-func TestCreateEFIVolume_Provider_SeedsFromTemplate(t *testing.T) {
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true})
-	svc := providerEFIService(t, provider)
-	template := varsTemplateBytes()
-
-	require.NoError(t, svc.createEFIVolumeViaProvider(context.Background(), "vol-efiseed001-efi", template))
-
-	volume, err := provider.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{
-		Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-efiseed001-efi",
-	})
-	require.NoError(t, err)
-	assert.Equal(t, int64(len(template)), volume.CapacityBytes,
-		"the store must be sized byte-exactly to the VARS template")
-
-	seed, ok := provider.SeedData("vol-efiseed001-efi")
-	require.True(t, ok)
-	assert.Equal(t, template, seed, "the template must reach the provider as seed bytes")
-}
-
-// A provider that rounds capacity to a GiB boundary produces a VARS region
-// pflash refuses, which would surface as an unbootable guest rather than a
-// failed launch. The mismatch has to fail the launch instead.
-func TestCreateEFIVolume_Provider_CapacityMismatchFails(t *testing.T) {
-	provider := &roundingProvider{
-		EBSProvider: ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true}),
-	}
-	svc := providerEFIService(t, provider)
-
-	err := svc.createEFIVolumeViaProvider(context.Background(), "vol-efiround01-efi", varsTemplateBytes())
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-// A relaunch reissues the same create. Reseeding would overwrite the BootOrder
-// the guest wrote on its first boot, so the second call must be a no-op.
-func TestCreateEFIVolume_Provider_RepeatDoesNotReseed(t *testing.T) {
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true})
-	svc := providerEFIService(t, provider)
-	ctx := context.Background()
-	template := varsTemplateBytes()
-
-	require.NoError(t, svc.createEFIVolumeViaProvider(ctx, "vol-efirepeat1-efi", template))
-	require.NoError(t, svc.createEFIVolumeViaProvider(ctx, "vol-efirepeat1-efi", template))
-
-	seed, ok := provider.SeedData("vol-efirepeat1-efi")
-	require.True(t, ok)
-	assert.Equal(t, template, seed)
-}
-
-// A seed above the wire bound must fail the launch with an error naming the
-// limit, not reach NATS and fail as an oversized publish.
-func TestCreateEFIVolume_Provider_OversizedTemplateFails(t *testing.T) {
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true})
-	svc := providerEFIService(t, provider)
-
-	err := svc.createEFIVolumeViaProvider(context.Background(), "vol-efitoobig1-efi", make([]byte, ebsprovider.MaxSeedBytes+1))
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-// TestPrepareEFIVolume_Provider_RegistersEBSRequest covers the whole function
-// against this host's real firmware, including the name the VM launches with.
-func TestPrepareEFIVolume_Provider_RegistersEBSRequest(t *testing.T) {
-	if _, _, _, err := vm.FirmwarePaths("x86_64"); err != nil {
-		t.Skipf("no UEFI firmware on this host: %v", err)
-	}
-
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true})
-	svc := providerEFIService(t, provider)
-	instance := &vm.VM{}
-
-	require.NoError(t, svc.prepareEFIVolume(context.Background(), "vol-efiprepare001", instance, "x86_64"))
-
-	instance.EBSRequests.Mu.Lock()
-	defer instance.EBSRequests.Mu.Unlock()
-	require.Len(t, instance.EBSRequests.Requests, 1)
-	assert.Equal(t, "vol-efiprepare001-efi", instance.EBSRequests.Requests[0].Name)
-	assert.True(t, instance.EBSRequests.Requests[0].EFI)
-	assert.False(t, instance.EBSRequests.Requests[0].Boot)
-
-	seed, ok := provider.SeedData("vol-efiprepare001-efi")
-	require.True(t, ok)
-	assert.NotEmpty(t, seed, "the store must be seeded from this host's VARS template")
-}
-
-// roundingProvider reports a GiB-rounded capacity the way a provider with no
-// sub-GiB volume granularity would.
-type roundingProvider struct {
-	ebsprovider.EBSProvider
-}
-
-func (p *roundingProvider) CreateVolume(ctx context.Context, req ebsprovider.CreateVolumeRequest) (*ebsprovider.Volume, error) {
-	volume, err := p.EBSProvider.CreateVolume(ctx, req)
-	if err != nil {
-		return nil, err
-	}
-	volume.CapacityBytes = 1 << 30
-	return volume, nil
-}

spinifex/domains/ec2/instance/service_impl.go

@@ -10,7 +10,6 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/tagmirror"
 	"log/slog"
 	"maps"
-	"os"
 	"slices"
 	"sort"
 	"strconv"
@@ -1265,6 +1264,10 @@ func (s *InstanceServiceImpl) StartInstance(ctx context.Context, instance *vm.VM
 				slog.WarnContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
 				return refusal
 			}
+			if errors.Is(err, vm.ErrUEFIClaimHeld) {
+				slog.WarnContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
+				return errInsufficientCapacity
+			}
 			slog.ErrorContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
 			return errors.New(awserrors.ErrorServerInternal)
 		}
@@ -1612,18 +1615,14 @@ func (s *InstanceServiceImpl) GenerateVolumes(ctx context.Context, input *ec2.Ru
 		return nil, err
 	}
 
-	// Step 2: Create EFI variable store (only when the AMI is UEFI; BIOS
-	// guests must not allocate an orphan VARS volume).
-	if instance.BootMode == "uefi" || instance.BootMode == "uefi-preferred" {
-		arch := instanceArchitecture(s.instanceTypes[*input.InstanceType])
-		err = s.prepareEFIVolume(ctx, imageId, instance, arch)
-		if err != nil {
-			return nil, err
-		}
+	// Step 2: Seed the UEFI variable store, which is not a volume. A failure
+	// terminates the instance, and terminate deletes whatever store it left.
+	if err := s.vmMgr.SeedUEFIVarStore(ctx, instance); err != nil {
+		slog.ErrorContext(ctx, "Failed to seed the UEFI variable store", "instanceId", instance.ID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
-	// The EFI store is internal, so it is not reported; every volume the
-	// request named is.
+	// Every volume the request named is reported.
 	volumeInfos := []VolumeInfo{
 		{
 			VolumeId:            imageId,
@@ -1885,81 +1884,6 @@ func rootVolumeIOPS(requested int) int {
 	return requested
 }
 
-// prepareEFIVolume creates the per-VM EFI variable store, sized exactly to the
-// firmware VARS template (pflash requires byte-exact size) and seeded from it.
-// arch is "x86_64" | "arm64".
-func (s *InstanceServiceImpl) prepareEFIVolume(ctx context.Context, volumeID string, instance *vm.VM, arch string) error {
-	codePath, varsTemplate, varsSize, err := vm.FirmwarePaths(arch)
-	if err != nil {
-		slog.ErrorContext(ctx, "UEFI firmware not installed on this host", "arch", arch, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	template, err := os.ReadFile(varsTemplate)
-	if err != nil {
-		slog.ErrorContext(ctx, "Failed to read VARS template", "path", varsTemplate, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	if int64(len(template)) != varsSize {
-		slog.ErrorContext(ctx, "VARS template size mismatch between stat and read", "path", varsTemplate, "statSize", varsSize, "readSize", len(template))
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	slog.InfoContext(ctx, "Preparing EFI variable store", "arch", arch, "firmwarePath", codePath, "varsTemplate", varsTemplate, "size", varsSize)
-
-	efiVolumeName := fmt.Sprintf("%s-efi", volumeID)
-	if s.ebsProvider == nil {
-		slog.ErrorContext(ctx, "no EBS provider configured", "op", "prepareEFIVolume")
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	if err := s.createEFIVolumeViaProvider(ctx, efiVolumeName, template); err != nil {
-		return err
-	}
-
-	instance.EBSRequests.Mu.Lock()
-	instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{
-		Name: efiVolumeName,
-		Boot: false,
-		EFI:  true,
-	})
-	instance.EBSRequests.Mu.Unlock()
-
-	return nil
-}
-
-// createEFIVolumeViaProvider allocates the EFI variable store through the
-// provider, seeding it with this node's firmware VARS template. The bytes
-// travel with the request so the region matches the firmware that will run.
-func (s *InstanceServiceImpl) createEFIVolumeViaProvider(ctx context.Context, efiVolumeName string, template []byte) error {
-	slog.InfoContext(ctx, "Creating EFI variable store via EBS provider", "name", efiVolumeName, "seedBytes", len(template))
-
-	// The store is internal to the launch, never attachable and filtered out of
-	// DescribeVolumes, so it gets no ebsmetadata document the way a root volume
-	// does. Its lifecycle is the root volume's DeleteVolume.
-	created, err := s.ebsProvider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{
-		Versioned:        ebsprovider.NewVersioned(),
-		VolumeID:         efiVolumeName,
-		CapacityRange:    ebsprovider.CapacityRange{RequiredBytes: int64(len(template))},
-		AvailabilityZone: s.config.AZ,
-		SeedData:         template,
-	})
-	if err != nil {
-		slog.ErrorContext(ctx, "Provider EFI volume creation failed", "name", efiVolumeName, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	if created == nil {
-		slog.ErrorContext(ctx, "Provider returned no volume for the EFI variable store", "name", efiVolumeName)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-
-	// pflash rejects a VARS region that is not byte-exact, so a provider that
-	// rounded the capacity up to a GiB boundary yields a guest that cannot boot.
-	if created.CapacityBytes != int64(len(template)) {
-		slog.ErrorContext(ctx, "Provider EFI volume capacity does not match the VARS template",
-			"name", efiVolumeName, "wantBytes", len(template), "gotBytes", created.CapacityBytes)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	return nil
-}
-
 // checkImageArchitecture refuses an AMI whose architecture the instance type
 // does not support, as AWS does. An unrecorded architecture on either side is
 // not a mismatch.
@@ -2760,7 +2684,7 @@ func (s *InstanceServiceImpl) capacityRefusal(ctx context.Context, instance *vm.
 func (s *InstanceServiceImpl) launchClaimedInstance(ctx context.Context, instance *vm.VM, undo func()) error {
 	// Before anything is stamped or claimed, so the record handed back is the
 	// one that was claimed.
-	if err := vm.CheckUEFIVarStore(instance); err != nil {
+	if err := s.vmMgr.CheckUEFIVarStore(ctx, instance); err != nil {
 		slog.ErrorContext(ctx, "launchClaimedInstance: refusing to launch a UEFI instance without exactly one variable store",
 			"instanceId", instance.ID, "err", err)
 		undo()
@@ -2893,6 +2817,10 @@ func (s *InstanceServiceImpl) launchClaimedInstance(ctx context.Context, instanc
 		if refusal := volumeHeldElsewhereError(err); refusal != nil {
 			return refusal
 		}
+		// A live node holds the variable store, so another node may take it.
+		if errors.Is(err, vm.ErrUEFIClaimHeld) {
+			return s.capacityRefusal(ctx, instance)
+		}
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 

spinifex/domains/ec2/instance/uefi_seed_test.go

@@ -0,0 +1,122 @@
+package instance
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const testSeedInstance = "i-0a1b2c3d4e5f60718"
+
+type seedInstanceTypes map[string]vm.InstanceTypeSpec
+
+func (r seedInstanceTypes) Resolve(name string) (vm.InstanceTypeSpec, bool) {
+	spec, ok := r[name]
+	return spec, ok
+}
+
+// uefiSeedingService is an instance service whose VM manager seeds stores into
+// a real JetStream's store records and claims and an in-memory object store,
+// under a stand-in x86_64 firmware.
+func uefiSeedingService(t *testing.T) (*InstanceServiceImpl, *ebsprovider.MemoryProvider, *uefistore.Store, *objectstore.MemoryObjectStore) {
+	t.Helper()
+	fwDir := t.TempDir()
+	code := filepath.Join(fwDir, "OVMF_CODE.fd")
+	vars := filepath.Join(fwDir, "OVMF_VARS.fd")
+	require.NoError(t, os.WriteFile(code, []byte("code"), 0o600))
+	require.NoError(t, os.WriteFile(vars, make([]byte, 4096), 0o600))
+	orig, had := vm.FirmwarePathCandidates["x86_64"]
+	vm.FirmwarePathCandidates["x86_64"] = []vm.FirmwareCandidate{{Code: code, VarsTemplate: vars}}
+	t.Cleanup(func() {
+		if had {
+			vm.FirmwarePathCandidates["x86_64"] = orig
+		} else {
+			delete(vm.FirmwarePathCandidates, "x86_64")
+		}
+	})
+
+	_, _, js := testutil.StartTestJetStream(t)
+	records := uefistore.NewStore(js)
+	require.NoError(t, records.Open(t.Context()))
+
+	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{CrashConsistentSnapshot: true, VolumeEnumeration: true})
+	seedProviderSnapshot(t, provider, "vol-origin", "snap-source")
+	objects := objectstore.NewMemoryObjectStore()
+	svc := providerRootVolumeService(t, provider, rootVolumeAMILoader(), objects)
+	svc.vmMgr = vm.NewManagerWithDeps(vm.Deps{
+		NodeID:        "node-a",
+		InstanceTypes: seedInstanceTypes{"t3.nano": {Architecture: "x86_64", VCPUs: 1, MemoryMiB: 512}},
+		UEFIVarStore: vm.UEFIVarStoreDeps{
+			Objects: objects, Bucket: testRootBucket, DataDir: t.TempDir(), Records: records,
+			Claims: uefistore.NewClaims(kvstore.OpenSyncBucket(js), "node-a"),
+		},
+	})
+	return svc, provider, records, objects
+}
+
+func uefiLaunch() (*ec2.RunInstancesInput, *vm.VM) {
+	input := &ec2.RunInstancesInput{ImageId: aws.String("ami-1"), InstanceType: aws.String("t3.nano")}
+	return input, &vm.VM{ID: testSeedInstance, AccountID: testRootAccount, InstanceType: "t3.nano", BootMode: "uefi"}
+}
+
+// A UEFI launch seeds its store as an object and a store record, and creates
+// no <root>-efi volume and no EFI request.
+func TestGenerateVolumes_UEFILaunchSeedsAStoreAndCreatesNoEFIVolume(t *testing.T) {
+	svc, provider, records, objects := uefiSeedingService(t)
+	input, instance := uefiLaunch()
+
+	infos, err := svc.GenerateVolumes(t.Context(), input, instance)
+	require.NoError(t, err)
+
+	require.Len(t, infos, 1, "only the root volume is reported")
+	instance.EBSRequests.Mu.Lock()
+	for _, req := range instance.EBSRequests.Requests {
+		assert.False(t, req.EFI, "no EFI request is appended")
+		assert.False(t, strings.HasSuffix(req.Name, "-efi"), req.Name)
+	}
+	instance.EBSRequests.Mu.Unlock()
+	listed, err := provider.ListVolumes(t.Context(), ebsprovider.ListVolumesRequest{Versioned: ebsprovider.NewVersioned()})
+	require.NoError(t, err)
+	for _, v := range listed.Volumes {
+		assert.False(t, strings.HasSuffix(v.ID, "-efi"), "no EFI volume is created: %s", v.ID)
+	}
+
+	rec, _, err := records.Get(t.Context(), testSeedInstance)
+	require.NoError(t, err)
+	assert.Equal(t, uefistore.StateLive, rec.State)
+	require.NotNil(t, rec.Pointer)
+	keys, err := uefistore.ListObjects(t.Context(), objects, testRootBucket, testSeedInstance)
+	require.NoError(t, err)
+	assert.Equal(t, []string{rec.Pointer.Key}, keys, "the pointer names the one template object")
+}
+
+// A seed that cannot be written fails the launch as InternalError, and no EFI
+// volume is created in its place.
+func TestGenerateVolumes_FailedSeedIsInternalError(t *testing.T) {
+	svc, provider, _, _ := uefiSeedingService(t)
+	svc.vmMgr = vm.NewManager()
+	input, instance := uefiLaunch()
+
+	_, err := svc.GenerateVolumes(t.Context(), input, instance)
+
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+	listed, err := provider.ListVolumes(t.Context(), ebsprovider.ListVolumesRequest{Versioned: ebsprovider.NewVersioned()})
+	require.NoError(t, err)
+	for _, v := range listed.Volumes {
+		assert.False(t, strings.HasSuffix(v.ID, "-efi"), v.ID)
+	}
+}

spinifex/domains/ec2/instance/uefistore/cleanup.go

@@ -25,9 +25,9 @@ func (s *Store) Exists(ctx context.Context, instanceID string) (bool, error) {
 	return s.kv.Exists(ctx, instanceID)
 }
 
-// MarkDeleting records that instanceID's store is to be deleted, and drops any
-// write-back still owed. It reports false with no error when the instance has
-// no store record. Only a terminate or an operator's forget may call it.
+// MarkDeleting records that instanceID's store is to be deleted. It reports
+// false with no error when the instance has no store record. Only a decided
+// terminate may call it.
 func (s *Store) MarkDeleting(ctx context.Context, instanceID string) (bool, error) {
 	for range markAttempts {
 		rec, rev, err := s.Get(ctx, instanceID)
@@ -41,7 +41,6 @@ func (s *Store) MarkDeleting(ctx context.Context, instanceID string) (bool, erro
 			return true, nil
 		}
 		rec.State = StateDeleting
-		rec.WriteBack = nil
 		_, err = s.CompareAndSet(ctx, rec, rev)
 		if err == nil {
 			return true, nil

spinifex/domains/ec2/instance/uefistore/cleanup_test.go

@@ -37,7 +37,7 @@ func keysUnder(t *testing.T, objects objectstore.ObjectStore, prefix string) []s
 }
 
 // Deletion is final: a writer holding the revision of a deleting record, such
-// as a write-back retry, must not bring the store back to live.
+// as a late pointer move, must not bring the store back to live.
 func TestCompareAndSet_RefusesDeletingToLive(t *testing.T) {
 	s, _ := openStore(t)
 	rev, err := s.Create(t.Context(), liveRecord())
@@ -47,8 +47,7 @@ func TestCompareAndSet_RefusesDeletingToLive(t *testing.T) {
 	rev, err = s.CompareAndSet(t.Context(), deleting, rev)
 	require.NoError(t, err)
 
-	revived := liveRecord()
-	revived.WriteBack = writeBack()
+	revived := pointedRecord()
 	_, err = s.CompareAndSet(t.Context(), revived, rev)
 	require.ErrorIs(t, err, uefistore.ErrDeleting)
 
@@ -58,24 +57,9 @@ func TestCompareAndSet_RefusesDeletingToLive(t *testing.T) {
 	assert.Equal(t, uefistore.StateDeleting, got.State)
 }
 
-// A write-back owed on a deleting store would put an object behind the delete.
-func TestCompareAndSet_RefusesDeletingWithAWriteBack(t *testing.T) {
+func TestMarkDeleting_MovesLiveToDeletingAndKeepsWhatItNames(t *testing.T) {
 	s, _ := openStore(t)
-	rev, err := s.Create(t.Context(), liveRecord())
-	require.NoError(t, err)
-	rec := liveRecord()
-	rec.State = uefistore.StateDeleting
-	rec.WriteBack = writeBack()
-
-	_, err = s.CompareAndSet(t.Context(), rec, rev)
-	require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
-}
-
-func TestMarkDeleting_MovesLiveToDeletingAndDropsTheWriteBack(t *testing.T) {
-	s, _ := openStore(t)
-	rec := liveRecord()
-	rec.WriteBack = writeBack()
-	_, err := s.Create(t.Context(), rec)
+	_, err := s.Create(t.Context(), pointedRecord())
 	require.NoError(t, err)
 
 	has, err := s.MarkDeleting(t.Context(), instanceID)
@@ -85,8 +69,8 @@ func TestMarkDeleting_MovesLiveToDeletingAndDropsTheWriteBack(t *testing.T) {
 	got, rev, err := s.Get(t.Context(), instanceID)
 	require.NoError(t, err)
 	assert.Equal(t, uefistore.StateDeleting, got.State)
-	assert.Nil(t, got.WriteBack)
 	assert.Equal(t, "111122223333", got.AccountID)
+	assert.Equal(t, pointer("b"), got.Pointer, "the pointer stays readable until the prefix is gone")
 
 	has, err = s.MarkDeleting(t.Context(), instanceID)
 	require.NoError(t, err)

spinifex/domains/ec2/instance/uefistore/fence.go

@@ -0,0 +1,93 @@
+package uefistore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+)
+
+// ClaimPrefix is the kvsync prefix of every store's claim. An instance's claim
+// is ClaimPrefix+instanceID, and names the node that may hold variable changes
+// no object has yet.
+const ClaimPrefix = "uefi."
+
+// NewClaims returns node's group of store claims over store. The daemon builds
+// it once per process, as kvsync requires of every prefix.
+func NewClaims(store kvsync.KVStore, node string) *kvsync.ClaimGroup {
+	return kvsync.NewClaimGroup(store, ClaimPrefix, node)
+}
+
+// ErrFenced is a stamp or pointer move refused because the record carries a
+// claim generation other than the caller's: a later acquisition has stamped
+// it, so the caller no longer owns the store and must change nothing.
+var ErrFenced = errors.New("uefistore: store record is fenced to another claim generation")
+
+// fenceAttempts bounds the read-then-write loops of Stamp and MovePointer. A
+// lost race is re-read, so a concurrent stamp is seen and refused, not retried.
+const fenceAttempts = 5
+
+// Stamp raises instanceID's record to generation, the claim token the caller
+// has just acquired or adopted, and returns the record it stamped, whose
+// pointer the caller boots from. One above generation is ErrFenced, and one
+// deleting is ErrDeleting. A record already at generation is still written by
+// CAS, because a follower can answer the read with an older pointer.
+func (s *Store) Stamp(ctx context.Context, instanceID string, generation uint64) (*UEFIStoreRecord, error) {
+	if generation == 0 {
+		return nil, fmt.Errorf("uefistore: stamp %s: generation 0 is no claim", instanceID)
+	}
+	for range fenceAttempts {
+		rec, rev, err := s.Get(ctx, instanceID)
+		if err != nil {
+			return nil, err
+		}
+		switch {
+		case rec.State == StateDeleting:
+			return nil, fmt.Errorf("%w: %s", ErrDeleting, instanceID)
+		case rec.Generation > generation:
+			return nil, fmt.Errorf("%w: %s is at generation %d, above the claim's %d", ErrFenced, instanceID, rec.Generation, generation)
+		}
+		rec.Generation = generation
+		_, err = s.CompareAndSet(ctx, rec, rev)
+		if err == nil {
+			return rec, nil
+		}
+		if !errors.Is(err, kvstore.ErrConflict) {
+			return nil, err
+		}
+	}
+	return nil, fmt.Errorf("uefistore: stamp %s: %w after %d attempts", instanceID, kvstore.ErrConflict, fenceAttempts)
+}
+
+// MovePointer points instanceID's record at ref only while the record is at
+// generation. A record at any other generation is ErrFenced and is left
+// unchanged. The leader answers the read, because a caller drops its copy on
+// ErrFenced or ErrDeleting, and the CAS keeps a lost race from succeeding.
+func (s *Store) MovePointer(ctx context.Context, instanceID string, generation uint64, ref UEFIVarStoreRef) error {
+	if err := ref.validate(instanceID); err != nil {
+		return fmt.Errorf("%w: move %s: %w", ErrInvalidRecord, instanceID, err)
+	}
+	for range fenceAttempts {
+		rec, rev, err := s.LeaderGet(ctx, instanceID)
+		if err != nil {
+			return err
+		}
+		switch {
+		case rec.State == StateDeleting:
+			return fmt.Errorf("%w: %s", ErrDeleting, instanceID)
+		case rec.Generation != generation:
+			return fmt.Errorf("%w: %s is at generation %d, and the writer holds %d", ErrFenced, instanceID, rec.Generation, generation)
+		}
+		rec.Pointer = &ref
+		_, err = s.CompareAndSet(ctx, rec, rev)
+		if err == nil {
+			return nil
+		}
+		if !errors.Is(err, kvstore.ErrConflict) {
+			return err
+		}
+	}
+	return fmt.Errorf("uefistore: move %s: %w after %d attempts", instanceID, kvstore.ErrConflict, fenceAttempts)
+}

spinifex/domains/ec2/instance/uefistore/fence_test.go

@@ -0,0 +1,213 @@
+package uefistore_test
+
+import (
+	"testing"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func revisionOf(t *testing.T, s *uefistore.Store) uint64 {
+	t.Helper()
+	_, rev, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	return rev
+}
+
+func TestStamp_RaisesTheGenerationAndReturnsThePointerToBootFrom(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), pointedRecord())
+	require.NoError(t, err)
+
+	rec, err := s.Stamp(t.Context(), instanceID, 9)
+	require.NoError(t, err)
+	assert.Equal(t, uint64(9), rec.Generation)
+	assert.Equal(t, pointer("b"), rec.Pointer)
+
+	got, _, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, rec, got, "the returned record is the one stored")
+}
+
+// A daemon restart adopts its claim at the same generation. The stamp still
+// writes by CAS, so a pointer read from a lagging follower fails the write and
+// is read again rather than booted from.
+func TestStamp_AtTheRecordsGenerationConfirmsTheReadByCAS(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), pointedRecord())
+	require.NoError(t, err)
+	before := revisionOf(t, s)
+
+	rec, err := s.Stamp(t.Context(), instanceID, 7)
+	require.NoError(t, err)
+	assert.Equal(t, pointer("b"), rec.Pointer)
+	assert.Equal(t, uint64(7), rec.Generation)
+	assert.Greater(t, revisionOf(t, s), before, "the read must be confirmed by a revision-checked write")
+}
+
+func TestStamp_RefusesAndChangesNothing(t *testing.T) {
+	tests := []struct {
+		name       string
+		deleting   bool
+		generation uint64
+		want       error
+	}{
+		{name: "a later acquisition stamped it", generation: 6, want: uefistore.ErrFenced},
+		{name: "the store is being deleted", deleting: true, generation: 8, want: uefistore.ErrDeleting},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s, _ := openStore(t)
+			rev, err := s.Create(t.Context(), pointedRecord())
+			require.NoError(t, err)
+			if tt.deleting {
+				_, err = s.MarkDeleting(t.Context(), instanceID)
+				require.NoError(t, err)
+				rev = revisionOf(t, s)
+			}
+
+			_, err = s.Stamp(t.Context(), instanceID, tt.generation)
+			require.ErrorIs(t, err, tt.want)
+			assert.Equal(t, rev, revisionOf(t, s))
+		})
+	}
+}
+
+func TestStamp_MissingRecordIsNotFound(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Stamp(t.Context(), instanceID, 1)
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+}
+
+// The record keeps only the current version, so each move replaces the
+// pointer and names nothing it replaced.
+func TestMovePointer_ReplacesThePointer(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), pointedRecord())
+	require.NoError(t, err)
+
+	for _, name := range []string{"c", "d"} {
+		require.NoError(t, s.MovePointer(t.Context(), instanceID, 7, *pointer(name)))
+	}
+
+	got, _, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, pointer("d"), got.Pointer)
+	assert.Equal(t, uint64(7), got.Generation)
+}
+
+// A move to the key the record already names is still a CAS at the revision
+// read, so a fenced writer that read a lagging follower cannot see success.
+func TestMovePointer_ToTheCurrentKeyConfirmsTheReadByCAS(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), pointedRecord())
+	require.NoError(t, err)
+	before := revisionOf(t, s)
+
+	require.NoError(t, s.MovePointer(t.Context(), instanceID, 7, *pointer("b")))
+	assert.Greater(t, revisionOf(t, s), before)
+}
+
+func TestMovePointer_RefusesAndChangesNothing(t *testing.T) {
+	tests := []struct {
+		name       string
+		deleting   bool
+		generation uint64
+		ref        *uefistore.UEFIVarStoreRef
+		want       error
+	}{
+		{name: "an earlier generation", generation: 6, ref: pointer("c"), want: uefistore.ErrFenced},
+		{name: "a later generation than the stamp", generation: 8, ref: pointer("c"), want: uefistore.ErrFenced},
+		{name: "a deleting store", deleting: true, generation: 7, ref: pointer("c"), want: uefistore.ErrDeleting},
+		{name: "a key under another instance", generation: 7,
+			ref:  &uefistore.UEFIVarStoreRef{Key: uefistore.KeyPrefix("i-0fedcba9876543210") + "c", Firmware: pointer("c").Firmware, WriterNode: "n"},
+			want: uefistore.ErrInvalidRecord},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s, _ := openStore(t)
+			_, err := s.Create(t.Context(), pointedRecord())
+			require.NoError(t, err)
+			if tt.deleting {
+				_, err = s.MarkDeleting(t.Context(), instanceID)
+				require.NoError(t, err)
+			}
+			rev := revisionOf(t, s)
+
+			err = s.MovePointer(t.Context(), instanceID, tt.generation, *tt.ref)
+			require.ErrorIs(t, err, tt.want)
+			got, gotRev, err := s.Get(t.Context(), instanceID)
+			require.NoError(t, err)
+			assert.Equal(t, rev, gotRev)
+			assert.Equal(t, pointer("b"), got.Pointer)
+		})
+	}
+}
+
+// claimedStore is a store record and a claim group per node over one
+// JetStream, as two daemons see them.
+type claimedStore struct {
+	records *uefistore.Store
+	a, b    *kvsync.Claim
+}
+
+func newClaimedStore(t *testing.T) *claimedStore {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	records := uefistore.NewStore(js)
+	require.NoError(t, records.Open(t.Context()))
+	_, err := records.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+	return &claimedStore{
+		records: records,
+		a:       uefistore.NewClaims(kvstore.OpenSyncBucket(js), "node-a").Get(instanceID),
+		b:       uefistore.NewClaims(kvstore.OpenSyncBucket(js), "node-b").Get(instanceID),
+	}
+}
+
+// A node partitioned long enough to be taken over is fenced by the new
+// owner's stamp: its pointer move fails and the record keeps what it had.
+func TestFence_TheOldHoldersMoveFailsAfterTheNewOwnersStamp(t *testing.T) {
+	c := newClaimedStore(t)
+	first, err := c.a.TryLock(t.Context())
+	require.NoError(t, err)
+	_, err = c.records.Stamp(t.Context(), instanceID, first.Generation)
+	require.NoError(t, err)
+	require.NoError(t, c.records.MovePointer(t.Context(), instanceID, first.Generation, *pointer("b")))
+
+	second, err := c.b.TakeOver(t.Context(), first)
+	require.NoError(t, err)
+	_, err = c.records.Stamp(t.Context(), instanceID, second.Generation)
+	require.NoError(t, err)
+	rev := revisionOf(t, c.records)
+
+	err = c.records.MovePointer(t.Context(), instanceID, first.Generation, *pointer("late"))
+	require.ErrorIs(t, err, uefistore.ErrFenced)
+	got, gotRev, err := c.records.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, rev, gotRev)
+	assert.Equal(t, pointer("b"), got.Pointer)
+	assert.Equal(t, second.Generation, got.Generation)
+}
+
+// The new owner reads the pointer after it stamps, so a move the old holder
+// landed between the takeover and the stamp is the one it boots from.
+func TestFence_AMoveBetweenTakeoverAndStampIsThePointerTheNewOwnerBoots(t *testing.T) {
+	c := newClaimedStore(t)
+	first, err := c.a.TryLock(t.Context())
+	require.NoError(t, err)
+	_, err = c.records.Stamp(t.Context(), instanceID, first.Generation)
+	require.NoError(t, err)
+
+	second, err := c.b.TakeOver(t.Context(), first)
+	require.NoError(t, err)
+	require.NoError(t, c.records.MovePointer(t.Context(), instanceID, first.Generation, *pointer("landed")))
+
+	rec, err := c.records.Stamp(t.Context(), instanceID, second.Generation)
+	require.NoError(t, err)
+	assert.Equal(t, keyOf("landed"), rec.Pointer.Key)
+}

spinifex/domains/ec2/instance/uefistore/object.go

@@ -0,0 +1,121 @@
+package uefistore
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/rand"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+)
+
+const (
+	// SourceTemplate is a store seeded from the launching node's VARS template.
+	SourceTemplate = "template"
+	// SourceWriteBack is a store written back from a node's live copy.
+	SourceWriteBack = "write-back"
+
+	// PutTimeout bounds one put of a store object. The object is a fixed-size
+	// region, so a put that takes longer is a wedged object store.
+	PutTimeout = 10 * time.Second
+)
+
+// ErrInvalidObject is a store object whose key, region or metadata cannot be
+// written as the owner's store.
+var ErrInvalidObject = errors.New("uefistore: invalid store object")
+
+// ObjectMeta is what a store object records about itself, as x-amz-meta-
+// entries, so its owner, writer and firmware are known from the object alone.
+type ObjectMeta struct {
+	OwnerID    string
+	WriterNode string
+	Firmware   FirmwareIdentity
+	CodePath   string
+	Source     string
+}
+
+// Metadata returns m as object metadata, keyed by the owner's kind.
+func (m ObjectMeta) Metadata() (map[string]*string, error) {
+	var ownerKey string
+	switch {
+	case strings.HasPrefix(m.OwnerID, "i-"):
+		ownerKey = "instance-id"
+	case strings.HasPrefix(m.OwnerID, "ami-"):
+		ownerKey = "image-id"
+	default:
+		return nil, fmt.Errorf("%w: owner %q is neither an instance nor an image", ErrInvalidObject, m.OwnerID)
+	}
+	switch {
+	case m.WriterNode == "":
+		return nil, fmt.Errorf("%w: %s names no writer node", ErrInvalidObject, m.OwnerID)
+	case m.Source == "":
+		return nil, fmt.Errorf("%w: %s names no source", ErrInvalidObject, m.OwnerID)
+	case m.Firmware.Arch == "" || m.Firmware.VarsSize <= 0 || m.Firmware.CodeSHA256 == "":
+		return nil, fmt.Errorf("%w: %s has an incomplete firmware identity", ErrInvalidObject, m.OwnerID)
+	}
+	return map[string]*string{
+		ownerKey:               aws.String(m.OwnerID),
+		"writer-node":          aws.String(m.WriterNode),
+		"firmware-arch":        aws.String(m.Firmware.Arch),
+		"firmware-vars-size":   aws.String(strconv.FormatInt(m.Firmware.VarsSize, 10)),
+		"firmware-code-sha256": aws.String(m.Firmware.CodeSHA256),
+		"firmware-code-path":   aws.String(m.CodePath),
+		"source":               aws.String(m.Source),
+	}, nil
+}
+
+// NewObjectKey returns a fresh key under ownerID's prefix, named by a random
+// version 4 UUID, so no write ever replaces an object a record points at.
+func NewObjectKey(ownerID string) (string, error) {
+	var b [16]byte
+	if _, err := rand.Read(b[:]); err != nil {
+		return "", fmt.Errorf("generate UEFI store key for %s: %w", ownerID, err)
+	}
+	b[6] = b[6]&0x0f | 0x40
+	b[8] = b[8]&0x3f | 0x80
+	return fmt.Sprintf("%s%x-%x-%x-%x-%x", KeyPrefix(ownerID), b[0:4], b[4:6], b[6:8], b[8:10], b[10:16]), nil
+}
+
+// PutObject writes region gzip-compressed to key with meta, within
+// PutTimeout. The region must be exactly the firmware's VARS size, because
+// pflash refuses any other.
+func PutObject(ctx context.Context, objects objectstore.ObjectStore, bucket, key string, region []byte, meta ObjectMeta) error {
+	if !ownsKey(meta.OwnerID, key) {
+		return fmt.Errorf("%w: key %q is not under %s", ErrInvalidObject, key, KeyPrefix(meta.OwnerID))
+	}
+	if int64(len(region)) != meta.Firmware.VarsSize {
+		return fmt.Errorf("%w: %s is a %d-byte region under firmware with a %d-byte VARS region",
+			ErrInvalidObject, key, len(region), meta.Firmware.VarsSize)
+	}
+	md, err := meta.Metadata()
+	if err != nil {
+		return err
+	}
+	var body bytes.Buffer
+	zw := gzip.NewWriter(&body)
+	if _, err := zw.Write(region); err != nil {
+		return fmt.Errorf("compress UEFI store %s: %w", key, err)
+	}
+	if err := zw.Close(); err != nil {
+		return fmt.Errorf("compress UEFI store %s: %w", key, err)
+	}
+
+	putCtx, cancel := context.WithTimeout(ctx, PutTimeout)
+	defer cancel()
+	if _, err := objects.PutObject(putCtx, &s3.PutObjectInput{
+		Bucket:   aws.String(bucket),
+		Key:      aws.String(key),
+		Body:     bytes.NewReader(body.Bytes()),
+		Metadata: md,
+	}); err != nil {
+		return fmt.Errorf("put UEFI store %s: %w", key, err)
+	}
+	return nil
+}

spinifex/domains/ec2/instance/uefistore/object_test.go

@@ -0,0 +1,164 @@
+package uefistore_test
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"errors"
+	"io"
+	"regexp"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const testBucket = "instances"
+
+func testMeta(owner string) uefistore.ObjectMeta {
+	return uefistore.ObjectMeta{
+		OwnerID:    owner,
+		WriterNode: "node-1",
+		Firmware:   uefistore.FirmwareIdentity{Arch: "arm64", VarsSize: 1 << 16, CodeSHA256: "abc123"},
+		CodePath:   "/usr/share/AAVMF/AAVMF_CODE.fd",
+		Source:     uefistore.SourceTemplate,
+	}
+}
+
+var uuidV4 = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)
+
+// Every key is a fresh version 4 UUID directly under the owner's prefix, so
+// a write never lands on an object a record already points at.
+func TestNewObjectKey_IsAFreshUUIDUnderTheOwnersPrefix(t *testing.T) {
+	seen := map[string]bool{}
+	for range 64 {
+		key, err := uefistore.NewObjectKey(instanceID)
+		require.NoError(t, err)
+		name, ok := bytes.CutPrefix([]byte(key), []byte(uefistore.KeyPrefix(instanceID)))
+		require.True(t, ok, key)
+		assert.Regexp(t, uuidV4, string(name))
+		assert.False(t, seen[key], "key %s repeated", key)
+		seen[key] = true
+	}
+}
+
+func TestObjectMeta_Metadata_NamesTheOwnerByKind(t *testing.T) {
+	md, err := testMeta(instanceID).Metadata()
+	require.NoError(t, err)
+	assert.Equal(t, map[string]string{
+		"instance-id":          instanceID,
+		"writer-node":          "node-1",
+		"firmware-arch":        "arm64",
+		"firmware-vars-size":   "65536",
+		"firmware-code-sha256": "abc123",
+		"firmware-code-path":   "/usr/share/AAVMF/AAVMF_CODE.fd",
+		"source":               "template",
+	}, aws.StringValueMap(md))
+
+	img := testMeta("ami-0123456789abcdef0")
+	img.Source = "image:ami-0fedcba9876543210"
+	md, err = img.Metadata()
+	require.NoError(t, err)
+	assert.Equal(t, "ami-0123456789abcdef0", aws.StringValue(md["image-id"]))
+	assert.NotContains(t, md, "instance-id")
+	assert.Equal(t, "image:ami-0fedcba9876543210", aws.StringValue(md["source"]))
+}
+
+func TestObjectMeta_Metadata_RefusesAnIncompleteObject(t *testing.T) {
+	tests := map[string]func(*uefistore.ObjectMeta){
+		"unknown owner":  func(m *uefistore.ObjectMeta) { m.OwnerID = "vol-1" },
+		"no writer":      func(m *uefistore.ObjectMeta) { m.WriterNode = "" },
+		"no source":      func(m *uefistore.ObjectMeta) { m.Source = "" },
+		"no arch":        func(m *uefistore.ObjectMeta) { m.Firmware.Arch = "" },
+		"no vars size":   func(m *uefistore.ObjectMeta) { m.Firmware.VarsSize = 0 },
+		"no code sha256": func(m *uefistore.ObjectMeta) { m.Firmware.CodeSHA256 = "" },
+	}
+	for name, mutate := range tests {
+		t.Run(name, func(t *testing.T) {
+			m := testMeta(instanceID)
+			mutate(&m)
+			_, err := m.Metadata()
+			require.ErrorIs(t, err, uefistore.ErrInvalidObject)
+		})
+	}
+}
+
+// The 64 MiB arm64 region is mostly erased flash, so it is stored compressed
+// and decompresses to the exact region.
+func TestPutObject_StoresTheRegionCompressed(t *testing.T) {
+	objects := objectstore.NewMemoryObjectStore()
+	meta := testMeta(instanceID)
+	region := bytes.Repeat([]byte{0xff}, int(meta.Firmware.VarsSize))
+	copy(region, "BootOrder")
+	key, err := uefistore.NewObjectKey(instanceID)
+	require.NoError(t, err)
+
+	require.NoError(t, uefistore.PutObject(t.Context(), objects, testBucket, key, region, meta))
+
+	out, err := objects.GetObject(t.Context(), &s3.GetObjectInput{Bucket: aws.String(testBucket), Key: aws.String(key)})
+	require.NoError(t, err)
+	body, err := io.ReadAll(out.Body)
+	require.NoError(t, err)
+	assert.Less(t, len(body), len(region)/10, "the erased region compresses")
+	gz, err := gzip.NewReader(bytes.NewReader(body))
+	require.NoError(t, err)
+	got, err := io.ReadAll(gz)
+	require.NoError(t, err)
+	assert.Equal(t, region, got)
+}
+
+// pflash needs a byte-exact region, and a key outside the owner's prefix is
+// one no store record owns; either is refused before anything is put.
+func TestPutObject_RefusesWhatTheOwnerCannotBoot(t *testing.T) {
+	meta := testMeta(instanceID)
+	good, err := uefistore.NewObjectKey(instanceID)
+	require.NoError(t, err)
+	other, err := uefistore.NewObjectKey("i-0fedcba9876543210")
+	require.NoError(t, err)
+	tests := []struct {
+		name   string
+		key    string
+		region []byte
+	}{
+		{name: "short region", key: good, region: make([]byte, meta.Firmware.VarsSize-1)},
+		{name: "long region", key: good, region: make([]byte, meta.Firmware.VarsSize+1)},
+		{name: "another owner's key", key: other, region: make([]byte, meta.Firmware.VarsSize)},
+		{name: "nested key", key: uefistore.KeyPrefix(instanceID) + "a/b", region: make([]byte, meta.Firmware.VarsSize)},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			objects := objectstore.NewMemoryObjectStore()
+			err := uefistore.PutObject(t.Context(), objects, testBucket, tt.key, tt.region, meta)
+			require.ErrorIs(t, err, uefistore.ErrInvalidObject)
+			out, listErr := objects.ListObjectsV2(t.Context(), &s3.ListObjectsV2Input{Bucket: aws.String(testBucket)})
+			require.NoError(t, listErr)
+			assert.Empty(t, out.Contents)
+		})
+	}
+}
+
+type failingPut struct {
+	*objectstore.MemoryObjectStore
+}
+
+func (failingPut) PutObject(ctx context.Context, _ *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
+	if _, ok := ctx.Deadline(); !ok {
+		return nil, errors.New("put had no deadline")
+	}
+	return nil, errors.New("predastore unreachable")
+}
+
+// A put runs under a deadline, so a wedged object store fails the write
+// rather than holding the caller, and its error reaches the caller.
+func TestPutObject_IsBoundedAndReportsAFailedPut(t *testing.T) {
+	key, err := uefistore.NewObjectKey(instanceID)
+	require.NoError(t, err)
+	meta := testMeta(instanceID)
+	err = uefistore.PutObject(t.Context(), failingPut{objectstore.NewMemoryObjectStore()}, testBucket, key, make([]byte, meta.Firmware.VarsSize), meta)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "predastore unreachable")
+}

spinifex/domains/ec2/instance/uefistore/prune.go

@@ -0,0 +1,73 @@
+package uefistore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+)
+
+// PruneMinAge is how old an object must be before a prune may delete it, so a
+// write-back whose pointer move has not landed yet never loses its object.
+const PruneMinAge = 10 * time.Minute
+
+// PruneResult is what one prune of a prefix did.
+type PruneResult struct {
+	Deleted int
+	// Young counts objects other than keep that were too new to delete, so the
+	// caller prunes the prefix again later.
+	Young int
+}
+
+// PruneObjects deletes every object under ownerID's prefix except keep that was
+// last modified before cutoff. keep must be the key the store record points at;
+// an empty or foreign keep deletes nothing. An object with no modification time
+// counts as young.
+func PruneObjects(ctx context.Context, objects objectstore.ObjectStore, bucket, ownerID, keep string, cutoff time.Time) (PruneResult, error) {
+	var res PruneResult
+	if err := checkOwnedKey(ownerID, keep); err != nil {
+		return res, fmt.Errorf("uefistore: refusing to prune %s: %w", ownerID, err)
+	}
+	contents, _, err := objectstore.ListAll(ctx, objects, &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucket),
+		Prefix: aws.String(KeyPrefix(ownerID)),
+	})
+	if err != nil {
+		return res, fmt.Errorf("uefistore: list %s: %w", KeyPrefix(ownerID), err)
+	}
+	for _, obj := range contents {
+		key := aws.StringValue(obj.Key)
+		if key == keep {
+			continue
+		}
+		if obj.LastModified == nil || !obj.LastModified.Before(cutoff) {
+			res.Young++
+			continue
+		}
+		_, err := objects.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
+		if err != nil && !objectstore.IsNoSuchKeyError(err) {
+			return res, fmt.Errorf("uefistore: delete %s: %w", key, err)
+		}
+		res.Deleted++
+	}
+	return res, nil
+}
+
+// checkOwnedKey fails a key that is empty or not one object under ownerID's
+// prefix.
+func checkOwnedKey(ownerID, key string) error {
+	if !validOwner(ownerID) {
+		return fmt.Errorf("%q is not an instance or image ID", ownerID)
+	}
+	if key == "" {
+		return errors.New("no key to keep")
+	}
+	if !ownsKey(ownerID, key) {
+		return fmt.Errorf("%s is not under %s", key, KeyPrefix(ownerID))
+	}
+	return nil
+}

spinifex/domains/ec2/instance/uefistore/prune_test.go

@@ -0,0 +1,84 @@
+package uefistore_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// datedStore lists each object with the modification time the test set; a
+// key it has none for is listed with none.
+type datedStore struct {
+	*objectstore.MemoryObjectStore
+
+	modified map[string]time.Time
+}
+
+func (s *datedStore) ListObjectsV2(ctx context.Context, in *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+	out, err := s.MemoryObjectStore.ListObjectsV2(ctx, in)
+	if err != nil {
+		return nil, err
+	}
+	for _, obj := range out.Contents {
+		obj.LastModified = nil
+		if at, ok := s.modified[aws.StringValue(obj.Key)]; ok {
+			obj.LastModified = aws.Time(at)
+		}
+	}
+	return out, nil
+}
+
+// A prune keeps the key it is told to and every object not older than the
+// cutoff, counting those it kept for age, and an undated object counts as
+// young (ADR-0003:S7).
+func TestS7_PruneObjectsKeepsThePointerAndEveryYoungObject(t *testing.T) {
+	const owner = "i-0123456789abcdef0"
+	prefix := uefistore.KeyPrefix(owner)
+	cutoff := time.Unix(1700000000, 0)
+	keep, stale, young, undated, other := prefix+"keep", prefix+"stale", prefix+"young", prefix+"undated", uefistore.KeyPrefix("i-0fedcba9876543210")+"stale"
+	store := &datedStore{MemoryObjectStore: objectstore.NewMemoryObjectStore(), modified: map[string]time.Time{
+		keep:  cutoff.Add(-time.Hour),
+		stale: cutoff.Add(-time.Second),
+		young: cutoff,
+		other: cutoff.Add(-time.Hour),
+	}}
+	for _, k := range []string{keep, stale, young, undated, other} {
+		putObject(t, store, k)
+	}
+
+	res, err := uefistore.PruneObjects(t.Context(), store, bucket, owner, keep, cutoff)
+
+	require.NoError(t, err)
+	assert.Equal(t, uefistore.PruneResult{Deleted: 1, Young: 2}, res)
+	assert.ElementsMatch(t, []string{keep, young, undated}, keysUnder(t, store, prefix),
+		"ADR-0003:S7: a prune must keep the pointed object and every object younger than the cutoff")
+	assert.Equal(t, []string{other}, keysUnder(t, store, uefistore.KeyPrefix("i-0fedcba9876543210")), "another owner's prefix is untouched")
+}
+
+// With no key to keep, or one outside the owner's prefix, a prune cannot tell
+// the pointer from a leftover, so it deletes nothing (ADR-0003:S7).
+func TestS7_PruneObjectsWithoutAKeyToKeepDeletesNothing(t *testing.T) {
+	const owner = "i-0123456789abcdef0"
+	prefix := uefistore.KeyPrefix(owner)
+	store := &datedStore{MemoryObjectStore: objectstore.NewMemoryObjectStore(), modified: map[string]time.Time{
+		prefix + "stale": time.Unix(1, 0),
+	}}
+	putObject(t, store, prefix+"stale")
+
+	for _, keep := range []string{"", uefistore.KeyPrefix("i-0fedcba9876543210") + "stale", prefix + "nested/key"} {
+		_, err := uefistore.PruneObjects(t.Context(), store, bucket, owner, keep, time.Now())
+		require.Error(t, err, keep)
+	}
+	_, err := uefistore.PruneObjects(t.Context(), store, bucket, "i-*", prefix+"stale", time.Now())
+	require.Error(t, err)
+
+	assert.Equal(t, []string{prefix + "stale"}, keysUnder(t, store, prefix),
+		"ADR-0003:S7: a prune with no valid pointer to keep must delete nothing")
+}

spinifex/domains/ec2/instance/uefistore/uefistore.go

@@ -1,6 +1,6 @@
 // Package uefistore holds the types that describe an instance's UEFI variable
-// store and the bucket of store records that carry its deletion and write-back
-// obligations past the life of the instance record.
+// store, the bucket of store records that carry its pointer and deletion
+// obligation past the life of the instance record, and the claim that fences it.
 package uefistore
 
 import (
@@ -37,8 +37,8 @@ const (
 var ErrInvalidRecord = errors.New("uefistore: invalid store record")
 
 // ErrDeleting is returned for a change that would return a deleting store to
-// live. Deletion follows only a terminate or an operator's forget, and both are
-// final, so a writer that meets it drops whatever it was about to record.
+// live. Deletion follows only a decided terminate, which is final, so a writer
+// that meets it drops whatever it was about to record.
 var ErrDeleting = errors.New("uefistore: store is being deleted")
 
 // FirmwareIdentity names the firmware a variable store was written under. A
@@ -53,22 +53,27 @@ type FirmwareIdentity struct {
 // UEFIStoreRecord is the durable owner of one instance's variable store
 // objects. It outlives the instance record, so an obligation recorded here is
 // not lost when a terminated record expires.
+//
+// Pointer names the only version of the store the record keeps, the object the
+// instance boots from, and only a writer whose claim token has generation
+// Generation may move it.
 type UEFIStoreRecord struct {
-	SchemaVersion uint16         `json:"schema_version"`
-	InstanceID    string         `json:"instance_id"`
-	AccountID     string         `json:"account_id"`
-	State         string         `json:"state"`
-	WriteBack     *UEFIWriteBack `json:"write_back,omitempty"`
+	SchemaVersion uint16           `json:"schema_version"`
+	InstanceID    string           `json:"instance_id"`
+	AccountID     string           `json:"account_id"`
+	State         string           `json:"state"`
+	Generation    uint64           `json:"generation"`
+	Pointer       *UEFIVarStoreRef `json:"pointer,omitempty"`
 }
 
-// UEFIWriteBack is a write-back that failed and is still owed. Key is the
-// object every retry puts to, so retries leave no extra objects; SeededFrom is
-// the pointer Node seeded from, which a retry must still find on the record.
-type UEFIWriteBack struct {
-	Node       string    `json:"node"`
-	Key        string    `json:"key"`
-	SeededFrom string    `json:"seeded_from"`
-	Since      time.Time `json:"since"`
+// UEFIVarStoreRef names the store object an instance launches from, with the
+// firmware it was written under, so a launch can check it without reading the
+// object's metadata.
+type UEFIVarStoreRef struct {
+	Key        string           `json:"key"`
+	Firmware   FirmwareIdentity `json:"firmware"`
+	WriterNode string           `json:"writer_node"`
+	WrittenAt  time.Time        `json:"written_at"`
 }
 
 // KeyPrefix is the object-store prefix every store object of ownerID (an
@@ -83,8 +88,8 @@ func ownsKey(ownerID, key string) bool {
 	return ok && name != "" && !strings.Contains(name, "/")
 }
 
-// validate checks rec against key. A write-back key outside the instance's own
-// prefix is refused because a superseded obligation deletes that key.
+// validate checks rec against key. The pointer must name one object directly
+// under the instance's own prefix, because a deletion acts on it.
 func (rec *UEFIStoreRecord) validate(key string) error {
 	switch {
 	case rec.SchemaVersion > RecordSchemaVersion:
@@ -98,29 +103,32 @@ func (rec *UEFIStoreRecord) validate(key string) error {
 		return fmt.Errorf("%w: instance_id %s stored at key %s", ErrInvalidRecord, rec.InstanceID, key)
 	case rec.State != StateLive && rec.State != StateDeleting:
 		return fmt.Errorf("%w: %s has state %q", ErrInvalidRecord, key, rec.State)
-	case rec.State == StateDeleting && rec.WriteBack != nil:
-		// A write-back retry puts an object, which would land behind the delete.
-		return fmt.Errorf("%w: %s is deleting with a write-back still owed", ErrInvalidRecord, key)
 	}
-	if wb := rec.WriteBack; wb != nil {
-		switch {
-		case wb.Node == "":
-			return fmt.Errorf("%w: %s write-back names no node", ErrInvalidRecord, key)
-		case wb.Since.IsZero():
-			return fmt.Errorf("%w: %s write-back has no since", ErrInvalidRecord, key)
-		case !ownsKey(rec.InstanceID, wb.Key):
-			return fmt.Errorf("%w: %s write-back key %q is not under %s", ErrInvalidRecord, key, wb.Key, KeyPrefix(rec.InstanceID))
-		case !ownsKey(rec.InstanceID, wb.SeededFrom):
-			return fmt.Errorf("%w: %s write-back seeded_from %q is not under %s", ErrInvalidRecord, key, wb.SeededFrom, KeyPrefix(rec.InstanceID))
+	if p := rec.Pointer; p != nil {
+		if err := p.validate(rec.InstanceID); err != nil {
+			return fmt.Errorf("%w: %s pointer: %w", ErrInvalidRecord, key, err)
 		}
 	}
 	return nil
 }
 
+// validate checks ref as a pointer of instanceID's store.
+func (ref *UEFIVarStoreRef) validate(instanceID string) error {
+	switch {
+	case !ownsKey(instanceID, ref.Key):
+		return fmt.Errorf("key %q is not under %s", ref.Key, KeyPrefix(instanceID))
+	case ref.Firmware.Arch == "" || ref.Firmware.VarsSize <= 0:
+		return fmt.Errorf("key %q names no firmware arch and VARS size", ref.Key)
+	case ref.WriterNode == "":
+		return fmt.Errorf("key %q names no writer node", ref.Key)
+	}
+	return nil
+}
+
 func bucketConfig() kvstore.Config {
 	return kvstore.Config{
 		Name:        BucketName,
-		Description: "Spinifex UEFI variable store records (deletion and write-back obligations)",
+		Description: "Spinifex UEFI variable store records (pointers and deletion obligations)",
 		History:     1,
 		// Left false: a lost bucket has lost obligations, and an empty
 		// recreated one would report every store as having none.
@@ -172,6 +180,19 @@ func (s *Store) Get(ctx context.Context, instanceID string) (*UEFIStoreRecord, u
 	return rec, rev, nil
 }
 
+// LeaderGet is Get answered by the stream leader, for a caller that deletes
+// objects by what the record does not point at.
+func (s *Store) LeaderGet(ctx context.Context, instanceID string) (*UEFIStoreRecord, uint64, error) {
+	rec, rev, err := s.kv.LeaderGet(ctx, instanceID)
+	if err != nil {
+		return nil, 0, err
+	}
+	if err := rec.validate(instanceID); err != nil {
+		return nil, 0, err
+	}
+	return rec, rev, nil
+}
+
 // Create writes a new record, which must be live: a store is recorded before
 // its first object is written. An existing record is kvstore.ErrExists. Every
 // write stamps rec with RecordSchemaVersion.

spinifex/domains/ec2/instance/uefistore/uefistore_test.go

@@ -30,15 +30,25 @@ func liveRecord() *uefistore.UEFIStoreRecord {
 	return &uefistore.UEFIStoreRecord{InstanceID: instanceID, AccountID: "111122223333", State: uefistore.StateLive}
 }
 
-func writeBack() *uefistore.UEFIWriteBack {
-	return &uefistore.UEFIWriteBack{
-		Node:       "node-2",
-		Key:        uefistore.KeyPrefix(instanceID) + "b",
-		SeededFrom: uefistore.KeyPrefix(instanceID) + "a",
-		Since:      time.Date(2026, 11, 3, 1, 2, 3, 0, time.UTC),
+func pointer(name string) *uefistore.UEFIVarStoreRef {
+	return &uefistore.UEFIVarStoreRef{
+		Key:        keyOf(name),
+		Firmware:   uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: 540672, CodeSHA256: "c0de"},
+		WriterNode: "node-2",
+		WrittenAt:  time.Date(2026, 11, 3, 1, 2, 3, 0, time.UTC),
 	}
 }
 
+func keyOf(name string) string { return uefistore.KeyPrefix(instanceID) + name }
+
+// pointedRecord is a live record stamped at generation 7 that points at "b".
+func pointedRecord() *uefistore.UEFIStoreRecord {
+	rec := liveRecord()
+	rec.Generation = 7
+	rec.Pointer = pointer("b")
+	return rec
+}
+
 func rawPut(t *testing.T, js jetstream.JetStream, key, body string) {
 	t.Helper()
 	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
@@ -66,8 +76,7 @@ func TestOpen_CreatesBucketWithNoTTLAndStampsVersion(t *testing.T) {
 
 func TestCreate_RoundTripsAndStampsSchemaVersion(t *testing.T) {
 	s, _ := openStore(t)
-	rec := liveRecord()
-	rec.WriteBack = writeBack()
+	rec := pointedRecord()
 
 	rev, err := s.Create(t.Context(), rec)
 	require.NoError(t, err)
@@ -82,9 +91,7 @@ func TestCreate_RoundTripsAndStampsSchemaVersion(t *testing.T) {
 // The wire names are what every release reads, so they are pinned.
 func TestRecord_WireShape(t *testing.T) {
 	s, js := openStore(t)
-	rec := liveRecord()
-	rec.WriteBack = writeBack()
-	_, err := s.Create(t.Context(), rec)
+	_, err := s.Create(t.Context(), pointedRecord())
 	require.NoError(t, err)
 
 	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
@@ -96,11 +103,12 @@ func TestRecord_WireShape(t *testing.T) {
 		"instance_id": "i-0123456789abcdef0",
 		"account_id": "111122223333",
 		"state": "live",
-		"write_back": {
-			"node": "node-2",
+		"generation": 7,
+		"pointer": {
 			"key": "uefi/i-0123456789abcdef0/b",
-			"seeded_from": "uefi/i-0123456789abcdef0/a",
-			"since": "2026-11-03T01:02:03Z"
+			"firmware": {"arch": "x86_64", "vars_size": 540672, "code_sha256": "c0de"},
+			"writer_node": "node-2",
+			"written_at": "2026-11-03T01:02:03Z"
 		}
 	}`, string(entry.Value()))
 }
@@ -144,8 +152,7 @@ func TestCompareAndSet_StaleRevisionConflictsAndWritesNothing(t *testing.T) {
 	first, err := s.Create(t.Context(), liveRecord())
 	require.NoError(t, err)
 
-	moved := liveRecord()
-	moved.WriteBack = writeBack()
+	moved := pointedRecord()
 	second, err := s.CompareAndSet(t.Context(), moved, first)
 	require.NoError(t, err)
 	assert.Greater(t, second, first)
@@ -159,7 +166,7 @@ func TestCompareAndSet_StaleRevisionConflictsAndWritesNothing(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, second, rev)
 	assert.Equal(t, uefistore.StateLive, got.State)
-	assert.Equal(t, writeBack(), got.WriteBack)
+	assert.Equal(t, pointer("b"), got.Pointer)
 }
 
 // Two actors that read the same revision race to change it; exactly one may
@@ -179,12 +186,11 @@ func TestCompareAndSet_ConcurrentWritersFromOneReadHaveOneWinner(t *testing.T) {
 	)
 	for i := range writers {
 		wg.Go(func() {
-			rec := liveRecord()
-			rec.WriteBack = writeBack()
-			rec.WriteBack.Node = fmt.Sprintf("node-%d", i)
+			rec := pointedRecord()
+			rec.Pointer.WriterNode = fmt.Sprintf("node-%d", i)
 			if _, err := s.CompareAndSet(context.Background(), rec, rev); err == nil {
 				mu.Lock()
-				wins = append(wins, rec.WriteBack.Node)
+				wins = append(wins, rec.Pointer.WriterNode)
 				mu.Unlock()
 			} else {
 				assert.ErrorIs(t, err, kvstore.ErrConflict)
@@ -196,7 +202,7 @@ func TestCompareAndSet_ConcurrentWritersFromOneReadHaveOneWinner(t *testing.T) {
 	require.Len(t, wins, 1)
 	got, _, err := s.Get(t.Context(), instanceID)
 	require.NoError(t, err)
-	assert.Equal(t, wins[0], got.WriteBack.Node)
+	assert.Equal(t, wins[0], got.Pointer.WriterNode)
 }
 
 func TestCompareAndDelete_StaleRevisionKeepsRecord(t *testing.T) {
@@ -217,29 +223,20 @@ func TestCompareAndDelete_StaleRevisionKeepsRecord(t *testing.T) {
 	require.ErrorIs(t, err, kvstore.ErrNotFound)
 }
 
-// A superseded obligation deletes its recorded key, so a key outside the
+// A deletion acts on the key the record points at, so a key outside the
 // instance's own prefix must never reach the bucket.
 func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
+	other := uefistore.KeyPrefix("i-0fedcba9876543210") + "b"
 	cases := map[string]func(*uefistore.UEFIStoreRecord){
-		"unknown state":            func(r *uefistore.UEFIStoreRecord) { r.State = "gone" },
-		"not an instance ID":       func(r *uefistore.UEFIStoreRecord) { r.InstanceID = "ami-0123456789abcdef0" },
-		"write-back with no node":  func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Node = "" },
-		"write-back with no since": func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Since = time.Time{} },
-		"write-back key under another instance": func(r *uefistore.UEFIStoreRecord) {
-			r.WriteBack.Key = uefistore.KeyPrefix("i-0fedcba9876543210") + "b"
-		},
-		"write-back key under an image": func(r *uefistore.UEFIStoreRecord) {
-			r.WriteBack.Key = uefistore.KeyPrefix("ami-0123456789abcdef0") + "b"
-		},
-		"write-back key is the prefix itself": func(r *uefistore.UEFIStoreRecord) {
-			r.WriteBack.Key = uefistore.KeyPrefix(instanceID)
-		},
-		"write-back key nested below the prefix": func(r *uefistore.UEFIStoreRecord) {
-			r.WriteBack.Key = uefistore.KeyPrefix(instanceID) + "x/y"
-		},
-		"write-back seeded from another instance": func(r *uefistore.UEFIStoreRecord) {
-			r.WriteBack.SeededFrom = uefistore.KeyPrefix("i-0fedcba9876543210") + "a"
-		},
+		"unknown state":                   func(r *uefistore.UEFIStoreRecord) { r.State = "gone" },
+		"not an instance ID":              func(r *uefistore.UEFIStoreRecord) { r.InstanceID = "ami-0123456789abcdef0" },
+		"pointer under another instance":  func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = other },
+		"pointer under an image":          func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = uefistore.KeyPrefix("ami-0123456789abcdef0") + "b" },
+		"pointer is the prefix itself":    func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = uefistore.KeyPrefix(instanceID) },
+		"pointer nested below the prefix": func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = keyOf("x/y") },
+		"pointer with no firmware arch":   func(r *uefistore.UEFIStoreRecord) { r.Pointer.Firmware.Arch = "" },
+		"pointer with no VARS size":       func(r *uefistore.UEFIStoreRecord) { r.Pointer.Firmware.VarsSize = 0 },
+		"pointer with no writer node":     func(r *uefistore.UEFIStoreRecord) { r.Pointer.WriterNode = "" },
 	}
 	for name, mutate := range cases {
 		t.Run(name, func(t *testing.T) {
@@ -247,8 +244,7 @@ func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
 			rev, err := s.Create(t.Context(), liveRecord())
 			require.NoError(t, err)
 
-			rec := liveRecord()
-			rec.WriteBack = writeBack()
+			rec := pointedRecord()
 			mutate(rec)
 			_, err = s.CompareAndSet(t.Context(), rec, rev)
 			require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
@@ -256,7 +252,7 @@ func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
 			got, gotRev, err := s.Get(t.Context(), instanceID)
 			require.NoError(t, err)
 			assert.Equal(t, rev, gotRev, "an invalid record must not be written")
-			assert.Nil(t, got.WriteBack)
+			assert.Nil(t, got.Pointer)
 		})
 	}
 }
@@ -265,11 +261,11 @@ func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
 // is reported rather than decoded into something a reaper would act on.
 func TestGet_RefusesRecordsItCannotTrust(t *testing.T) {
 	cases := map[string]string{
-		"newer schema":       `{"schema_version":2,"instance_id":"i-0123456789abcdef0","state":"live"}`,
-		"no schema":          `{"instance_id":"i-0123456789abcdef0","state":"live"}`,
-		"another instance":   `{"schema_version":1,"instance_id":"i-0fedcba9876543210","state":"deleting"}`,
-		"unknown state":      `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"purged"}`,
-		"foreign write-back": `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"live","write_back":{"node":"n","key":"uefi/i-0fedcba9876543210/b","seeded_from":"uefi/i-0123456789abcdef0/a","since":"2026-11-03T00:00:00Z"}}`,
+		"newer schema":     `{"schema_version":2,"instance_id":"i-0123456789abcdef0","state":"live"}`,
+		"no schema":        `{"instance_id":"i-0123456789abcdef0","state":"live"}`,
+		"another instance": `{"schema_version":1,"instance_id":"i-0fedcba9876543210","state":"deleting"}`,
+		"unknown state":    `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"purged"}`,
+		"foreign pointer":  `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"live","generation":3,"pointer":{"key":"uefi/i-0fedcba9876543210/b","firmware":{"arch":"x86_64","vars_size":540672},"writer_node":"n"}}`,
 	}
 	for name, body := range cases {
 		t.Run(name, func(t *testing.T) {

spinifex/domains/ec2/instance/volume_held_elsewhere_test.go

@@ -102,3 +102,38 @@ func TestStartPaths_ReportAVolumeHeldElsewhereByCode(t *testing.T) {
 		assert.Equal(t, awserrors.ErrorIncorrectState, awserrors.ValidErrorCodeFromError(err))
 	})
 }
+
+// uefiClaimHeldMounter fails the launch the way a start does while a live
+// node holds the instance's UEFI variable store.
+type uefiClaimHeldMounter struct{ raceVolumeMounter }
+
+func (uefiClaimHeldMounter) Mount(context.Context, *vm.VM) error {
+	return fmt.Errorf("%w: node-a holds i-uefi: %w", vm.ErrUEFIClaimHeld, vm.ErrMountRetryable)
+}
+
+// A held UEFI store is a capacity refusal on both start paths, so the caller
+// tries another node rather than reporting a fault.
+func TestStartPaths_ReportAHeldUEFIStoreAsCapacity(t *testing.T) {
+	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
+	types, _ := defaultPrepareInstanceTypes()
+
+	t.Run("claimed relaunch", func(t *testing.T) {
+		mgr := vm.NewManager()
+		mgr.SetDeps(vm.Deps{NodeID: "node-b", VolumeMounter: uefiClaimHeldMounter{}})
+		svc := &InstanceServiceImpl{resourceMgr: &fakeResourceCapacityProvider{instanceTypes: types}, vmMgr: mgr}
+
+		err := svc.launchClaimedInstance(t.Context(), &vm.VM{ID: "i-uefi-claim", AccountID: "acc", InstanceType: "t3.micro", Status: vm.StatePending}, func() {})
+		assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, awserrors.ValidErrorCodeFromError(err))
+	})
+
+	t.Run("live owner", func(t *testing.T) {
+		const id = "i-uefi-live"
+		mgr := mgrWith(map[string]*vm.VM{id: {ID: id, Status: vm.StateStopped, InstanceType: "t3.micro"}})
+		mgr.SetDeps(vm.Deps{NodeID: "node-b", VolumeMounter: uefiClaimHeldMounter{}})
+		v, _ := mgr.Get(id)
+		svc := &InstanceServiceImpl{resourceMgr: &fakeResourceCapacityProvider{instanceTypes: types}, vmMgr: mgr}
+
+		err := svc.StartInstance(t.Context(), v, ec2v1.EC2InstanceCommand{ID: id})
+		assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, awserrors.ValidErrorCodeFromError(err))
+	})
+}

spinifex/domains/ec2/snapshot/delete_snapshot_pagination_test.go

@@ -1,83 +0,0 @@
-package snapshot_test
-
-import (
-	"context"
-	"fmt"
-	"strings"
-	"testing"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/mulgadc/spinifex/internal/testkit/pagedstore"
-	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
-	ec2snapshot "github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
-	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-const testAccountID = "111122223333"
-
-// TestDeleteSnapshot_RemovesEveryObjectAcrossMultiplePages proves DeleteSnapshot
-// follows ListObjectsV2's continuation token to exhaustion. Before this fix it
-// read only the first page: above the page size it deleted a partial set of
-// objects, reported success, and left the rest orphaned under the snapshot
-// prefix.
-func TestDeleteSnapshot_RemovesEveryObjectAcrossMultiplePages(t *testing.T) {
-	const pageSize = 3
-	store := pagedstore.New(pageSize)
-	cfg := &config.Config{
-		Predastore: config.PredastoreConfig{
-			Bucket:    "test-bucket",
-			AccessKey: "test-owner-123",
-		},
-	}
-	svc := ec2snapshot.NewSnapshotServiceImplWithStore(cfg, store, nil)
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})
-	svc.SetEBSProvider(provider)
-
-	_, err := provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{
-		Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-1",
-		CapacityRange:    ebsprovider.CapacityRange{RequiredBytes: 10 * 1024 * 1024 * 1024},
-		AvailabilityZone: "us-east-1a",
-	})
-	require.NoError(t, err)
-	require.NoError(t, ebsmetadata.NewStore(store, "test-bucket").PutVolume(context.Background(), ebsmetadata.Volume{
-		VolumeID: "vol-1", TenantID: testAccountID, CapacityGiB: 10, AvailabilityZone: "us-east-1a",
-	}))
-
-	snap, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{
-		VolumeId: aws.String("vol-1"),
-	}, testAccountID)
-	require.NoError(t, err)
-	snapshotID := *snap.SnapshotId
-
-	// Inflate the snapshot's object prefix well past one page: metadata.json
-	// (written by CreateSnapshot) plus 10 synthetic block objects.
-	const extraObjects = 10
-	for i := range extraObjects {
-		_, err := store.PutObject(context.Background(), &s3.PutObjectInput{
-			Bucket: aws.String("test-bucket"),
-			Key:    aws.String(fmt.Sprintf("%s/block-%03d", snapshotID, i)),
-			Body:   strings.NewReader("x"),
-		})
-		require.NoError(t, err)
-	}
-
-	store.Calls = 0
-	_, err = svc.DeleteSnapshot(context.Background(), &ec2.DeleteSnapshotInput{
-		SnapshotId: snap.SnapshotId,
-	}, testAccountID)
-	require.NoError(t, err)
-
-	wantPages := (extraObjects + 1 + pageSize - 1) / pageSize
-	assert.GreaterOrEqual(t, store.Calls, wantPages, "the listing must span more than one page for this test to prove anything")
-
-	remaining, err := store.MemoryObjectStore.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{
-		Bucket: aws.String("test-bucket"), Prefix: aws.String(snapshotID + "/"),
-	})
-	require.NoError(t, err)
-	assert.Empty(t, remaining.Contents, "every object under the snapshot prefix must be deleted, not just the first page")
-}

spinifex/domains/ec2/snapshot/service_impl.go

@@ -13,7 +13,6 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/mulgadc/bluebottle/pkg/safecast"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
@@ -655,48 +654,15 @@ func (s *SnapshotServiceImpl) DeleteSnapshot(ctx context.Context, input *ec2.Del
 		return nil, providerDeleteError(snapshotID, err)
 	}
 
-	objects, _, err := objectstore.ListAll(ctx, s.store, &s3.ListObjectsV2Input{
-		Bucket: aws.String(s.config.Predastore.Bucket),
-		Prefix: aws.String(snapshotID + "/"),
-	})
-	if err != nil {
-		slog.ErrorContext(ctx, "DeleteSnapshot failed to list objects", "snapshotId", snapshotID, "err", err)
-		return nil, errors.New(awserrors.ErrorServerInternal)
-	}
-
-	// A key the sweep could not delete has to keep the document alive. The
-	// listing no longer enumerates snap- prefixes, so deleting the document over
-	// surviving chunks strands them where nothing can see or reclaim them.
-	var sweepErr error
-	for _, obj := range objects {
-		if obj == nil || obj.Key == nil {
-			continue
-		}
-		_, err := s.store.DeleteObject(ctx, &s3.DeleteObjectInput{
-			Bucket: aws.String(s.config.Predastore.Bucket),
-			Key:    obj.Key,
-		})
-		if err != nil && !objectstore.IsNoSuchKeyError(err) {
-			slog.ErrorContext(ctx, "DeleteSnapshot failed to delete object", "key", *obj.Key, "err", err)
-			sweepErr = errors.Join(sweepErr, err)
-		}
-	}
-	if sweepErr != nil {
-		slog.ErrorContext(ctx, "DeleteSnapshot leaving the document in place after an incomplete sweep",
-			"snapshotId", snapshotID, "err", sweepErr)
-		return nil, errors.New(awserrors.ErrorServerInternal)
-	}
-
-	// The document no longer sits inside the swept prefix, so it has to go
-	// explicitly, and last: a crash with it still present leaves a snapshot that
-	// DescribeSnapshots shows and DeleteSnapshot can retry, rather than chunks
-	// nothing enumerates.
+	// The provider has removed the snapshot's storage. The document goes after
+	// it, so a provider failure above leaves a snapshot DescribeSnapshots shows
+	// and DeleteSnapshot can retry against an idempotent provider delete.
 	if err := s.metadata.DeleteSnapshot(ctx, accountID, snapshotID); err != nil && !objectstore.IsNoSuchKeyError(err) {
 		slog.ErrorContext(ctx, "DeleteSnapshot failed to delete metadata document", "snapshotId", snapshotID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
-	// Remove from KV after S3 cleanup. Failure is logged but not fatal —
+	// Remove from KV after the document. Failure is logged but not fatal —
 	// a phantom entry safely blocks volume deletion rather than allowing it.
 	if err := s.removeSnapshotRefForCleanup(ctx, cfg.VolumeID, snapshotID); err != nil {
 		slog.Warn("DeleteSnapshot failed to remove snapshot ref from KV", "snapshotId", snapshotID, "volumeId", cfg.VolumeID, "err", err)

spinifex/domains/ec2/snapshot/snapshot_partition_test.go

@@ -7,7 +7,7 @@ package snapshot
 import (
 	"context"
 	"errors"
-	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"slices"
 	"strings"
 	"testing"
 
@@ -18,15 +18,16 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-// The document no longer lives inside the prefix DeleteSnapshot sweeps, so it
-// has to be deleted explicitly. Miss that and DescribeSnapshots keeps returning
-// a snapshot whose chunks are gone.
+// The document lives outside the provider's storage, so it has to be deleted
+// explicitly. Miss that and DescribeSnapshots keeps returning a snapshot whose
+// storage is gone.
 func TestDeleteSnapshot_LeavesNoDocument(t *testing.T) {
 	ctx := context.Background()
 	svc, store := setupTestSnapshotService(t)
@@ -101,47 +102,59 @@ func TestDeleteSnapshot_DocumentDeleteFailureSurfaces(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
 
-// A chunk the sweep could not delete has to keep the document alive. The
-// listing no longer enumerates snap- prefixes, so deleting the document over
-// surviving chunks strands them where no API can see or reclaim them, and the
-// delete cannot be retried to finish.
-func TestDeleteSnapshot_ChunkSweepFailureRetainsDocument(t *testing.T) {
+// The provider owns the snapshot's storage and deletes it; the control plane
+// touches only metadata-store keys. An object under the snapshot ID survives
+// because the memory provider never wrote it and nothing else may delete it.
+func TestDeleteSnapshot_TouchesOnlyMetadataStoreKeys(t *testing.T) {
 	ctx := context.Background()
-	objects := &deleteFailingStore{MemoryObjectStore: objectstore.NewMemoryObjectStore()}
+	objects := recordingstore.New()
 	cfg := &config.Config{Predastore: config.PredastoreConfig{Bucket: "test-bucket"}}
 	svc := NewSnapshotServiceImplWithStore(cfg, objects, nil)
 	svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}))
-	createTestVolume(t, svc, objects.MemoryObjectStore, "vol-sweepfail", 8)
+	createTestVolume(t, svc, objects.MemoryObjectStore, "vol-scope", 8)
 
-	snap, err := svc.CreateSnapshot(ctx, &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-sweepfail")}, testAccountID)
+	snap, err := svc.CreateSnapshot(ctx, &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-scope")}, testAccountID)
 	require.NoError(t, err)
 	snapshotID := aws.StringValue(snap.SnapshotId)
-
-	// A block chunk under the swept prefix, which is viperblock's half of the
-	// snapshot and the thing the document must not outlive.
-	_, err = objects.PutObject(ctx, &s3.PutObjectInput{
-		Bucket: aws.String("test-bucket"), Key: aws.String(snapshotID + "/chunk-0"),
-		Body: strings.NewReader("blocks"),
+	providerKey := snapshotID + "/chunk-0"
+	_, err = objects.MemoryObjectStore.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String("test-bucket"), Key: aws.String(providerKey), Body: strings.NewReader("blocks"),
 	})
 	require.NoError(t, err)
-	objects.failPrefix = snapshotID + "/"
 
+	objects.Reset()
 	_, err = svc.DeleteSnapshot(ctx, &ec2.DeleteSnapshotInput{SnapshotId: snap.SnapshotId}, testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-
-	_, err = svc.metadata.GetSnapshot(ctx, testAccountID, snapshotID)
-	require.NoError(t, err, "an incomplete sweep must leave the document so the delete can be retried")
-
-	listed, err := svc.DescribeSnapshots(ctx, &ec2.DescribeSnapshotsInput{}, testAccountID)
 	require.NoError(t, err)
-	require.Len(t, listed.Snapshots, 1, "the snapshot must stay visible while its chunks survive")
-	assert.Equal(t, snapshotID, aws.StringValue(listed.Snapshots[0].SnapshotId))
+
+	// Spelled out rather than derived: this is the layout claim itself.
+	metadataPrefixes := []string{
+		"spinifex/ebsmetadata/v2/snapshots/",
+		"spinifex/ebsmetadata/v2/volumes/",
+		"spinifex/ebsmetadata/v2/amis/",
+	}
+	touched := map[string][]string{
+		"listed":  objects.ListPrefixes(),
+		"read":    objects.Gets(),
+		"headed":  objects.Heads(),
+		"written": objects.Puts(),
+		"deleted": objects.Deletes(),
+	}
+	for verb, keys := range touched {
+		for _, key := range keys {
+			assert.True(t, slices.ContainsFunc(metadataPrefixes, func(prefix string) bool { return strings.HasPrefix(key, prefix) }),
+				"DeleteSnapshot %s %q, outside the metadata store", verb, key)
+		}
+	}
+	snapshotKey, err := ebsmetadata.SnapshotKey(testAccountID, snapshotID)
+	require.NoError(t, err)
+	assert.Equal(t, []string{snapshotKey}, objects.Deletes(), "the snapshot's document is the one key it deletes")
+
+	_, err = objects.MemoryObjectStore.HeadObject(ctx, &s3.HeadObjectInput{Bucket: aws.String("test-bucket"), Key: aws.String(providerKey)})
+	require.NoError(t, err, "an object in the provider's storage must be left to the provider")
 }
 
 // deleteFailingStore fails DeleteObject for keys under failPrefix, leaving
-// every other operation intact. failPrefix is settable after construction so a
-// test can target a generated snapshot ID.
+// every other operation intact.
 type deleteFailingStore struct {
 	*objectstore.MemoryObjectStore
 
@@ -155,48 +168,6 @@ func (s *deleteFailingStore) DeleteObject(ctx context.Context, input *s3.DeleteO
 	return s.MemoryObjectStore.DeleteObject(ctx, input)
 }
 
-// racingDeleteStore lets a concurrent deleter win every DeleteObject once race
-// is set, so the caller's own delete answers NoSuchKey as Predastore would.
-type racingDeleteStore struct {
-	*objectstore.MemoryObjectStore
-
-	race bool
-}
-
-func (s *racingDeleteStore) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
-	if s.race {
-		_, _ = s.MemoryObjectStore.DeleteObject(ctx, input)
-	}
-	return s.MemoryObjectStore.DeleteObject(ctx, input)
-}
-
-// A chunk or document another deleter already removed is deleted, not a
-// failure: the sweep and the document delete must both finish.
-func TestDeleteSnapshot_ConcurrentlyDeletedKeysAreDone(t *testing.T) {
-	ctx := context.Background()
-	objects := &racingDeleteStore{MemoryObjectStore: objectstore.NewMemoryObjectStore()}
-	cfg := &config.Config{Predastore: config.PredastoreConfig{Bucket: "test-bucket"}}
-	svc := NewSnapshotServiceImplWithStore(cfg, objects, nil)
-	svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}))
-	createTestVolume(t, svc, objects.MemoryObjectStore, "vol-race", 8)
-
-	snap, err := svc.CreateSnapshot(ctx, &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-race")}, testAccountID)
-	require.NoError(t, err)
-	snapshotID := aws.StringValue(snap.SnapshotId)
-	_, err = objects.PutObject(ctx, &s3.PutObjectInput{
-		Bucket: aws.String("test-bucket"), Key: aws.String(snapshotID + "/chunk-0"),
-		Body: strings.NewReader("blocks"),
-	})
-	require.NoError(t, err)
-	objects.race = true
-
-	_, err = svc.DeleteSnapshot(ctx, &ec2.DeleteSnapshotInput{SnapshotId: snap.SnapshotId}, testAccountID)
-	require.NoError(t, err)
-
-	_, err = svc.metadata.GetSnapshot(ctx, testAccountID, snapshotID)
-	assert.True(t, objectstore.IsNoSuchKeyError(err), "the document must be gone, got %v", err)
-}
-
 // Isolation must be enforced *by* the read, not by a filter after it. A
 // whole-tree listing plus a correct in-memory filter would return the same
 // snapshots, so this asserts the access pattern: the listing prefix is the

spinifex/domains/ecs/eni_test.go

@@ -322,7 +322,7 @@ func TestReaper_Awsvpc_ReleasesENI(t *testing.T) {
 	rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
 	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
 
-	sc := NewScheduler(nc, svc, "test-holder")
+	sc := testScheduler(t, nc, svc, "test-holder")
 	sc.reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
 
 	assert.Equal(t, 1, eni.releaseCalls)

spinifex/domains/ecs/enumeration_failure_test.go

@@ -27,7 +27,7 @@ func dropRevisit(pass func(context.Context) (time.Duration, error)) func(context
 // drained, services never relaunched, stopped tasks never pruned.
 func TestSchedulerPasses_EnumerationFailureIsReported(t *testing.T) {
 	svc, nc := newTestService(t)
-	sc := NewScheduler(nc, svc, "test-holder")
+	sc := testScheduler(t, nc, svc, "test-holder")
 	_, err := svc.CreateCluster(context.Background(), &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
 	require.NoError(t, err)
 

spinifex/domains/ecs/instance_recovery_test.go

@@ -49,7 +49,7 @@ func TestRegister_RestoresReapedInstanceToActive(t *testing.T) {
 	rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
 	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
 
-	NewScheduler(nc, svc, "test-holder").reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
+	testScheduler(t, nc, svc, "test-holder").reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
 
 	reaped := instanceStatus(t, svc, "web", "i-1")
 	require.Equal(t, InstanceStatusDraining, reaped.Status)
@@ -89,7 +89,7 @@ func TestReaper_ReleasesCapacityAndRecoversClean(t *testing.T) {
 	seeded.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
 	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &seeded))
 
-	NewScheduler(nc, svc, "test-holder").reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
+	testScheduler(t, nc, svc, "test-holder").reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
 
 	reaped := instanceStatus(t, svc, "web", "i-1")
 	require.Equal(t, InstanceStatusDraining, reaped.Status)

spinifex/domains/ecs/revisit_test.go

@@ -50,7 +50,7 @@ func schedulerRig(t *testing.T) (*Scheduler, *Service, jetstream.KeyValue) {
 	registerInstance(t, svc, "web", "i-1", 4096, 8192)
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
-	return NewScheduler(nc, svc, "test-holder"), svc, kv
+	return testScheduler(t, nc, svc, "test-holder"), svc, kv
 }
 
 // runOneTask places a task and returns its ID, read back from the record rather

spinifex/domains/ecs/scheduler.go

@@ -13,17 +13,19 @@ import (
 
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/reconciler"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
-// Scheduler timing. The leader lease lives in spinifex-ecs-leader (60s TTL); the
-// holder refreshes well inside that. The reaper marks an instance DRAINING after
-// three missed 30s heartbeats.
+// KeyScheduler is the ECS scheduler's leader lease in the sync bucket.
+const KeyScheduler = "ecs.scheduler"
+
+// Scheduler timing. A follower asks again at the lease's renewal rate. The
+// reaper marks an instance DRAINING after three missed 30s heartbeats.
 const (
-	schedulerLeaderKey  = "scheduler"
-	leaseRefresh        = 20 * time.Second
+	schedulerLeaseTTL   = 60 * time.Second
+	leaseRefresh        = schedulerLeaseTTL / 3
 	reaperInterval      = 30 * time.Second
 	heartbeatTimeout    = 90 * time.Second
 	stoppedReasonReaped = "ContainerInstance disconnected"
@@ -59,17 +61,14 @@ const (
 	reconcileResync = 5 * time.Minute
 )
 
-// Scheduler is the per-daemon ECS control loop. A single leader (elected via the
-// shared leader bucket) owns the Layer-2 bus subscriptions and the heartbeat
+// Scheduler is the per-daemon ECS control loop. A single leader (elected on its
+// lease) owns the Layer-2 bus subscriptions and the heartbeat
 // reaper; losers idle and retry. Placement itself happens synchronously in the
 // RunTask handler, so a brief leaderless gap never blocks RunTask.
 type Scheduler struct {
-	nc     *nats.Conn
-	svc    *Service
-	holder string
-
-	lease    *kvlease.Lease
-	leaseErr error
+	nc    *nats.Conn
+	svc   *Service
+	lease *kvsync.Lease
 
 	// wake carries a bus message that changed something the service reconcile
 	// reads. Fed by the task-state subscription and not by the heartbeat one:
@@ -89,47 +88,28 @@ type Scheduler struct {
 	subs []*nats.Subscription
 }
 
-// NewScheduler constructs a Scheduler. holder identifies this daemon in the lease.
-func NewScheduler(nc *nats.Conn, svc *Service, holder string) *Scheduler {
-	// One slot: it means "something the reconcile reads has changed", and a
-	// second arriving before the pass runs says nothing the first did not.
-	sc := &Scheduler{nc: nc, svc: svc, holder: holder, wake: make(chan struct{}, 1)}
-	sc.lease, sc.leaseErr = kvlease.New(kvlease.Config{
-		Name:   "ecs/scheduler",
-		Bucket: sc.leaderBucket,
-		Key:    schedulerLeaderKey,
-		Holder: holder,
-		TTL:    KVBucketECSLeaderTTL,
-		Renew:  leaseRefresh,
-		Retry:  leaseRefresh,
-		// The leader owns the Layer-2 bus subscriptions and the instance-role
-		// converge pass. A node that can do neither stands down rather than hold
-		// the lease and process nothing.
-		OnGained: sc.onGained,
-		OnLost:   sc.unsubscribeBus,
-	})
-	return sc
+// NewSchedulerLease returns the ECS scheduler's leader lease for holder.
+func NewSchedulerLease(store kvsync.KVStore, holder string) *kvsync.Lease {
+	return kvsync.NewLease(store, KeyScheduler, holder, schedulerLeaseTTL)
 }
 
-func (sc *Scheduler) leaderBucket(ctx context.Context) (jetstream.KeyValue, error) {
-	js, err := jetstream.New(sc.nc)
-	if err != nil {
-		return nil, err
-	}
-	return InitLeaderBucket(ctx, js)
+// NewScheduler constructs a Scheduler that leads while it holds lease.
+func NewScheduler(nc *nats.Conn, svc *Service, lease *kvsync.Lease) *Scheduler {
+	// One slot: it means "something the reconcile reads has changed", and a
+	// second arriving before the pass runs says nothing the first did not.
+	return &Scheduler{nc: nc, svc: svc, lease: lease, wake: make(chan struct{}, 1)}
 }
 
 // Run drives the leadership + reaper loop until ctx is cancelled. It is intended
 // to run as a daemon-boot goroutine; panics are the caller's recover concern.
 func (sc *Scheduler) Run(ctx context.Context) {
-	if sc.leaseErr != nil {
-		slog.ErrorContext(ctx, "ECS scheduler: lease config invalid", "holder", sc.holder, "err", sc.leaseErr)
-		return
-	}
 	leaseDone := make(chan struct{})
 	go func() {
 		defer close(leaseDone)
-		sc.lease.Run(ctx)
+		// The leader owns the Layer-2 bus subscriptions and the instance-role
+		// converge pass. A node that can do neither stands down rather than hold
+		// the lease and process nothing.
+		kvsync.RunElection(ctx, sc.lease, sc.onGained, sc.unsubscribeBus)
 	}()
 
 	// Two loops rather than four tickers, split by what wakes them. Services are

spinifex/domains/ecs/scheduler_test.go

@@ -6,7 +6,9 @@ package ecs
 
 import (
 	"context"
+	"sync"
 	"testing"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ecs"
@@ -14,26 +16,67 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/nats-io/nats.go"
 )
 
+// testScheduler is holder's scheduler on a sync bucket of its own, as a
+// separate daemon's would be.
+func testScheduler(t *testing.T, nc *nats.Conn, svc *Service, holder string) *Scheduler {
+	t.Helper()
+	return NewScheduler(nc, svc, NewSchedulerLease(kvstore.OpenSyncBucket(testutil.NewJetStream(t, nc)), holder))
+}
+
+// electOnce runs sc's election with the scheduler's own callbacks and returns
+// what its first onGained returned. stop ends the election and waits for it.
+func electOnce(t *testing.T, sc *Scheduler) (stop func(), gainedErr error) {
+	t.Helper()
+	ctx, cancel := context.WithCancel(t.Context())
+	gained := make(chan error, 1)
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		kvsync.RunElection(ctx, sc.lease, func(ctx context.Context) error {
+			err := sc.onGained(ctx)
+			gained <- err
+			return err
+		}, sc.unsubscribeBus)
+	}()
+	stop = func() { cancel(); <-done }
+	t.Cleanup(stop)
+	select {
+	case gainedErr = <-gained:
+	case <-time.After(10 * time.Second):
+		t.Fatal("the election never gained the lease")
+	}
+	return sync.OnceFunc(stop), gainedErr
+}
+
+// requireLeaseKeyAbsent fails unless no node holds the scheduler's key.
+func requireLeaseKeyAbsent(t *testing.T, nc *nats.Conn, msg string) {
+	t.Helper()
+	_, _, err := kvstore.OpenSyncBucket(testutil.NewJetStream(t, nc)).Load(t.Context(), KeyScheduler)
+	require.ErrorIs(t, err, kvsync.ErrNotFound, msg)
+}
+
 // TestScheduler_StandsDownWithoutIAM is the fleet outage this guards: a leader
 // that cannot converge instance roles holds the lease, writes no sts:AssumeRole
 // grant, and every agent's credentials lapse an hour later.
 func TestScheduler_StandsDownWithoutIAM(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
+	_, nc, _ := testutil.StartTestJetStream(t)
 	svc := NewService(nc, testRegion, "")
-	sc := NewScheduler(nc, svc, "holder-a")
-	require.NoError(t, sc.leaseErr)
+	sc := testScheduler(t, nc, svc, "holder-a")
 
-	assert.False(t, sc.lease.TryAcquire(t.Context()), "a node without IAM must refuse the lease")
+	stop, err := electOnce(t, sc)
+	require.Error(t, err, "a node without IAM must refuse the lease")
+	stop()
 	assert.False(t, sc.lease.Held())
 	assert.Zero(t, sc.subCount())
 
 	// The key must be free, or a healthy node waits out the full TTL.
-	kv, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
-	_, err = kv.Get(t.Context(), schedulerLeaderKey)
-	require.Error(t, err)
+	requireLeaseKeyAbsent(t, nc, "a node that stood down left its key")
 }
 
 // TestConvergeInstanceRoles_RewritesProvisionedAccounts pins the pass that
@@ -49,7 +92,7 @@ func TestConvergeInstanceRoles_RewritesProvisionedAccounts(t *testing.T) {
 	require.NoError(t, err)
 
 	iam.RolePolicies[ecsInstanceRoleName] = `{"Version":"2012-10-17","Statement":[]}`
-	sc := NewScheduler(nc, svc, "holder-a")
+	sc := testScheduler(t, nc, svc, "holder-a")
 	require.NoError(t, sc.convergeInstanceRoles(t.Context()))
 
 	assert.Contains(t, iam.RolePolicies[ecsInstanceRoleName], "sts:AssumeRole")
@@ -65,7 +108,7 @@ func TestConvergeInstanceRoles_SkipsAccountsWithoutTheRole(t *testing.T) {
 	_, err := svc.CreateCluster(context.Background(), &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
 	require.NoError(t, err)
 
-	sc := NewScheduler(nc, svc, "holder-a")
+	sc := testScheduler(t, nc, svc, "holder-a")
 	require.NoError(t, sc.convergeInstanceRoles(t.Context()))
 
 	assert.Zero(t, iam.CreateRoleCalls)
@@ -77,7 +120,7 @@ func TestConvergeInstanceRoles_SkipsAccountsWithoutTheRole(t *testing.T) {
 // caller that reaches the pass without going through leader election.
 func TestConvergeInstanceRoles_NilIAMIsAnError(t *testing.T) {
 	_, nc, _ := testutil.StartTestJetStream(t)
-	sc := NewScheduler(nc, NewService(nc, testRegion, ""), "holder-a")
+	sc := testScheduler(t, nc, NewService(nc, testRegion, ""), "holder-a")
 
 	err := sc.convergeInstanceRoles(t.Context())
 	require.Error(t, err)
@@ -94,3 +137,59 @@ func TestECSInstanceRolePolicyDoc_GrantsAssumeRole(t *testing.T) {
 	assert.Contains(t, doc, `"ecs:*"`)
 	assert.Contains(t, doc, `"ecr:GetAuthorizationToken"`)
 }
+
+// containerInstanceStatus is i-1's status in the web cluster.
+func containerInstanceStatus(t *testing.T, svc *Service) string {
+	t.Helper()
+	di, err := svc.DescribeContainerInstances(context.Background(), &ecs.DescribeContainerInstancesInput{
+		Cluster: aws.String("web"), ContainerInstances: []*string{aws.String("i-1")},
+	}, testAccountID)
+	require.NoError(t, err)
+	return aws.StringValue(di.ContainerInstances[0].Status)
+}
+
+// A leader that loses the lease must drop the bus and stop reaping, or two
+// nodes would mark instances DRAINING and stop their tasks at once. Re-elected,
+// it takes both back up.
+func TestScheduler_LosingTheLeaseDropsTheBusAndTheReaperUntilReElection(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	svc := NewService(nc, testRegion, "").WithDeps(Deps{IAM: newStubIAM()})
+	_, err := svc.CreateCluster(context.Background(), &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
+	require.NoError(t, err)
+	registerInstance(t, svc, "web", "i-1", 1024, 2048)
+	kv, err := svc.bucket(t.Context(), testAccountID)
+	require.NoError(t, err)
+	var rec InstanceRecord
+	_, err = taskdefinition.GetJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec)
+	require.NoError(t, err)
+	rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
+
+	store := kvstore.OpenSyncBucket(testutil.NewJetStream(t, nc))
+	sc := NewScheduler(nc, svc, kvsync.NewLease(store, KeyScheduler, "holder-a", 300*time.Millisecond))
+	ctx, cancel := context.WithCancel(t.Context())
+	done := make(chan struct{})
+	go func() { defer close(done); kvsync.RunElection(ctx, sc.lease, sc.onGained, sc.unsubscribeBus) }()
+	t.Cleanup(func() { cancel(); <-done })
+
+	held := func() bool { return sc.lease.Held() && sc.subCount() == 4 }
+	require.Eventually(t, held, 5*time.Second, 10*time.Millisecond, "the leader subscribes to the four bus subjects")
+
+	syncKV, err := store.KV(t.Context())
+	require.NoError(t, err)
+	_, err = syncKV.Put(t.Context(), KeyScheduler, []byte(`{"holder":"holder-b","generation":0}`))
+	require.NoError(t, err)
+	require.Eventually(t, func() bool { return !sc.lease.Held() && sc.subCount() == 0 }, 5*time.Second, 10*time.Millisecond,
+		"a lost lease must drop every bus subscription")
+
+	revisit, err := sc.timerPass(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, leaseRefresh, revisit)
+	assert.Equal(t, "ACTIVE", containerInstanceStatus(t, svc), "a node that lost the lease must not reap")
+
+	require.NoError(t, syncKV.Delete(t.Context(), KeyScheduler))
+	require.Eventually(t, held, 5*time.Second, 10*time.Millisecond, "re-elected, the bus comes back")
+	_, err = sc.timerPass(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, "DRAINING", containerInstanceStatus(t, svc), "the re-elected leader reaps again")
+}

spinifex/domains/ecs/service_test.go

@@ -14,6 +14,7 @@ import (
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -571,7 +572,7 @@ func TestScheduler_ReapBucket_StopsStaleInstanceTasks(t *testing.T) {
 	rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
 	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
 
-	sc := NewScheduler(nc, svc, "test-holder")
+	sc := testScheduler(t, nc, svc, "test-holder")
 	sc.reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
 
 	di, err := svc.DescribeContainerInstances(context.Background(), &ecs.DescribeContainerInstancesInput{
@@ -586,32 +587,38 @@ func TestScheduler_ReapBucket_StopsStaleInstanceTasks(t *testing.T) {
 	assert.Equal(t, stoppedReasonReaped, aws.StringValue(dt.Tasks[0].StoppedReason))
 }
 
-// Leader election: first holder wins Create; a second holder is rejected.
+// Leader election: the first holder wins; a second node, and a second attempt
+// in the same process, are both refused while it leads.
 func TestScheduler_AcquireLease_SingleLeader(t *testing.T) {
 	_, nc, _ := testutil.StartTestJetStream(t)
 	svc := NewService(nc, testRegion, "").WithDeps(Deps{IAM: newStubIAM()})
-	a := NewScheduler(nc, svc, "holder-a")
-	b := NewScheduler(nc, svc, "holder-b")
-	require.True(t, a.lease.TryAcquire(t.Context()))
-	assert.False(t, b.lease.TryAcquire(t.Context()))
-	assert.True(t, a.lease.TryAcquire(t.Context()), "a holder re-attempting keeps leadership")
+	a := testScheduler(t, nc, svc, "holder-a")
+	b := testScheduler(t, nc, svc, "holder-b")
+	tok, err := a.lease.TryLock(t.Context())
+	require.NoError(t, err)
+	t.Cleanup(func() { a.lease.Unlock(context.Background(), tok) })
+
+	_, err = b.lease.TryLock(t.Context())
+	assert.ErrorAs(t, err, new(*kvsync.HeldError))
+	_, err = a.lease.TryLock(t.Context())
+	assert.ErrorAs(t, err, new(*kvsync.HeldError), "a second attempt in the leading process is refused")
+	assert.True(t, a.lease.Held(), "a refused second attempt keeps leadership")
 }
 
-// Leadership owns the Layer-2 bus subscriptions. This pins the wiring — gaining
-// the lease subscribes, losing it unsubscribes — which is the regression this
-// conversion could introduce and the one thing kvlease cannot check itself.
+// Leadership owns the Layer-2 bus subscriptions. This pins the wiring: gaining
+// the lease subscribes, losing it unsubscribes.
 func TestScheduler_LeadershipWiresBusSubscriptions(t *testing.T) {
 	_, nc, _ := testutil.StartTestJetStream(t)
 	svc := NewService(nc, testRegion, "").WithDeps(Deps{IAM: newStubIAM()})
-	sc := NewScheduler(nc, svc, "holder-a")
-	require.NoError(t, sc.leaseErr)
+	sc := testScheduler(t, nc, svc, "holder-a")
 
 	require.Zero(t, sc.subCount(), "subscriptions must not exist before election")
 
-	require.True(t, sc.lease.TryAcquire(t.Context()))
+	stop, err := electOnce(t, sc)
+	require.NoError(t, err)
 	assert.Equal(t, 4, sc.subCount(), "leader must own the register, heartbeat, task-state and reconcile-wake subscriptions")
 
-	sc.lease.Release(t.Context())
+	stop()
 	assert.Zero(t, sc.subCount(), "a released leader must drop its subscriptions")
 }
 
@@ -620,20 +627,21 @@ func TestScheduler_LeadershipWiresBusSubscriptions(t *testing.T) {
 func TestScheduler_LoserHoldsNoSubscriptions(t *testing.T) {
 	_, nc, _ := testutil.StartTestJetStream(t)
 	svc := NewService(nc, testRegion, "").WithDeps(Deps{IAM: newStubIAM()})
-	a := NewScheduler(nc, svc, "holder-a")
-	b := NewScheduler(nc, svc, "holder-b")
+	a := testScheduler(t, nc, svc, "holder-a")
+	b := testScheduler(t, nc, svc, "holder-b")
 
-	require.True(t, a.lease.TryAcquire(t.Context()))
-	require.False(t, b.lease.TryAcquire(t.Context()))
+	_, err := electOnce(t, a)
+	require.NoError(t, err)
+	_, err = b.lease.TryLock(t.Context())
+	require.ErrorAs(t, err, new(*kvsync.HeldError))
 	assert.Zero(t, b.subCount())
 	assert.Equal(t, 4, a.subCount())
 }
 
 func TestScheduler_RunReleasesLeaseOnShutdown(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
+	_, nc, _ := testutil.StartTestJetStream(t)
 	svc := NewService(nc, testRegion, "").WithDeps(Deps{IAM: newStubIAM()})
-	sc := NewScheduler(nc, svc, "holder-a")
-	require.NoError(t, sc.leaseErr)
+	sc := testScheduler(t, nc, svc, "holder-a")
 
 	ctx, cancel := context.WithCancel(t.Context())
 	done := make(chan struct{})
@@ -643,10 +651,7 @@ func TestScheduler_RunReleasesLeaseOnShutdown(t *testing.T) {
 	cancel()
 	<-done
 
-	kv, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
-	_, err = kv.Get(t.Context(), schedulerLeaderKey)
-	require.Error(t, err, "Run returned with the lease key still present")
+	requireLeaseKeyAbsent(t, nc, "Run returned with the lease key still present")
 }
 
 // --- GPU placement dimension (Epic C2) ---

spinifex/domains/ecs/store.go

@@ -4,9 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"time"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
@@ -19,18 +17,11 @@ import (
 // cluster, task-definition, task, service, and container-instance state. It is
 // created lazily on first cluster touch (no daemon-boot pre-creation) so accounts
 // without any ECS clusters never grow a bucket (ecs-v1.md Q2).
-//
-// Shared leader bucket "spinifex-ecs-leader" holds 60s-TTL CAS locks keyed by
-// "{accountID}/{clusterName}" for the per-cluster scheduler leader election
-// (Q3). Created once at daemon boot.
+
 const (
 	KVBucketECSAccountPrefix  = "ecs-account-"
 	KVBucketECSAccountVersion = 1
 	KVBucketECSAccountHistory = 1
-
-	KVBucketECSLeader        = "spinifex-ecs-leader"
-	KVBucketECSLeaderVersion = 1
-	KVBucketECSLeaderTTL     = 60 * time.Second
 )
 
 // Key-path helpers for the per-account bucket. The layout matches ecs-v1.md Q2.
@@ -119,15 +110,15 @@ func CapacityProviderKey(name string) string {
 	return CapacityProvidersPrefix() + name
 }
 
-// Store is the per-daemon ECS KV handle. Per-account and leader buckets are
-// accessed via the package-level factories below.
+// Store is the per-daemon ECS KV handle. Per-account buckets are accessed via
+// the package-level factories below.
 type Store struct {
 	nc *nats.Conn
 }
 
 // NewStore constructs a Store bound to the supplied NATS connection. It does not
 // touch JetStream — per-account buckets are created lazily by
-// GetOrCreateAccountBucket and the leader bucket by InitLeaderBucket.
+// GetOrCreateAccountBucket.
 func NewStore(nc *nats.Conn) (*Store, error) {
 	if nc == nil {
 		return nil, errors.New("ecs store: nats connection is nil")
@@ -185,13 +176,3 @@ func GetOrCreateAccountBucket(ctx context.Context, js jetstream.JetStream, accou
 	}
 	return kv, nil
 }
-
-// InitLeaderBucket creates (or attaches to) the shared spinifex-ecs-leader bucket
-// used for per-cluster scheduler leader-lease CAS locks.
-func InitLeaderBucket(ctx context.Context, js jetstream.JetStream) (jetstream.KeyValue, error) {
-	return kvlease.OpenBucket(ctx, js, kvlease.BucketConfig{
-		Name:    KVBucketECSLeader,
-		TTL:     KVBucketECSLeaderTTL,
-		Version: KVBucketECSLeaderVersion,
-	})
-}

spinifex/domains/ecs/store_test.go

@@ -1,13 +1,9 @@
 package ecs
 
 import (
-	"encoding/json"
-	"strings"
 	"testing"
-	"time"
 
 	"github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -46,71 +42,6 @@ func TestGetOrCreateAccountBucket_Idempotent(t *testing.T) {
 	assert.Equal(t, kv1.Bucket(), kv2.Bucket())
 }
 
-func TestInitLeaderBucket_Idempotent(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	js := testutil.NewJetStream(t, nc)
-
-	kv1, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
-	require.NotNil(t, kv1)
-	assert.Equal(t, KVBucketECSLeader, kv1.Bucket())
-
-	kv2, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
-	require.NotNil(t, kv2)
-	assert.Equal(t, KVBucketECSLeader, kv2.Bucket())
-}
-
-// The scheduler calls InitLeaderBucket on every tick. Creating first and falling
-// back on already-exists reaches the same handle, so idempotency alone cannot
-// tell the two orders apart — only the API traffic can.
-//
-// On a multi-node cluster the stored bucket is replicated while this call asks
-// for the default, and JetStream answers the mismatch with an error rather than
-// a no-op: 10058, once per tick, forever. That is invisible from the caller and
-// showed up only as a 16% JetStream API error rate on the meta leader.
-func TestInitLeaderBucket_AttachesWithoutCreating(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
-
-	// Watch from before the first call. The server queues the API advisory after
-	// the reply the caller waits on, so subscribing once the bucket exists can
-	// still catch the create that first call legitimately made.
-	creates := make(chan string, 8)
-	sub, err := nc.Subscribe("$JS.EVENT.ADVISORY.API", func(m *nats.Msg) {
-		var adv struct {
-			Subject string `json:"subject"`
-		}
-		if json.Unmarshal(m.Data, &adv) == nil &&
-			strings.HasPrefix(adv.Subject, "$JS.API.STREAM.CREATE.") {
-			creates <- adv.Subject
-		}
-	})
-	require.NoError(t, err)
-	defer func() { _ = sub.Unsubscribe() }()
-	require.NoError(t, nc.Flush())
-
-	_, err = InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
-
-	// Take the first, legitimate create off the channel before the call under
-	// test, so anything left there afterwards can only have come from it.
-	select {
-	case <-creates:
-	case <-time.After(5 * time.Second):
-		t.Fatal("the first InitLeaderBucket must create the bucket")
-	}
-
-	_, err = InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
-	require.NoError(t, nc.Flush())
-
-	select {
-	case subject := <-creates:
-		t.Fatalf("InitLeaderBucket issued %s on an existing bucket; attach before create", subject)
-	case <-time.After(250 * time.Millisecond):
-	}
-}
-
 // Key-path helpers must produce the ecs-v1.md Q2 layout exactly: prefixes are
 // what the List* enumerations watch, so a drift here silently breaks listing.
 func TestKeyPaths(t *testing.T) {

spinifex/domains/eks/awsapi/gateway_wire_test.go

@@ -14,6 +14,7 @@ import (
 	ekscluster "github.com/mulgadc/spinifex/spinifex/domains/eks/cluster"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -55,7 +56,7 @@ func TestEKSRequest_InternalAddonsWireBody(t *testing.T) {
 		ControlPlaneNodes: []ekscluster.ControlPlaneNode{{InstanceID: cpInstanceID}},
 	}))
 
-	svc, err := ekscluster.NewEKSServiceImpl(ekscluster.EKSServiceDeps{NATSConn: nc, Region: authzRegion})
+	svc, err := ekscluster.NewEKSServiceImpl(ekscluster.EKSServiceDeps{NATSConn: nc, Region: authzRegion, Leases: ekscluster.NewClusterLeases(kvstore.OpenSyncBucket(js), "test")})
 	require.NoError(t, err)
 	sub, err := nc.Subscribe("eks.ListStagedAddonManifests", func(m *nats.Msg) {
 		acct := natsmsg.AccountIDFromMsg(m)
@@ -112,7 +113,7 @@ func TestEKSRequest_InternalWorkersWireBody(t *testing.T) {
 		require.NoError(t, ekscluster.PutNodegroupRecord(t.Context(), kv, rec))
 	}
 
-	svc, err := ekscluster.NewEKSServiceImpl(ekscluster.EKSServiceDeps{NATSConn: nc, Region: authzRegion})
+	svc, err := ekscluster.NewEKSServiceImpl(ekscluster.EKSServiceDeps{NATSConn: nc, Region: authzRegion, Leases: ekscluster.NewClusterLeases(kvstore.OpenSyncBucket(js), "test")})
 	require.NoError(t, err)
 	sub, err := nc.Subscribe("eks.ListManagedWorkers", func(m *nats.Msg) {
 		acct := natsmsg.AccountIDFromMsg(m)

spinifex/domains/eks/cluster/cluster_reconciler.go

@@ -16,8 +16,8 @@ import (
 	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/reconciler"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
 
 	"github.com/nats-io/nats.go"
@@ -25,8 +25,7 @@ import (
 )
 
 const (
-	defaultReconcileLeaseRefresh = 20 * time.Second
-	defaultReconcileInterval     = 30 * time.Second
+	defaultReconcileInterval = 30 * time.Second
 	// reconcileResync is the drift backstop for a loop whose signals now arrive as
 	// watch updates and state reports. It also bounds every deadline below it, so
 	// the 15-minute create timeout is served by the resync rather than by a timer.
@@ -123,6 +122,13 @@ type natsSubscriber interface {
 // ErrReconcilerLeaseLost is returned by Run when the leader lease is lost mid-loop.
 var ErrReconcilerLeaseLost = errors.New("eks: reconciler lease lost")
 
+// closedLost stands in for the Lost channel of a reconciler that never acquired.
+var closedLost = func() <-chan struct{} {
+	ch := make(chan struct{})
+	close(ch)
+	return ch
+}()
+
 // ErrReconcilerClusterDeleting is returned by Run when meta.Status is DELETING;
 // DeleteCluster owns teardown so the reconciler exits.
 var ErrReconcilerClusterDeleting = errors.New("eks: cluster deleting")
@@ -138,15 +144,12 @@ type HTTPDoer interface {
 // ClusterReconciler drives one cluster's CREATING → ACTIVE → DELETING lifecycle
 // under a leader lease. One instance per (accountID, clusterName).
 type ClusterReconciler struct {
-	leaderKV    jetstream.KeyValue
 	acctKV      jetstream.KeyValue
 	accountID   string
 	clusterName string
-	holderID    string
 	healthURL   string
 
-	lease         *kvlease.Lease
-	leaseRefresh  time.Duration
+	lease         *kvsync.Lease
 	interval      time.Duration
 	healthTimeout time.Duration
 	createTimeout time.Duration
@@ -166,6 +169,9 @@ type ClusterReconciler struct {
 	// terminal records why the loop is ending — a terminal cluster status or a
 	// lost lease — and stopRun cancels it. reconciler.Run has no exit condition
 	// of its own, so ending it is cancelling it, and this is what Run reports.
+	// lost is the Lost channel of the acquisition AcquireLease took.
+	lost <-chan struct{}
+
 	terminalMu sync.Mutex
 	terminal   error
 	stopRun    context.CancelFunc
@@ -226,11 +232,6 @@ func WithReconcileInterval(d time.Duration) ReconcilerOption {
 	return func(r *ClusterReconciler) { r.interval = d }
 }
 
-// WithLeaseRefresh overrides the leader-lease refresh period (should be < KVBucketEKSLeaderTTL).
-func WithLeaseRefresh(d time.Duration) ReconcilerOption {
-	return func(r *ClusterReconciler) { r.leaseRefresh = d }
-}
-
 // WithHealthzTimeout overrides the per-probe HTTP timeout.
 func WithHealthzTimeout(d time.Duration) ReconcilerOption {
 	return func(r *ClusterReconciler) { r.healthTimeout = d }
@@ -335,12 +336,12 @@ func WithEtcdResetRecovery(grace, backoff time.Duration, maxAttempts int) Reconc
 	}
 }
 
-// NewClusterReconciler constructs a ClusterReconciler. healthURL is the
-// fully-qualified probe target (e.g. https://nlb-dns:443/healthz) or empty
-// to skip /healthz probing (CREATING transitions on bootstrap state alone).
-func NewClusterReconciler(leaderKV, acctKV jetstream.KeyValue, accountID, clusterName, holderID, healthURL string, opts ...ReconcilerOption) (*ClusterReconciler, error) {
-	if leaderKV == nil {
-		return nil, errors.New("eks: NewClusterReconciler nil leaderKV")
+// NewClusterReconciler constructs a ClusterReconciler that leads while it holds
+// lease, the cluster's lease from ClusterLeases. healthURL is the probe target
+// (e.g. https://nlb-dns:443/healthz) or empty to skip /healthz probing.
+func NewClusterReconciler(lease *kvsync.Lease, acctKV jetstream.KeyValue, accountID, clusterName, healthURL string, opts ...ReconcilerOption) (*ClusterReconciler, error) {
+	if lease == nil {
+		return nil, errors.New("eks: NewClusterReconciler nil lease")
 	}
 	if acctKV == nil {
 		return nil, errors.New("eks: NewClusterReconciler nil acctKV")
@@ -351,17 +352,12 @@ func NewClusterReconciler(leaderKV, acctKV jetstream.KeyValue, accountID, cluste
 	if clusterName == "" {
 		return nil, errors.New("eks: NewClusterReconciler empty clusterName")
 	}
-	if holderID == "" {
-		return nil, errors.New("eks: NewClusterReconciler empty holderID")
-	}
 	r := &ClusterReconciler{
-		leaderKV:        leaderKV,
 		acctKV:          acctKV,
 		accountID:       accountID,
 		clusterName:     clusterName,
-		holderID:        holderID,
 		healthURL:       healthURL,
-		leaseRefresh:    defaultReconcileLeaseRefresh,
+		lease:           lease,
 		interval:        defaultReconcileInterval,
 		healthTimeout:   defaultHealthzTimeout,
 		createTimeout:   defaultCreateTimeout,
@@ -390,42 +386,23 @@ func NewClusterReconciler(leaderKV, acctKV jetstream.KeyValue, accountID, cluste
 	for _, o := range opts {
 		o(r)
 	}
-	lease, err := kvlease.New(kvlease.Config{
-		Name:   "eks/cluster-reconciler",
-		Bucket: kvlease.StaticBucket(leaderKV),
-		Key:    reconcilerLeaderKey(accountID, clusterName),
-		Holder: holderID,
-		Attrs:  []any{"account", accountID, "cluster", clusterName},
-		TTL:    KVBucketEKSLeaderTTL,
-		Renew:  r.leaseRefresh,
-	})
-	if err != nil {
-		return nil, fmt.Errorf("cluster reconciler lease: %w", err)
-	}
-	r.lease = lease
 	return r, nil
 }
 
-func reconcilerLeaderKey(accountID, clusterName string) string {
-	return accountID + "/" + clusterName
-}
-
-// AcquireLease attempts a CAS Create on the leader bucket. Returns
-// (release, true) on success; (nil, false) otherwise. The bucket TTL reaps
-// stale leases when release does not run.
+// AcquireLease takes the cluster's lease. Returns (release, true) on success;
+// (nil, false) when another holder, or another run in this process, has it.
 func (r *ClusterReconciler) AcquireLease(ctx context.Context) (func(), bool) {
-	if !r.lease.TryAcquire(ctx) {
-		return nil, false
+	release, lost, ok := tryLease(ctx, r.lease)
+	if ok {
+		r.lost = lost
 	}
-	return func() { r.lease.Release(ctx) }, true
+	return release, ok
 }
 
 // Run drives reconcile until ctx is cancelled, the lease is lost, or the cluster
-// reaches a terminal state. Caller must AcquireLease first.
+// reaches a terminal state. Caller must AcquireLease first; without it Run
+// returns ErrReconcilerLeaseLost at once.
 func (r *ClusterReconciler) Run(ctx context.Context) error {
-	if r.lease == nil {
-		return errors.New("ClusterReconciler: Run called without AcquireLease")
-	}
 	runCtx, stop := context.WithCancel(ctx)
 	defer stop()
 	r.stopRun = stop
@@ -463,11 +440,16 @@ func (r *ClusterReconciler) Run(ctx context.Context) error {
 	}
 
 	// The lease turning over is not a KV write this loop watches, so losing it has
-	// to end the loop from outside it.
+	// to end the loop from outside it. The channel is the acquisition's own, taken
+	// with its token, so a later acquisition's cannot stand in for it.
+	lost := r.lost
+	if lost == nil {
+		lost = closedLost
+	}
 	go func() {
 		select {
 		case <-runCtx.Done():
-		case <-r.lease.Lost():
+		case <-lost:
 			r.endRun(ErrReconcilerLeaseLost)
 		}
 	}()

spinifex/domains/eks/cluster/cluster_reconciler_state_test.go

@@ -21,14 +21,12 @@ func newStateReconcilerHarness(t *testing.T, opts ...ReconcilerOption) (*Cluster
 	t.Helper()
 	_, nc, _ := testutil.StartTestJetStream(t)
 	js := testutil.NewJetStream(t, nc)
-	leaderKV, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
 	acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
 	require.NoError(t, err)
 	require.NoError(t, PutClusterMeta(t.Context(), acctKV, sampleClusterMeta("alpha")))
 
 	base := []ReconcilerOption{WithStateSource(nc, StateSubject(testAccountID, "alpha"))}
-	r, err := NewClusterReconciler(leaderKV, acctKV, testAccountID, "alpha", "holder-1", "", append(base, opts...)...)
+	r, err := NewClusterReconciler(otherNodeLease(js, "holder-1"), acctKV, testAccountID, "alpha", "", append(base, opts...)...)
 	require.NoError(t, err)
 	return r, nc, acctKV
 }
@@ -146,7 +144,6 @@ func TestClusterReconciler_StateReportSubscriptionDrivesActive(t *testing.T) {
 	t.Parallel()
 	r, nc, acctKV := newStateReconcilerHarness(t,
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 	freshenClusterCreatedAt(t, acctKV)
 	seedBootstrapState(t, acctKV)

spinifex/domains/eks/cluster/cluster_reconciler_test.go

@@ -12,6 +12,8 @@ import (
 	"time"
 
 	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -54,39 +56,47 @@ func onePass(t *testing.T, r *ClusterReconciler) error {
 	return err
 }
 
-func newReconcilerHarness(t *testing.T, healthURL string, opts ...ReconcilerOption) (*ClusterReconciler, jetstream.KeyValue, jetstream.KeyValue) {
+func newReconcilerHarness(t *testing.T, healthURL string, opts ...ReconcilerOption) (*ClusterReconciler, jetstream.JetStream, jetstream.KeyValue) {
+	t.Helper()
+	return newReconcilerHarnessTTL(t, clusterLeaseTTL, healthURL, opts...)
+}
+
+// newReconcilerHarnessTTL is newReconcilerHarness with a lease of the given ttl,
+// for a test that has to see a renewal.
+func newReconcilerHarnessTTL(t *testing.T, ttl time.Duration, healthURL string, opts ...ReconcilerOption) (*ClusterReconciler, jetstream.JetStream, jetstream.KeyValue) {
 	t.Helper()
 	_, nc, _ := testutil.StartTestJetStream(t)
 	js := testutil.NewJetStream(t, nc)
-	leaderKV, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
 	acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
 	require.NoError(t, err)
 	require.NoError(t, PutClusterMeta(t.Context(), acctKV, sampleClusterMeta("alpha")))
 
-	r, err := NewClusterReconciler(leaderKV, acctKV, testAccountID, "alpha", "holder-1", healthURL, opts...)
+	leases := kvsync.NewLeaseGroup(kvstore.OpenSyncBucket(js), KeyPrefixCluster, "holder-1", ttl)
+	r, err := NewClusterReconciler(leases.Get(clusterLeaseName(testAccountID, "alpha")), acctKV, testAccountID, "alpha", healthURL, opts...)
 	require.NoError(t, err)
-	return r, leaderKV, acctKV
+	return r, js, acctKV
+}
+
+// otherNodeLease is the cluster's reconciler lease as another node sees it.
+func otherNodeLease(js jetstream.JetStream, holder string) *kvsync.Lease {
+	return NewClusterLeases(kvstore.OpenSyncBucket(js), holder).Get(clusterLeaseName(testAccountID, "alpha"))
 }
 
 func TestNewClusterReconciler_EmptyInputsRejected(t *testing.T) {
 	t.Parallel()
 	_, nc, _ := testutil.StartTestJetStream(t)
 	js := testutil.NewJetStream(t, nc)
-	leaderKV, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
 	acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
 	require.NoError(t, err)
+	lease := otherNodeLease(js, "h")
 
-	_, err = NewClusterReconciler(nil, acctKV, testAccountID, "alpha", "h", "")
+	_, err = NewClusterReconciler(nil, acctKV, testAccountID, "alpha", "")
 	require.Error(t, err)
-	_, err = NewClusterReconciler(leaderKV, nil, testAccountID, "alpha", "h", "")
+	_, err = NewClusterReconciler(lease, nil, testAccountID, "alpha", "")
 	require.Error(t, err)
-	_, err = NewClusterReconciler(leaderKV, acctKV, "", "alpha", "h", "")
+	_, err = NewClusterReconciler(lease, acctKV, "", "alpha", "")
 	require.Error(t, err)
-	_, err = NewClusterReconciler(leaderKV, acctKV, testAccountID, "", "h", "")
-	require.Error(t, err)
-	_, err = NewClusterReconciler(leaderKV, acctKV, testAccountID, "alpha", "", "")
+	_, err = NewClusterReconciler(lease, acctKV, testAccountID, "", "")
 	require.Error(t, err)
 }
 
@@ -99,21 +109,21 @@ func TestClusterReconciler_AcquireLeaseFirstHolderWins(t *testing.T) {
 	require.NotNil(t, release)
 	defer release()
 
-	// Re-acquiring from the same holder succeeds: the lease is already held.
+	// A second run in the same process is refused while the first holds it.
 	release2, ok2 := r.AcquireLease(t.Context())
-	assert.True(t, ok2)
-	assert.NotNil(t, release2)
+	assert.False(t, ok2)
+	assert.Nil(t, release2)
 }
 
 func TestClusterReconciler_AcquireLeaseSecondHolderLoses(t *testing.T) {
 	t.Parallel()
-	r1, leaderKV, acctKV := newReconcilerHarness(t, "")
+	r1, js, acctKV := newReconcilerHarness(t, "")
 
 	release, ok := r1.AcquireLease(t.Context())
 	require.True(t, ok)
 	defer release()
 
-	r2, err := NewClusterReconciler(leaderKV, acctKV, testAccountID, "alpha", "holder-2", "")
+	r2, err := NewClusterReconciler(otherNodeLease(js, "holder-2"), acctKV, testAccountID, "alpha", "")
 	require.NoError(t, err)
 	release2, ok2 := r2.AcquireLease(t.Context())
 	assert.False(t, ok2)
@@ -154,7 +164,6 @@ func TestClusterReconciler_CreatingTransitionsToActiveOnReadyAndHealthz(t *testi
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 	release, ok := r.AcquireLease(t.Context())
 	require.True(t, ok)
@@ -193,7 +202,6 @@ func TestClusterReconciler_CreatingStaysWhenBootstrapMissing(t *testing.T) {
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 	release, ok := r.AcquireLease(t.Context())
 	require.True(t, ok)
@@ -228,7 +236,6 @@ func TestClusterReconciler_CreatingStaysWhenJWKSUnverified(t *testing.T) {
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 	release, ok := r.AcquireLease(t.Context())
 	require.True(t, ok)
@@ -262,7 +269,6 @@ func TestClusterReconciler_CreatingStaysWhenHealthzFails(t *testing.T) {
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 	release, ok := r.AcquireLease(t.Context())
 	require.True(t, ok)
@@ -287,7 +293,6 @@ func TestClusterReconciler_CreatingTimesOutToFailed(t *testing.T) {
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 		WithCreateTimeout(1*time.Millisecond),
 	)
 	release, ok := r.AcquireLease(t.Context())
@@ -320,7 +325,6 @@ func TestClusterReconciler_CreatingWithinDeadlineStaysCreating(t *testing.T) {
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 		WithCreateTimeout(time.Hour),
 	)
 	release, ok := r.AcquireLease(t.Context())
@@ -345,7 +349,6 @@ func TestClusterReconciler_DeletingExitsLoop(t *testing.T) {
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 	release, ok := r.AcquireLease(t.Context())
 	require.True(t, ok)
@@ -363,11 +366,10 @@ func TestClusterReconciler_DeletingExitsLoop(t *testing.T) {
 func TestClusterReconciler_LostLeaseExitsLoop(t *testing.T) {
 	t.Parallel()
 	stub := &stubHTTPDoer{status: http.StatusOK}
-	r, leaderKV, acctKV := newReconcilerHarness(t,
+	r, js, acctKV := newReconcilerHarnessTTL(t, 300*time.Millisecond,
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Second),
-		WithLeaseRefresh(20*time.Millisecond),
 	)
 	release, ok := r.AcquireLease(t.Context())
 	require.True(t, ok)
@@ -381,16 +383,17 @@ func TestClusterReconciler_LostLeaseExitsLoop(t *testing.T) {
 	runErr := make(chan error, 1)
 	go func() { runErr <- r.Run(ctx) }()
 
-	// Steal the lease in another holder's name. The renewal loses its CAS at the
-	// stale revision, which closes Lost and exits Run.
-	time.Sleep(40 * time.Millisecond)
-	_, err := leaderKV.Put(t.Context(), reconcilerLeaderKey(testAccountID, "alpha"), []byte("holder-2"))
+	// Overwrite the key in another holder's name. The next renewal loses its
+	// CAS at the stale revision, which closes Lost and exits Run.
+	syncKV, err := kvstore.OpenSyncBucket(js).KV(t.Context())
+	require.NoError(t, err)
+	_, err = syncKV.Put(t.Context(), KeyPrefixCluster+clusterLeaseName(testAccountID, "alpha"), []byte(`{"holder":"holder-2","generation":0}`))
 	require.NoError(t, err)
 
 	select {
 	case err := <-runErr:
 		assert.ErrorIs(t, err, ErrReconcilerLeaseLost)
-	case <-time.After(2 * time.Second):
+	case <-time.After(5 * time.Second):
 		t.Fatal("Run did not exit after lease loss")
 	}
 }
@@ -402,7 +405,6 @@ func TestClusterReconciler_ActiveProbesAndWarnsOnHealthzFail(t *testing.T) {
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 	release, ok := r.AcquireLease(t.Context())
 	require.True(t, ok)
@@ -438,7 +440,6 @@ func TestClusterReconciler_ActiveHealthRecoversClearsIssue(t *testing.T) {
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 	release, ok := r.AcquireLease(t.Context())
 	require.True(t, ok)
@@ -468,9 +469,111 @@ func TestClusterReconciler_ProbeHealthzEmptyURLNoop(t *testing.T) {
 		"",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 
 	require.NoError(t, r.probeHealthz(context.Background()))
 	assert.EqualValues(t, 0, stub.calls.Load(), "no HTTP call when URL empty")
 }
+
+// putForeignHolder writes the cluster's lease key in another holder's name and
+// never renews it, standing in for a node that died holding it.
+func putForeignHolder(t *testing.T, js jetstream.JetStream) {
+	t.Helper()
+	syncKV, err := kvstore.OpenSyncBucket(js).KV(t.Context())
+	require.NoError(t, err)
+	_, err = syncKV.Put(t.Context(), KeyPrefixCluster+clusterLeaseName(testAccountID, "alpha"), []byte(`{"holder":"holder-2","generation":0}`))
+	require.NoError(t, err)
+}
+
+func requireStillRunning(t *testing.T, finished <-chan struct{}) {
+	t.Helper()
+	select {
+	case <-finished:
+		t.Fatal("the reconciler goroutine exited while its cluster still wants one")
+	default:
+	}
+}
+
+// A dead node's key is not cleared by the store, so the reconciler has to keep
+// asking until it has sat a TTL; one attempt would leave the cluster unreconciled.
+func TestRunClusterReconciler_TakesOverADeadHoldersKey(t *testing.T) {
+	t.Parallel()
+	r, js, acctKV := newReconcilerHarnessTTL(t, 300*time.Millisecond, "https://nlb.example/healthz",
+		WithHTTPClient(&stubHTTPDoer{status: http.StatusOK}), WithReconcileInterval(10*time.Second))
+	freshenClusterCreatedAt(t, acctKV)
+	seedBootstrapState(t, acctKV)
+	putForeignHolder(t, js)
+
+	release, finished, err := RunClusterReconciler(t.Context(), r.lease, acctKV, testAccountID, "alpha", "https://nlb.example/healthz",
+		WithHTTPClient(&stubHTTPDoer{status: http.StatusOK}), WithReconcileInterval(10*time.Second))
+	require.NoError(t, err, "a held lease is contended for, not an error")
+	defer release()
+
+	require.Eventually(t, r.lease.Held, 5*time.Second, 10*time.Millisecond, "the dead holder's key must be taken over")
+	requireStillRunning(t, finished)
+}
+
+// Losing the lease ends a Run, not the reconciler: it contends again and
+// resumes once it holds the lease once more.
+func TestRunClusterReconciler_ResumesAfterALostLease(t *testing.T) {
+	t.Parallel()
+	r, js, acctKV := newReconcilerHarnessTTL(t, 300*time.Millisecond, "https://nlb.example/healthz")
+	freshenClusterCreatedAt(t, acctKV)
+	seedBootstrapState(t, acctKV)
+	release, finished, err := RunClusterReconciler(t.Context(), r.lease, acctKV, testAccountID, "alpha", "https://nlb.example/healthz",
+		WithHTTPClient(&stubHTTPDoer{status: http.StatusOK}), WithReconcileInterval(10*time.Second))
+	require.NoError(t, err)
+	defer release()
+	require.Eventually(t, r.lease.Held, 5*time.Second, 10*time.Millisecond)
+
+	putForeignHolder(t, js)
+	require.Eventually(t, func() bool { return !r.lease.Held() }, 5*time.Second, 10*time.Millisecond, "the overwrite must cost the lease")
+	require.Eventually(t, r.lease.Held, 5*time.Second, 10*time.Millisecond, "the reconciler must win the lease back")
+	requireStillRunning(t, finished)
+}
+
+// A cluster that has left CREATING and ACTIVE wants no reconciler, so one that
+// cannot get the lease stops asking.
+func TestRunClusterReconciler_StopsContendingForADeletingCluster(t *testing.T) {
+	t.Parallel()
+	r, js, acctKV := newReconcilerHarnessTTL(t, 300*time.Millisecond, "https://nlb.example/healthz")
+	meta, err := GetClusterMeta(t.Context(), acctKV, "alpha")
+	require.NoError(t, err)
+	meta.Status = ClusterStatusDeleting
+	require.NoError(t, PutClusterMeta(t.Context(), acctKV, meta))
+	putForeignHolder(t, js)
+
+	release, finished, err := RunClusterReconciler(t.Context(), r.lease, acctKV, testAccountID, "alpha", "")
+	require.NoError(t, err)
+	defer release()
+	select {
+	case <-finished:
+	case <-time.After(5 * time.Second):
+		t.Fatal("the reconciler kept contending for a DELETING cluster")
+	}
+	assert.False(t, r.lease.Held())
+}
+
+// Run watches the Lost channel of the acquisition AcquireLease took. A later
+// acquisition of the same lease must not keep a Run alive whose own was lost.
+func TestClusterReconciler_RunWatchesItsOwnAcquisition(t *testing.T) {
+	t.Parallel()
+	r, js, acctKV := newReconcilerHarnessTTL(t, 300*time.Millisecond, "https://nlb.example/healthz",
+		WithHTTPClient(&stubHTTPDoer{status: http.StatusOK}), WithReconcileInterval(10*time.Second))
+	_, ok := r.AcquireLease(t.Context())
+	require.True(t, ok)
+
+	putForeignHolder(t, js)
+	require.Eventually(t, func() bool { return !r.lease.Held() }, 5*time.Second, 10*time.Millisecond)
+	syncKV, err := kvstore.OpenSyncBucket(js).KV(t.Context())
+	require.NoError(t, err)
+	require.NoError(t, syncKV.Delete(t.Context(), KeyPrefixCluster+clusterLeaseName(testAccountID, "alpha")))
+	tok, err := r.lease.TryLock(t.Context())
+	require.NoError(t, err, "a later acquisition of the same lease")
+	defer r.lease.Unlock(t.Context(), tok)
+
+	seedBootstrapState(t, acctKV)
+	ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+	defer cancel()
+	assert.ErrorIs(t, r.Run(ctx), ErrReconcilerLeaseLost, "the acquisition this Run was given is gone")
+}

spinifex/domains/eks/cluster/cluster_teardown.go

@@ -7,7 +7,6 @@ import (
 	"log/slog"
 	"time"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	telemetry "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
 	"github.com/nats-io/nats.go/jetstream"
@@ -22,12 +21,6 @@ var teardownConcurrency = 4
 // writer added refs to the record while it was tearing down.
 const maxTeardownPasses = 5
 
-// Teardown lease timing. TTL must match the leader bucket's TTL; vars for tests.
-var (
-	teardownLeaseTTL   = KVBucketEKSLeaderTTL
-	teardownLeaseRenew = defaultReconcileLeaseRefresh
-)
-
 var (
 	errTeardownNotRecorded = errors.New("eks: cluster is not DELETING, teardown refused")
 	errTeardownLeaseLost   = errors.New("eks: teardown lease lost")
@@ -35,45 +28,17 @@ var (
 	errLaunchInFlight      = errors.New("eks: cluster create launch still running")
 )
 
-// teardownLeaderKey is the per-cluster cross-node gate for a DELETING
-// teardown. It is distinct from reconcilerLeaderKey so a delete never contends
-// with the still-live reconciler, which holds that lease until it reads DELETING.
-func teardownLeaderKey(accountID, clusterName string) string {
-	return reconcilerLeaderKey(accountID, clusterName) + "/teardown"
-}
-
 // teardownLease is a held teardown lease: lost closes if renewal loses the key.
 type teardownLease struct {
 	release func()
 	lost    <-chan struct{}
 }
 
-// acquireTeardownLease takes the per-cluster teardown lease, renewing it on
-// ctx. The lease re-adopts a key its own holder already has, so it serialises
-// nodes, not goroutines; startTeardown's in-process gate covers the latter. A
-// nil leaderKV (single-node/test) skips gating.
+// acquireTeardownLease takes the per-cluster teardown lease, renewing it until
+// released. It refuses when another node holds it or another goroutine here does.
 func (s *EKSServiceImpl) acquireTeardownLease(ctx context.Context, accountID, clusterName string) (teardownLease, bool) {
-	if s.leaderKV == nil {
-		return teardownLease{release: func() {}}, true
-	}
-	key := teardownLeaderKey(accountID, clusterName)
-	lease, err := kvlease.New(kvlease.Config{
-		Name:   "eks/teardown",
-		Bucket: kvlease.StaticBucket(s.leaderKV),
-		Key:    key,
-		Holder: s.deps.HolderID,
-		Attrs:  []any{"account", accountID, "cluster", clusterName},
-		TTL:    teardownLeaseTTL,
-		Renew:  teardownLeaseRenew,
-	})
-	if err != nil {
-		slog.ErrorContext(ctx, "eks: teardown lease config invalid", "key", key, "err", err)
-		return teardownLease{}, false
-	}
-	if !lease.TryAcquire(ctx) {
-		return teardownLease{}, false
-	}
-	return teardownLease{release: func() { lease.Release(ctx) }, lost: lease.Lost()}, true
+	release, lost, ok := tryLease(ctx, s.leases.Get(teardownLeaseName(accountID, clusterName)))
+	return teardownLease{release: release, lost: lost}, ok
 }
 
 // teardownPassTimeout bounds one converger run, so a step that hangs cannot
@@ -81,11 +46,11 @@ func (s *EKSServiceImpl) acquireTeardownLease(ctx context.Context, accountID, cl
 var teardownPassTimeout = 15 * time.Minute
 
 // holdTeardown claims the right to tear a cluster down on this node: the
-// in-process gate, which the lease cannot provide because it re-adopts a key
-// its own node holds, then the cross-node lease. It refuses, without waiting,
-// when either is held, or, if capped, when the node is at teardownConcurrency.
-func (s *EKSServiceImpl) holdTeardown(accountID, name string, capped bool) (release func(), lost <-chan struct{}, ok bool) {
-	key := teardownLeaderKey(accountID, name)
+// in-process gate, which also bounds this node's teardowns, then the cross-node
+// lease. It refuses, without waiting, when either is held, or, if capped, when
+// the node is at teardownConcurrency.
+func (s *EKSServiceImpl) holdTeardown(accountID, name string, capped bool) (release func(retire bool), lost <-chan struct{}, ok bool) {
+	key := teardownLeaseName(accountID, name)
 	s.teardownMu.Lock()
 	if s.teardownRunning == nil {
 		s.teardownRunning = map[string]struct{}{}
@@ -110,7 +75,15 @@ func (s *EKSServiceImpl) holdTeardown(accountID, name string, capped bool) (rele
 		leave()
 		return nil, nil, false
 	}
-	return func() { lease.release(); leave() }, lease.lost, true
+	// Retiring drops the teardown lease before the gate opens: every Get of it
+	// is under the gate, so no later teardown of a recreated name can hold it.
+	return func(retire bool) {
+		lease.release()
+		if retire {
+			s.leases.Forget(key)
+		}
+		leave()
+	}, lease.lost, true
 }
 
 // startTeardown hands one DELETING cluster to a background converger and
@@ -122,7 +95,8 @@ func (s *EKSServiceImpl) startTeardown(accountID, name string) bool {
 		return false
 	}
 	s.teardownWG.Go(func() {
-		defer release()
+		deleted := false
+		defer func() { release(deleted) }()
 		ctx, cancel := context.WithTimeout(s.bgCtx, teardownPassTimeout)
 		defer cancel()
 		started := time.Now()
@@ -131,6 +105,7 @@ func (s *EKSServiceImpl) startTeardown(accountID, name string) bool {
 				"cluster", name, "account", accountID, "err", err)
 			return
 		}
+		deleted = true
 		slog.Info("eks: cluster teardown complete", "cluster", name, "account", accountID,
 			"duration_ms", telemetry.Millis(time.Since(started)))
 	})
@@ -211,6 +186,9 @@ func (s *EKSServiceImpl) convergeClusterDeletion(ctx context.Context, accountID,
 			return terr
 		}
 
+		// Forgotten while the record still blocks a same-name create, and after
+		// teardownClusterInfra stopped this node's reconciler.
+		s.leases.Forget(clusterLeaseName(accountID, name))
 		done, err := s.removeClusterRecord(ctx, acctKV, name, meta)
 		if err != nil {
 			if !errors.Is(err, errNodegroupsAttached) && !errors.Is(err, errTeardownNotRecorded) {

spinifex/domains/eks/cluster/delete_cluster_async_test.go

@@ -12,6 +12,8 @@ import (
 	"github.com/aws/aws-sdk-go/service/eks"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -132,22 +134,18 @@ func TestS8_EKSSecondDeleteAnswersFromTheRecordWithoutASecondTeardown(t *testing
 	assert.Len(t, f.inst.terminated(), 1, "the repeat request must start no teardown of its own")
 }
 
-// TestS8_EKSTeardownLeaseOutlivesTheRequest: the teardown lease renews on the
-// service's context, so cancelling the request context mid-teardown does not
-// let the lease expire and a second node start a duplicate teardown.
+// TestS8_EKSTeardownLeaseOutlivesTheRequest: the teardown lease renews on its
+// own, so cancelling the request context mid-teardown does not let the lease
+// lapse and a second node start a duplicate teardown.
 func TestS8_EKSTeardownLeaseOutlivesTheRequest(t *testing.T) {
 	f := newDeleteClusterFixture(t, "alpha")
-	_, nc, _ := testutil.StartTestJetStream(t)
-	js := testutil.NewJetStream(t, nc)
-	// A short TTL with frequent renewals keeps the wait brief while leaving
-	// several renewal chances inside each TTL window.
+	// A short TTL keeps the wait brief while leaving several renewal chances
+	// inside each TTL window.
 	const ttl = 400 * time.Millisecond
-	short, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: "eks-teardown-short-ttl", TTL: ttl})
-	require.NoError(t, err)
-	f.svc.leaderKV = short
-	origTTL, origRenew := teardownLeaseTTL, teardownLeaseRenew
-	teardownLeaseTTL, teardownLeaseRenew = ttl, 50*time.Millisecond
-	t.Cleanup(func() { teardownLeaseTTL, teardownLeaseRenew = origTTL, origRenew })
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+	f.svc.leases = kvsync.NewLeaseGroup(kvstore.OpenSyncBucket(js), KeyPrefixCluster, "node-1", ttl)
+	other := kvsync.NewLeaseGroup(kvstore.OpenSyncBucket(js), KeyPrefixCluster, "node-2", ttl).
+		Get(teardownLeaseName(testAccountID, "alpha"))
 
 	entered, release := blockTerminate(t, f.inst)
 	ctx, cancel := context.WithCancel(context.Background())
@@ -155,19 +153,22 @@ func TestS8_EKSTeardownLeaseOutlivesTheRequest(t *testing.T) {
 	waitEntered(t, entered)
 	cancel()
 
+	// The other node first sees the key, then sees it again past a full TTL: it
+	// takes over only if no renewal moved the key in between.
+	_, err := other.TryLock(t.Context())
+	require.ErrorAs(t, err, new(*kvsync.HeldError))
 	time.Sleep(ttl * 5 / 2)
-	_, err = short.Create(t.Context(), teardownLeaderKey(testAccountID, "alpha"), []byte("node-2"))
-	require.ErrorIs(t, err, jetstream.ErrKeyExists,
+	_, err = other.TryLock(t.Context())
+	require.ErrorAs(t, err, new(*kvsync.HeldError),
 		"the running teardown must still hold its lease after the request context is gone")
 
 	release()
 	f.svc.WaitTeardowns()
 }
 
-// TestS8_EKSKickAndSameNodeSweepRunOneTeardown: the teardown lease re-adopts a
-// key its own node holds, so only the in-process gate stops a same-node reaper
-// sweep from starting a second teardown beside the request's. Each member must
-// be terminated exactly once.
+// TestS8_EKSKickAndSameNodeSweepRunOneTeardown: a same-node reaper sweep must
+// not start a second teardown beside the request's. Each member must be
+// terminated exactly once.
 func TestS8_EKSKickAndSameNodeSweepRunOneTeardown(t *testing.T) {
 	t.Parallel()
 	f := newDeleteClusterFixture(t, "alpha")

spinifex/domains/eks/cluster/delete_cluster_review2_test.go

@@ -180,7 +180,7 @@ func TestEKSBootstrapRunRemovesItsOwnEntryOnExit(t *testing.T) {
 	eventually(t, func() bool {
 		f.svc.bootstrapMu.Lock()
 		defer f.svc.bootstrapMu.Unlock()
-		_, ok := f.svc.bootstraps[reconcilerLeaderKey(testAccountID, "alpha")]
+		_, ok := f.svc.bootstraps[clusterLeaseName(testAccountID, "alpha")]
 		return !ok
 	}, "an exited bootstrap must remove its own entry")
 }

spinifex/domains/eks/cluster/delete_cluster_review_test.go

@@ -252,8 +252,7 @@ func TestS2_EKSFailedClusterReclaimRunsUnderTheTeardownLease(t *testing.T) {
 	require.NoError(t, err)
 	meta.Status = ClusterStatusFailed
 	require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta))
-	_, err = f.svc.leaderKV.Create(t.Context(), teardownLeaderKey(testAccountID, "alpha"), []byte("node-2"))
-	require.NoError(t, err)
+	holdElsewhere(t, f.svc, teardownLeaseName(testAccountID, "alpha"))
 
 	_, err = f.svc.CreateCluster(context.Background(), createInput("alpha"), testAccountID, "")
 	code, _, _ := awserrors.ResolveErrorDetail(err)

spinifex/domains/eks/cluster/delete_cluster_test.go

@@ -15,6 +15,8 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/systeminstance"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
@@ -83,6 +85,7 @@ func newEKSServiceFixture(t *testing.T) *eksServiceFixture {
 		GatewayBaseURL:      "https://gw.local:9999",
 		Region:              "us-east-1",
 		HolderID:            "node-1",
+		Leases:              NewClusterLeases(kvstore.OpenSyncBucket(js), "node-1"),
 		SystemGatewayURL:    "https://gw.local:9999",
 		SystemAccessKey:     "AKIAEXAMPLE",
 		SystemSecretKey:     "s3cr3t-key",
@@ -371,6 +374,33 @@ func TestDeleteCluster_EgressEIPReleaseRealErrorLeavesMeta(t *testing.T) {
 	assert.NotEmpty(t, meta.EgressEIPAllocationID, "alloc ID must remain for retry")
 }
 
+// otherNodeLeases is the cluster lease group as another node holding holder
+// sees it, on its own sync bucket over the same JetStream as svc.
+func otherNodeLeases(t *testing.T, svc *EKSServiceImpl, holder string) *kvsync.LeaseGroup {
+	t.Helper()
+	return NewClusterLeases(kvstore.OpenSyncBucket(testutil.NewJetStream(t, svc.deps.NATSConn)), holder)
+}
+
+// holdElsewhere has another node take the named lease until the test ends.
+func holdElsewhere(t *testing.T, svc *EKSServiceImpl, name string) {
+	t.Helper()
+	lease := otherNodeLeases(t, svc, "node-2").Get(name)
+	tok, err := lease.TryLock(t.Context())
+	require.NoError(t, err)
+	t.Cleanup(func() { lease.Unlock(context.Background(), tok) })
+}
+
+// leaseKeyHeld reports whether any node holds the named lease's key.
+func leaseKeyHeld(t *testing.T, svc *EKSServiceImpl, name string) bool {
+	t.Helper()
+	_, _, err := kvstore.OpenSyncBucket(testutil.NewJetStream(t, svc.deps.NATSConn)).Load(t.Context(), KeyPrefixCluster+name)
+	if errors.Is(err, kvsync.ErrNotFound) {
+		return false
+	}
+	require.NoError(t, err)
+	return true
+}
+
 // TestDeleteClusterLeaseHeldElsewhereLeavesTeardownToTheHolder: when another
 // node holds the teardown lease, DeleteCluster still records DELETING and
 // returns it, but runs no teardown of its own and leaves the record for the
@@ -379,8 +409,7 @@ func TestDeleteClusterLeaseHeldElsewhereLeavesTeardownToTheHolder(t *testing.T)
 	t.Parallel()
 	f := newDeleteClusterFixture(t, "alpha")
 
-	_, err := f.svc.leaderKV.Create(t.Context(), teardownLeaderKey(testAccountID, "alpha"), []byte("node-2"))
-	require.NoError(t, err)
+	holdElsewhere(t, f.svc, teardownLeaseName(testAccountID, "alpha"))
 
 	deleteAndSettle(t, f.svc, "alpha")
 
@@ -399,8 +428,73 @@ func TestDeleteClusterTeardownReleasesItsLease(t *testing.T) {
 
 	deleteAndSettle(t, f.svc, "alpha")
 
-	_, getErr := f.svc.leaderKV.Get(t.Context(), teardownLeaderKey(testAccountID, "alpha"))
-	assert.ErrorIs(t, getErr, jetstream.ErrKeyNotFound, "the teardown lease must be released after the teardown")
+	assert.False(t, leaseKeyHeld(t, f.svc, teardownLeaseName(testAccountID, "alpha")),
+		"the teardown lease must be released after the teardown")
+}
+
+// TestDeleteClusterForgetsTheClusterLeases: a deleted cluster's reconciler and
+// teardown leases leave the group, so a cluster recreated under the same name
+// starts from fresh ones.
+func TestDeleteClusterForgetsTheClusterLeases(t *testing.T) {
+	t.Parallel()
+	f := newDeleteClusterFixture(t, "alpha")
+	names := []string{clusterLeaseName(testAccountID, "alpha"), teardownLeaseName(testAccountID, "alpha")}
+	before := make([]*kvsync.Lease, len(names))
+	for i, name := range names {
+		before[i] = f.svc.leases.Get(name)
+	}
+
+	deleteAndSettle(t, f.svc, "alpha")
+
+	_, err := GetClusterMeta(t.Context(), f.kv, "alpha")
+	require.ErrorIs(t, err, ErrClusterNotFound, "the teardown must have completed")
+	for i, name := range names {
+		assert.NotSame(t, before[i], f.svc.leases.Get(name), "%s must be forgotten", name)
+	}
+}
+
+// TestDeleteClusterCleanupLeavesARecreatedClustersLease: once the record is
+// gone a same-name cluster may lead, and the old teardown's cleanup, still
+// running, must not release the lease it took.
+func TestDeleteClusterCleanupLeavesARecreatedClustersLease(t *testing.T) {
+	t.Parallel()
+	f := newDeleteClusterFixture(t, "alpha")
+	entered, release := blockTerminate(t, f.inst)
+	_, err := f.svc.DeleteCluster(context.Background(), deleteInput("alpha"), testAccountID)
+	require.NoError(t, err)
+	waitEntered(t, entered)
+
+	// Holding the gate's mutex parks the old teardown before its gate opens.
+	f.svc.teardownMu.Lock()
+	release()
+	require.Eventually(t, func() bool {
+		_, err := GetClusterMeta(t.Context(), f.kv, "alpha")
+		return errors.Is(err, ErrClusterNotFound)
+	}, 10*time.Second, 10*time.Millisecond, "the old teardown must remove the record")
+
+	replacement := f.svc.leases.Get(clusterLeaseName(testAccountID, "alpha"))
+	tok, err := replacement.TryLock(t.Context())
+	require.NoError(t, err, "the recreated cluster's reconciler takes the free lease")
+	f.svc.teardownMu.Unlock()
+	f.svc.WaitTeardowns()
+
+	assert.True(t, replacement.Held(), "the old teardown's cleanup released the recreated cluster's lease")
+	replacement.Unlock(t.Context(), tok)
+}
+
+// TestDeleteClusterKeepsTheLeasesOfAnIncompleteTeardown: a teardown that
+// fails leaves the record DELETING for retry, so its leases stay in the group.
+func TestDeleteClusterKeepsTheLeasesOfAnIncompleteTeardown(t *testing.T) {
+	t.Parallel()
+	f := newDeleteClusterFixture(t, "alpha")
+	f.eip.releaseErr = errors.New("AddressLimitExceeded")
+	name := teardownLeaseName(testAccountID, "alpha")
+	before := f.svc.leases.Get(name)
+
+	deleteAndSettle(t, f.svc, "alpha")
+
+	requireTeardownRetained(t, f.kv, "alpha")
+	assert.Same(t, before, f.svc.leases.Get(name))
 }
 
 func TestTeardownLeaseReleaseSurvivesCallerCancellation(t *testing.T) {
@@ -413,8 +507,7 @@ func TestTeardownLeaseReleaseSurvivesCallerCancellation(t *testing.T) {
 	cancel()
 	lease.release()
 
-	_, err := f.svc.leaderKV.Get(t.Context(), teardownLeaderKey(testAccountID, "alpha"))
-	require.ErrorIs(t, err, jetstream.ErrKeyNotFound)
+	assert.False(t, leaseKeyHeld(t, f.svc, teardownLeaseName(testAccountID, "alpha")))
 }
 
 // TestDeleteCluster_ManagedCPVPCReclaimsCustomerVPCSGs guards the contract: in

spinifex/domains/eks/cluster/eks_deleting_reaper_test.go

@@ -62,8 +62,7 @@ func TestDeletingReaperLeavesAHeldTeardownAlone(t *testing.T) {
 	t.Parallel()
 	f := newDeleteClusterFixture(t, "beta")
 	markDeleting(t, f, "beta", time.Second)
-	_, err := f.svc.leaderKV.Create(t.Context(), teardownLeaderKey(testAccountID, "beta"), []byte("node-2"))
-	require.NoError(t, err)
+	holdElsewhere(t, f.svc, teardownLeaseName(testAccountID, "beta"))
 
 	assert.Equal(t, 0, sweepAndSettle(t, f))
 

spinifex/domains/eks/cluster/eks_nodegroup_deleting_reaper.go

@@ -106,18 +106,22 @@ func (r *EKSNodegroupDeletingReaper) due(rec *NodegroupRecord) bool {
 // startNodegroupDelete runs one converger pass in the background and reports
 // whether it started; a pass already running here or on another node refuses.
 func (s *EKSServiceImpl) startNodegroupDelete(accountID string, kv jetstream.KeyValue, cluster, ng string) bool {
-	release, _, ok := s.holdTeardown(accountID, cluster+"/nodegroup/"+ng, false)
+	gate := cluster + "/nodegroup/" + ng
+	release, _, ok := s.holdTeardown(accountID, gate, false)
 	if !ok {
 		return false
 	}
 	s.teardownWG.Go(func() {
-		defer release()
+		deleted := false
+		defer func() { release(deleted) }()
 		ctx, cancel := context.WithTimeout(s.bgCtx, teardownPassTimeout)
 		defer cancel()
 		if err := s.convergeNodegroupDeletion(ctx, accountID, kv, cluster, ng); err != nil {
 			slog.Warn("eks: nodegroup delete incomplete, the record stays DELETING for retry",
 				"cluster", cluster, "nodegroup", ng, "account", accountID, "err", err)
+			return
 		}
+		deleted = true
 	})
 	return true
 }

spinifex/domains/eks/cluster/leases.go

@@ -0,0 +1,48 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+)
+
+// KeyPrefixCluster is the prefix of every per-cluster lease in the sync bucket.
+const KeyPrefixCluster = "eks.cluster."
+
+// clusterLeaseTTL is every per-cluster lease's time to live.
+const clusterLeaseTTL = 60 * time.Second
+
+// NewClusterLeases returns the per-cluster lease group for holder. The EKS
+// owner builds it once per process: a cluster's reconciler and teardown leases
+// are members of it, and a deleted cluster's are forgotten.
+func NewClusterLeases(store kvsync.KVStore, holder string) *kvsync.LeaseGroup {
+	return kvsync.NewLeaseGroup(store, KeyPrefixCluster, holder, clusterLeaseTTL)
+}
+
+// tryLease makes one attempt at lease, logging any failure but a held lease,
+// and returns its release, which survives a cancelled ctx, and its Lost.
+func tryLease(ctx context.Context, lease *kvsync.Lease) (func(), <-chan struct{}, bool) {
+	tok, err := lease.TryLock(ctx)
+	if err != nil {
+		if _, held := errors.AsType[*kvsync.HeldError](err); !held {
+			slog.WarnContext(ctx, "eks: lease attempt failed", "key", lease.Key(), "err", err)
+		}
+		return nil, nil, false
+	}
+	return func() { lease.Unlock(ctx, tok) }, lease.Lost(), true
+}
+
+// clusterLeaseName names a cluster's reconciler lease in the group.
+func clusterLeaseName(accountID, clusterName string) string {
+	return accountID + "/" + clusterName
+}
+
+// teardownLeaseName names the per-cluster cross-node gate for a DELETING
+// teardown. It is distinct from clusterLeaseName so a delete never contends
+// with the still-live reconciler, which holds that lease until it reads DELETING.
+func teardownLeaseName(accountID, clusterName string) string {
+	return clusterLeaseName(accountID, clusterName) + "/teardown"
+}

spinifex/domains/eks/cluster/nodegroup_delete_test.go

@@ -662,3 +662,23 @@ func TestDeleteNodegroup_WorkerStuckPastDeadlineEndsDeleteFailedKeepingProfile(t
 	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
 	assert.NotContains(t, fixtureIAM(t, f).Profiles, rec.InstanceProfileName)
 }
+
+// A finished nodegroup delete drops its teardown lease from the group, so the
+// group does not grow with every nodegroup ever deleted and a recreated
+// nodegroup of the same name starts from a fresh lease.
+func TestNodegroupDelete_RetiresItsTeardownLease(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	name := teardownLeaseName(testAccountID, "c1/nodegroup/ng1")
+	before := f.svc.leases.Get(name)
+
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	sweepNodegroups(t, f)
+
+	_, err = GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
+	require.ErrorIs(t, err, ErrNodegroupNotFound, "the delete must have completed")
+	assert.NotSame(t, before, f.svc.leases.Get(name), "the finished delete's teardown lease must leave the group")
+}

spinifex/domains/eks/cluster/reconciler_registry.go

@@ -5,14 +5,12 @@ import (
 	"errors"
 	"log/slog"
 	"sync"
+	"time"
 
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
-// ErrLeaseHeld signals the leader lease is held by another node.
-// Spawn treats it as benign and retries on the next tick.
-var ErrLeaseHeld = errors.New("eks: reconciler lease held by another node")
-
 // ReconcilerRegistry tracks active per-cluster reconciler goroutines; one entry per (accountID, clusterName).
 type ReconcilerRegistry struct {
 	mu      sync.Mutex
@@ -64,9 +62,6 @@ func (r *ReconcilerRegistry) Spawn(parent context.Context, accountID, clusterNam
 		r.mu.Unlock()
 		cancel()
 		close(done)
-		if errors.Is(err, ErrLeaseHeld) {
-			return nil
-		}
 		return err
 	}
 
@@ -151,33 +146,67 @@ func registryKey(accountID, clusterName string) string {
 	return accountID + "/" + clusterName
 }
 
-// RunClusterReconciler is the production SpawnReconcilerFn: constructs a ClusterReconciler,
-// acquires the lease, and drives Run in a goroutine. Returns ErrLeaseHeld if another node wins.
+// RunClusterReconciler contends for the cluster's lease and drives Run while it
+// holds it, in a goroutine, until ctx ends or the cluster leaves CREATING and
+// ACTIVE. A held lease is asked for again every ttl/3, so a dead holder's key is
+// taken over once it has sat a TTL, and a lost one is contended for again.
+// release stops the goroutine and returns once the lease is unlocked.
 func RunClusterReconciler(
 	ctx context.Context,
-	leaderKV, acctKV jetstream.KeyValue,
-	accountID, clusterName, holderID, healthURL string,
+	lease *kvsync.Lease, acctKV jetstream.KeyValue,
+	accountID, clusterName, healthURL string,
 	opts ...ReconcilerOption,
 ) (func(), <-chan struct{}, error) {
-	r, err := NewClusterReconciler(leaderKV, acctKV, accountID, clusterName, holderID, healthURL, opts...)
+	newRun := func() (*ClusterReconciler, error) {
+		return NewClusterReconciler(lease, acctKV, accountID, clusterName, healthURL, opts...)
+	}
+	r, err := newRun()
 	if err != nil {
 		return nil, nil, err
 	}
-	release, ok := r.AcquireLease(ctx)
-	if !ok {
-		slog.Info("RunClusterReconciler: lease held elsewhere, skipping spawn",
-			"accountID", accountID, "cluster", clusterName)
-		return nil, nil, ErrLeaseHeld
-	}
-	// Closed when Run returns so the registry can drop this entry on a self-exit
-	// (terminal state / lost lease), letting a later same-name create re-spawn.
+	runCtx, cancel := context.WithCancel(ctx)
 	finished := make(chan struct{})
 	go func() {
 		defer close(finished)
-		if runErr := r.Run(ctx); runErr != nil && !errors.Is(runErr, context.Canceled) {
-			slog.Info("RunClusterReconciler: Run exited",
-				"accountID", accountID, "cluster", clusterName, "err", runErr)
+		for {
+			if release, ok := r.AcquireLease(runCtx); ok {
+				err := r.Run(runCtx)
+				release()
+				if !errors.Is(err, ErrReconcilerLeaseLost) {
+					if err != nil && !errors.Is(err, context.Canceled) {
+						slog.Info("RunClusterReconciler: Run exited",
+							"accountID", accountID, "cluster", clusterName, "err", err)
+					}
+					return
+				}
+				slog.Info("RunClusterReconciler: lease lost, contending again",
+					"accountID", accountID, "cluster", clusterName)
+			}
+			if !clusterContended(runCtx, acctKV, clusterName) {
+				return
+			}
+			select {
+			case <-runCtx.Done():
+				return
+			case <-time.After(lease.TTL() / 3):
+			}
+			// A fresh reconciler per holding: a finished Run keeps its exit reason.
+			r, _ = newRun()
 		}
 	}()
-	return release, finished, nil
+	return func() { cancel(); <-finished }, finished, nil
+}
+
+// clusterContended reports whether the cluster is still one a reconciler is
+// wanted for. A read that fails is not evidence it is gone, so it keeps trying.
+func clusterContended(ctx context.Context, acctKV jetstream.KeyValue, clusterName string) bool {
+	meta, err := GetClusterMeta(ctx, acctKV, clusterName)
+	switch {
+	case errors.Is(err, ErrClusterNotFound):
+		return false
+	case err != nil:
+		return ctx.Err() == nil
+	default:
+		return meta.Status == ClusterStatusCreating || meta.Status == ClusterStatusActive
+	}
 }

spinifex/domains/eks/cluster/reconciler_registry_test.go

@@ -63,34 +63,6 @@ func TestReconcilerRegistry_SpawnFnErrorRemovesEntry(t *testing.T) {
 	assert.False(t, reg.Has("111122223333", "alpha"), "entry must not linger after spawn failure")
 }
 
-func TestReconcilerRegistry_LeaseHeldDropsEntryAndRetries(t *testing.T) {
-	t.Parallel()
-	reg := NewReconcilerRegistry()
-	var spawnCalls atomic.Int32
-	leaseHeld := atomic.Bool{}
-	leaseHeld.Store(true)
-	fn := func(ctx context.Context, _, _ string) (func(), <-chan struct{}, error) {
-		spawnCalls.Add(1)
-		if leaseHeld.Load() {
-			return nil, nil, ErrLeaseHeld
-		}
-		go func() { <-ctx.Done() }()
-		return func() {}, nil, nil
-	}
-
-	// Lease held elsewhere: Spawn is benign (nil), records no phantom holder.
-	require.NoError(t, reg.Spawn(t.Context(), "111122223333", "alpha", fn))
-	assert.False(t, reg.Has("111122223333", "alpha"), "lease-held must not leave a holder")
-
-	// Holder's TTL lapses; a later Spawn re-invokes fn and now takes over —
-	// no daemon restart needed.
-	leaseHeld.Store(false)
-	require.NoError(t, reg.Spawn(t.Context(), "111122223333", "alpha", fn))
-	assert.True(t, reg.Has("111122223333", "alpha"))
-	assert.EqualValues(t, 2, spawnCalls.Load(), "second Spawn must re-attempt after lease-held")
-	reg.StopAll()
-}
-
 // TestReconcilerRegistry_SelfExitDropsEntryAndRespawns covers a reconciler that
 // exits on its own — a terminal cluster state or a lost lease — with no Stop()
 // to cancel its ctx. The registry must drop the entry (and release the lease) so

spinifex/domains/eks/cluster/service_impl.go

@@ -34,6 +34,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/paging"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/idempotency"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -55,6 +56,9 @@ type EKSServiceDeps struct {
 	Region         string
 	HolderID       string
 
+	// Leases is the per-cluster lease group, from NewClusterLeases.
+	Leases *kvsync.LeaseGroup
+
 	// InternalSuffix is the AWS-parity internal DNS suffix (e.g. spinifex.internal)
 	// used to compose the worker's ECR registry host.
 	InternalSuffix string
@@ -262,7 +266,7 @@ type csiVolumeReclaimer interface {
 // EKSServiceImpl is the daemon-side EKSService implementation.
 type EKSServiceImpl struct {
 	deps     EKSServiceDeps
-	leaderKV jetstream.KeyValue
+	leases   *kvsync.LeaseGroup
 	registry *ReconcilerRegistry
 	owner    *access.Owner
 
@@ -340,7 +344,7 @@ const (
 	defaultLaunchStaleAfter     = 90 * time.Second
 )
 
-// NewEKSServiceImpl initialises EKSServiceImpl, wiring the leader KV and reconciler registry.
+// NewEKSServiceImpl initialises EKSServiceImpl, wiring the lease group and reconciler registry.
 func NewEKSServiceImpl(deps EKSServiceDeps) (*EKSServiceImpl, error) {
 	if err := addon.ValidateCatalog(); err != nil {
 		return nil, fmt.Errorf("eks: validate add-on catalog: %w", err)
@@ -348,21 +352,13 @@ func NewEKSServiceImpl(deps EKSServiceDeps) (*EKSServiceImpl, error) {
 	if deps.NATSConn == nil {
 		return nil, errors.New("eks: NewEKSServiceImpl nil NATSConn")
 	}
-	js, err := jetstream.New(deps.NATSConn)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
+	if deps.Leases == nil {
+		return nil, errors.New("eks: NewEKSServiceImpl nil Leases")
 	}
-	// The leader bucket outlives every request, so its open is bounded by the
-	// service's own background context rather than a caller's.
 	ctx, cancel := context.WithCancel(context.Background())
-	leaderKV, err := InitLeaderBucket(ctx, js)
-	if err != nil {
-		cancel()
-		return nil, err
-	}
 	return &EKSServiceImpl{
 		deps:                     deps,
-		leaderKV:                 leaderKV,
+		leases:                   deps.Leases,
 		registry:                 NewReconcilerRegistry(),
 		owner:                    access.New(deps.Region),
 		bgCtx:                    ctx,
@@ -1384,7 +1380,7 @@ func (s *EKSServiceImpl) claimClusterName(ctx context.Context, accountID string,
 		if !held {
 			return errors.New(awserrors.ErrorEKSResourceInUse)
 		}
-		defer release()
+		defer release(false)
 	}
 	var oldRefs *ClusterMeta
 	reclaimed := false
@@ -2368,7 +2364,7 @@ func (s *EKSServiceImpl) spawnBootstrap(accountID, clusterName string, kv jetstr
 		return
 	}
 	ctx, cancel := context.WithCancel(s.bgCtx)
-	key := reconcilerLeaderKey(accountID, clusterName)
+	key := clusterLeaseName(accountID, clusterName)
 	entry := &bootstrapRun{cancel: cancel}
 	s.bootstrapMu.Lock()
 	if s.bootstraps == nil {
@@ -2413,7 +2409,7 @@ type bootstrapRun struct {
 // stopBootstrap cancels this process's bootstrap subscriber for a cluster, so
 // a cluster being torn down stops collecting artifacts into its prefix.
 func (s *EKSServiceImpl) stopBootstrap(accountID, clusterName string) {
-	key := reconcilerLeaderKey(accountID, clusterName)
+	key := clusterLeaseName(accountID, clusterName)
 	s.bootstrapMu.Lock()
 	run := s.bootstraps[key]
 	delete(s.bootstraps, key)
@@ -2454,7 +2450,7 @@ func (s *EKSServiceImpl) spawnReconciler(accountID, clusterName string, _ *Clust
 		opts = append(opts, WithEtcdResetRecovery(0, 0, 0))
 	}
 	spawn := func(ctx context.Context, _, _ string) (func(), <-chan struct{}, error) {
-		return RunClusterReconciler(ctx, s.leaderKV, acctKV, accountID, clusterName, s.deps.HolderID, "", opts...)
+		return RunClusterReconciler(ctx, s.leases.Get(clusterLeaseName(accountID, clusterName)), acctKV, accountID, clusterName, "", opts...)
 	}
 	if err := s.registry.Spawn(s.bgCtx, accountID, clusterName, spawn); err != nil {
 		slog.Error("spawnReconciler: registry spawn", "cluster", clusterName, "err", err)

spinifex/domains/eks/cluster/service_impl_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/domains/eks/access"
 	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
@@ -20,12 +21,20 @@ import (
 
 func setupTestService(t *testing.T) *EKSServiceImpl {
 	t.Helper()
-	_, nc, _ := testutil.StartTestJetStream(t)
-	svc, err := NewEKSServiceImpl(EKSServiceDeps{NATSConn: nc})
+	_, nc, js := testutil.StartTestJetStream(t)
+	svc, err := NewEKSServiceImpl(EKSServiceDeps{NATSConn: nc, Leases: NewClusterLeases(kvstore.OpenSyncBucket(js), "test")})
 	require.NoError(t, err)
 	return svc
 }
 
+// A service without the daemon's lease group would contend on keys no other
+// owner sees, so construction refuses it.
+func TestNewEKSServiceImpl_RequiresLeases(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	_, err := NewEKSServiceImpl(EKSServiceDeps{NATSConn: nc})
+	require.ErrorContains(t, err, "nil Leases")
+}
+
 // TestEKSServiceImpl_ClusterLifecycleShimMode covers the four lifecycle
 // methods when the service is constructed with only NATS wiring (the path the
 // daemon-handler routing test uses): orchestration deps

spinifex/domains/eks/cluster/store.go

@@ -5,9 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"strings"
-	"time"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
@@ -19,10 +17,6 @@ import (
 // Per-account bucket "eks-account-{accountID}" holds all customer-visible
 // cluster state. It is created lazily on first cluster touch (no daemon-boot
 // pre-creation) so accounts without any EKS clusters never grow a bucket.
-//
-// Shared leader bucket "spinifex-eks-leader" holds 60s-TTL CAS locks
-// keyed by "{accountID}/{clusterName}" for the per-cluster reconciler
-// leader election. Created once at daemon boot.
 const (
 	KVBucketEKSAccountPrefix  = "eks-account-"
 	KVBucketEKSAccountVersion = 1
@@ -33,10 +27,6 @@ const (
 	// survives a DeleteCluster. Keep this decoupled from the schema version so a
 	// future version bump can't silently widen history and resurrect ciphertext.
 	KVBucketEKSAccountHistory = 1
-
-	KVBucketEKSLeader        = "spinifex-eks-leader"
-	KVBucketEKSLeaderVersion = 1
-	KVBucketEKSLeaderTTL     = 60 * time.Second
 )
 
 // Key-path helpers for the per-account bucket. The layout matches the
@@ -123,15 +113,15 @@ func RecoveryDirectiveKey(cluster, instanceID string) string {
 	return RecoveryPrefix(cluster) + instanceID
 }
 
-// Store is the per-daemon EKS KV handle. Per-account and leader buckets are
-// accessed via the package-level factories below.
+// Store is the per-daemon EKS KV handle. Per-account buckets are accessed via
+// the package-level factories below.
 type Store struct {
 	nc *nats.Conn
 }
 
 // NewStore constructs a Store bound to the supplied NATS connection. It does
 // not touch JetStream — per-account buckets are created lazily by
-// GetOrCreateAccountBucket and the leader bucket by InitLeaderBucket.
+// GetOrCreateAccountBucket.
 func NewStore(nc *nats.Conn) (*Store, error) {
 	if nc == nil {
 		return nil, errors.New("eks store: nats connection is nil")
@@ -212,13 +202,3 @@ func GetOrCreateAccountBucket(ctx context.Context, js jetstream.JetStream, accou
 func OpenAccountBucket(ctx context.Context, js jetstream.JetStream, accountID string) (jetstream.KeyValue, error) {
 	return js.KeyValue(ctx, AccountBucketName(accountID))
 }
-
-// InitLeaderBucket creates (or attaches to) the shared spinifex-eks-leader
-// bucket used for per-cluster reconciler leader-lease CAS locks.
-func InitLeaderBucket(ctx context.Context, js jetstream.JetStream) (jetstream.KeyValue, error) {
-	return kvlease.OpenBucket(ctx, js, kvlease.BucketConfig{
-		Name:    KVBucketEKSLeader,
-		TTL:     KVBucketEKSLeaderTTL,
-		Version: KVBucketEKSLeaderVersion,
-	})
-}

spinifex/domains/eks/cluster/store_test.go

@@ -49,40 +49,6 @@ func TestGetOrCreateAccountBucket_ReplicasClamped(t *testing.T) {
 	assert.Equal(t, 1, si.Config.Replicas)
 }
 
-func TestInitLeaderBucket_Idempotent(t *testing.T) {
-	t.Parallel()
-	_, nc, _ := testutil.StartTestJetStream(t)
-	js := testutil.NewJetStream(t, nc)
-
-	kv1, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
-	require.NotNil(t, kv1)
-	assert.Equal(t, KVBucketEKSLeader, kv1.Bucket())
-
-	kv2, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
-	require.NotNil(t, kv2)
-	assert.Equal(t, KVBucketEKSLeader, kv2.Bucket())
-}
-
-// TestInitLeaderBucket_ReplicasClamped confirms the leader bucket is created
-// with the requested replica count, clamped to a minimum of 1.
-func TestInitLeaderBucket_ReplicasClamped(t *testing.T) {
-	t.Parallel()
-	_, nc, _ := testutil.StartTestJetStream(t)
-	js := testutil.NewJetStream(t, nc)
-
-	kv, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
-	require.NotNil(t, kv)
-
-	stream, err := js.Stream(t.Context(), "KV_"+KVBucketEKSLeader)
-	require.NoError(t, err)
-	si, err := stream.Info(t.Context())
-	require.NoError(t, err)
-	assert.Equal(t, 1, si.Config.Replicas)
-}
-
 func TestKeyPaths_MatchQ2Spec(t *testing.T) {
 	cases := []struct {
 		name string

spinifex/domains/elbv2/lb_repair.go

@@ -10,20 +10,16 @@ import (
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/systeminstance"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
 	"github.com/nats-io/nats.go"
-	"github.com/nats-io/nats.go/jetstream"
 )
 
-const (
-	// KVBucketELBv2Leader holds the lease key of the data-path reconciler.
-	KVBucketELBv2Leader        = "spinifex-elbv2-leader"
-	KVBucketELBv2LeaderVersion = 1
-	KVBucketELBv2LeaderTTL     = 60 * time.Second
+// KeyDataPath is the data-path reconciler's leader lease in the sync bucket.
+const KeyDataPath = "elbv2.datapath"
 
-	lbRepairLeaderKey     = "datapath"
-	lbRepairLeaseRenew    = 20 * time.Second
+const (
+	lbRepairLeaseTTL      = 60 * time.Second
 	lbRepairPassInterval  = 30 * time.Second
 	lbRepairMinFailing    = 60 * time.Second
 	lbRepairMinVMAge      = 5 * time.Minute
@@ -87,34 +83,29 @@ func newLBRepairReconciler(svc *ELBv2ServiceImpl, rebooter SystemInstanceReboote
 	}
 }
 
-// StartDataPathReconciler elects a leader among the daemons and runs the repair
-// pass on it every lbRepairPassInterval until ctx or the service is cancelled.
-func (s *ELBv2ServiceImpl) StartDataPathReconciler(ctx context.Context, holder string) {
-	if s.InstanceLauncher == nil || s.store == nil || s.store.js == nil {
+// NewDataPathLease returns the data-path reconciler's leader lease for holder.
+func NewDataPathLease(store kvsync.KVStore, holder string) *kvsync.Lease {
+	return kvsync.NewLease(store, KeyDataPath, holder, lbRepairLeaseTTL)
+}
+
+// StartDataPathReconciler contends for lease and, while holding it, runs the
+// repair pass every lbRepairPassInterval until ctx or the service is cancelled.
+func (s *ELBv2ServiceImpl) StartDataPathReconciler(ctx context.Context, lease *kvsync.Lease) {
+	if s.InstanceLauncher == nil || s.store == nil || lease == nil {
 		return
 	}
-	lease, err := kvlease.New(kvlease.Config{
-		Name:   "elbv2/datapath",
-		Bucket: s.leaderBucket,
-		Key:    lbRepairLeaderKey,
-		Holder: holder,
-		TTL:    KVBucketELBv2LeaderTTL,
-		Renew:  lbRepairLeaseRenew,
-		Retry:  lbRepairLeaseRenew,
-	})
-	if err != nil {
-		slog.ErrorContext(ctx, "ELBv2 data-path reconciler: lease config invalid", "err", err)
-		return
+	ctx, cancel := context.WithCancel(ctx)
+	if s.ctx != nil {
+		context.AfterFunc(s.ctx, cancel)
 	}
 	rec := newLBRepairReconciler(s, s.InstanceLauncher, lease.Held)
-	go lease.Run(ctx)
+	go kvsync.RunElection(ctx, lease, nil, nil)
 	go func() {
+		defer cancel()
 		ticker := time.NewTicker(lbRepairPassInterval)
 		defer ticker.Stop()
 		for {
 			select {
-			case <-s.ctx.Done():
-				return
 			case <-ctx.Done():
 				return
 			case <-ticker.C:
@@ -124,14 +115,6 @@ func (s *ELBv2ServiceImpl) StartDataPathReconciler(ctx context.Context, holder s
 	}()
 }
 
-func (s *ELBv2ServiceImpl) leaderBucket(ctx context.Context) (jetstream.KeyValue, error) {
-	return kvlease.OpenBucket(ctx, s.store.js, kvlease.BucketConfig{
-		Name:    KVBucketELBv2Leader,
-		TTL:     KVBucketELBv2LeaderTTL,
-		Version: KVBucketELBv2LeaderVersion,
-	})
-}
-
 // candidate is a load balancer that is failing and may be repaired.
 type lbRepairCandidate struct {
 	lb    *LoadBalancerRecord

spinifex/domains/elbv2/lb_repair_test.go

@@ -11,6 +11,8 @@ import (
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/systeminstance"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -179,6 +181,45 @@ func TestLBRepair_NonHolderNeverRepairs(t *testing.T) {
 	assert.Len(t, e.reboots(), 1)
 }
 
+// Leadership comes from the data-path lease: a node repairs nothing while
+// another holds it, repairs once elected, and stops the moment it is taken over.
+func TestLBRepair_ActsOnlyWhileHoldingTheDataPathLease(t *testing.T) {
+	t.Parallel()
+	e := newRepairEnv(t)
+	e.seed("lbr4a", true)
+	e.seed("lbr4b", true)
+	e.fail("lbr4b", 3*time.Minute)
+
+	store := kvstore.OpenSyncBucket(e.svc.store.js)
+	lease := kvsync.NewLease(store, KeyDataPath, "node-a", 300*time.Millisecond)
+	e.rec.isLeader = lease.Held
+	other := NewDataPathLease(kvstore.OpenSyncBucket(e.svc.store.js), "node-b")
+	tok, err := other.TryLock(t.Context())
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(t.Context())
+	done := make(chan struct{})
+	go func() { defer close(done); kvsync.RunElection(ctx, lease, nil, nil) }()
+	t.Cleanup(func() { cancel(); <-done })
+
+	e.pass()
+	assert.False(t, lease.Held())
+	assert.Empty(t, e.reboots(), "a node that lost the election must not repair")
+
+	other.Unlock(t.Context(), tok)
+	require.Eventually(t, lease.Held, 5*time.Second, 5*time.Millisecond)
+	e.pass()
+	assert.Equal(t, []string{"i-lbr4b"}, e.reboots())
+
+	syncKV, err := store.KV(t.Context())
+	require.NoError(t, err)
+	_, err = syncKV.Put(t.Context(), KeyDataPath, []byte(`{"holder":"node-b","generation":0}`))
+	require.NoError(t, err)
+	require.Eventually(t, func() bool { return !lease.Held() }, 5*time.Second, 5*time.Millisecond)
+	e.pass()
+	assert.Equal(t, []string{"i-lbr4b"}, e.reboots(), "a node whose lease was taken over must not repair")
+}
+
 func TestLBRepair_OneRepairPerPass(t *testing.T) {
 	t.Parallel()
 	e := newRepairEnv(t)

spinifex/domains/network/reconcile/drift.go

@@ -14,6 +14,7 @@ import (
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
 	loop "github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/reconciler"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -82,14 +83,14 @@ func intentSource(js jetstream.JetStream) loop.Source {
 }
 
 // DriftLoop reconciles on every intent change and every DriftInterval, gated on
-// AcquireLeader so only one vpcd scans at a time. An incomplete pass asks to be
+// AcquireLeader on lease so only one vpcd scans at a time. An incomplete pass asks to be
 // revisited on a short backoff instead. Returns when ctx is cancelled.
 //
 // startup is the outcome of the bootstrap ReconcileApplyOnly pass, or nil when
 // this node did not run one. It seeds the first deadline rather than triggering
 // a pass: that bootstrap pass is the startup scan, and running a second one here
 // would duplicate it and pull a scan forward on every clean boot.
-func DriftLoop(ctx context.Context, rec Reconciler, nc *nats.Conn, localAZ, holder string, startup error) {
+func DriftLoop(ctx context.Context, rec Reconciler, nc *nats.Conn, lease *kvsync.Lease, localAZ string, startup error) {
 	js, err := jetstream.New(nc)
 	if err != nil {
 		slog.Error("reconcile/drift: JetStream context unavailable, drift loop disabled", "err", err)
@@ -99,7 +100,7 @@ func DriftLoop(ctx context.Context, rec Reconciler, nc *nats.Conn, localAZ, hold
 	loop.Run(ctx, loop.Config{
 		Name:      "vpcd/drift",
 		Sources:   []loop.Source{intentSource(js)},
-		Reconcile: driftPass(rec, nc, js, localAZ, holder, startup),
+		Reconcile: driftPass(rec, lease, js, localAZ, startup),
 		Resync:    DriftInterval,
 		Debounce:  driftDebounce,
 	})
@@ -108,7 +109,7 @@ func DriftLoop(ctx context.Context, rec Reconciler, nc *nats.Conn, localAZ, hold
 // driftPass returns the reconcile pass, closing over the backoff ladder and the
 // last pass time. Run never calls it concurrently with itself, which is why
 // neither needs a lock.
-func driftPass(rec Reconciler, nc *nats.Conn, js jetstream.JetStream, localAZ, holder string, startup error) func(context.Context) (time.Duration, error) {
+func driftPass(rec Reconciler, lease *kvsync.Lease, js jetstream.JetStream, localAZ string, startup error) func(context.Context) (time.Duration, error) {
 	var backoff time.Duration
 	var lastPass time.Time
 	seeded := false
@@ -127,7 +128,8 @@ func driftPass(rec Reconciler, nc *nats.Conn, js jetstream.JetStream, localAZ, h
 			return wait, nil
 		}
 		previous := backoff
-		backoff = nextDriftBackoff(backoff, runDriftCycle(ctx, rec, nc, js, localAZ, holder))
+		retry, err := runDriftCycle(ctx, rec, lease, js, localAZ)
+		backoff = nextDriftBackoff(backoff, err)
 		// Stamped after the scan, not before: the floor is a gap between passes,
 		// and a pass slower than the floor would otherwise leave none at all.
 		lastPass = time.Now()
@@ -136,7 +138,7 @@ func driftPass(rec Reconciler, nc *nats.Conn, js jetstream.JetStream, localAZ, h
 		}
 		// The outcome is logged by the cycle, and an unconverged pass is a change
 		// of schedule rather than a failure of the loop.
-		return backoff, nil
+		return loop.Earliest(backoff, retry), nil
 	}
 }
 
@@ -157,22 +159,22 @@ func nextDriftBackoff(current time.Duration, err error) time.Duration {
 
 // runDriftCycle is one pass body, split out so tests can drive it directly.
 // Returns the reconcile outcome so the caller can size the next wait; a cycle
-// this node did not win the election for returns nil.
-func runDriftCycle(ctx context.Context, rec Reconciler, nc *nats.Conn, js jetstream.JetStream, localAZ, holder string) error {
-	release, elected := AcquireLeader(ctx, nc, KVBucketVPCDReconcile, holder)
+// this node did not win the election for returns nil and the election's revisit.
+func runDriftCycle(ctx context.Context, rec Reconciler, lease *kvsync.Lease, js jetstream.JetStream, localAZ string) (time.Duration, error) {
+	release, retry, elected := ElectPass(ctx, lease)
 	if !elected {
-		return nil
+		return retry, nil
 	}
 	defer release()
 
 	intent, err := LoadIntentFromKV(ctx, js, localAZ)
 	if err != nil {
 		slog.Warn("reconcile/drift: load intent failed", "err", err)
-		return err
+		return 0, err
 	}
 	if err := rec.Reconcile(ctx, intent); err != nil {
 		slog.Warn("reconcile/drift: reconcile failed", "err", err)
-		return err
+		return 0, err
 	}
-	return nil
+	return 0, nil
 }

spinifex/domains/network/reconcile/drift_test.go

@@ -72,7 +72,7 @@ func startDriftLoop(t *testing.T, rec Reconciler, nc *nats.Conn, startup error)
 	done := make(chan struct{})
 	go func() {
 		defer close(done)
-		DriftLoop(ctx, rec, nc, "us-east-1a", "node-1", startup)
+		DriftLoop(ctx, rec, nc, vpcdLease(testutil.NewJetStream(t, nc), "node-1"), "us-east-1a", startup)
 	}()
 	t.Cleanup(func() {
 		cancel()
@@ -171,34 +171,38 @@ func TestDriftLoop_ConvergedPassReturnsToDriftInterval(t *testing.T) {
 // win must not be reported as converged-or-broken; it reports nil and the caller
 // leaves the cadence alone.
 func TestRunDriftCycle_ReportsReconcileOutcome(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 
 	rec := &stubReconciler{outcomes: []error{incompleteErr()}}
-	err := runDriftCycle(t.Context(), rec, nc, js, "us-east-1a", "node-1")
+	_, err := runDriftCycle(t.Context(), rec, vpcdLease(js, "node-1"), js, "us-east-1a")
 	if !errors.Is(err, ErrPassIncomplete) {
 		t.Fatalf("runDriftCycle = %v, want ErrPassIncomplete propagated from the pass", err)
 	}
 
 	converged := &stubReconciler{outcomes: []error{nil}}
-	if err := runDriftCycle(t.Context(), converged, nc, js, "us-east-1a", "node-1"); err != nil {
+	if _, err := runDriftCycle(t.Context(), converged, vpcdLease(js, "node-1"), js, "us-east-1a"); err != nil {
 		t.Errorf("runDriftCycle = %v, want nil for a converged pass", err)
 	}
 }
 
 // A cycle this node loses the election for must not run a pass at all.
 func TestRunDriftCycle_SkipsWhenNotElected(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 
-	release, ok := AcquireLeader(t.Context(), nc, KVBucketVPCDReconcile, "other-node")
+	release, ok := AcquireLeader(t.Context(), vpcdLease(js, "other-node"))
 	if !ok {
 		t.Fatal("failed to take the lock the test needs held")
 	}
 	defer release()
 
 	rec := &stubReconciler{outcomes: []error{incompleteErr()}}
-	if err := runDriftCycle(t.Context(), rec, nc, js, "us-east-1a", "node-1"); err != nil {
+	retry, err := runDriftCycle(t.Context(), rec, vpcdLease(js, "node-1"), js, "us-east-1a")
+	if err != nil {
 		t.Errorf("runDriftCycle = %v, want nil when not elected", err)
 	}
+	if retry <= 0 || retry > LeaderTTL/3 {
+		t.Errorf("revisit = %v, want within %v so a dead holder is taken over promptly", retry, LeaderTTL/3)
+	}
 	if got := rec.callCount(); got != 0 {
 		t.Errorf("reconcile ran %d times without winning the election, want 0", got)
 	}

spinifex/domains/network/reconcile/drift_watch_test.go

@@ -213,11 +213,11 @@ func TestIntentSource_CoversEveryBucketTheIntentLoadReads(t *testing.T) {
 // The floor is what stops a busy cluster scanning continuously, so it is pinned
 // on the pass rather than inferred from the loop's timing.
 func TestDriftPass_FloorsTheChangeDrivenRate(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	shrinkDriftTiming(t, time.Minute, time.Hour)
 
 	rec := &stubReconciler{outcomes: []error{nil}}
-	pass := driftPass(rec, nc, js, "us-east-1a", "node-1", nil)
+	pass := driftPass(rec, vpcdLease(js, "node-1"), js, "us-east-1a", nil)
 
 	// The seed pass does no work: the bootstrap ReconcileApplyOnly was the
 	// startup scan and this one must not repeat it.

spinifex/domains/network/reconcile/host_ingress_test.go

@@ -358,7 +358,7 @@ func TestHostIngress_IntentReadFailureNeverPrunes(t *testing.T) {
 // while another node holds the drift lease and no event arrives, well inside
 // its resync.
 func TestRunHostIngress_IntentWriteConvergesEveryNodeWithoutLeaseOrEvent(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	ctx := t.Context()
 	vpcs, err := kvstore.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 1)
 	if err != nil {
@@ -369,7 +369,7 @@ func TestRunHostIngress_IntentWriteConvergesEveryNodeWithoutLeaseOrEvent(t *test
 		t.Fatalf("create IGW bucket: %v", err)
 	}
 	put(t, vpcs, "acct/vpc-1", ec2vpc.VPCRecord{VpcId: "vpc-1", CidrBlock: "10.1.0.0/16"})
-	release, elected := reconcile.AcquireLeader(ctx, nc, reconcile.KVBucketVPCDReconcile, "node-elsewhere")
+	release, elected := reconcile.AcquireLeader(ctx, reconcile.NewLeaderLease(kvstore.OpenSyncBucket(js), reconcile.KeyVPCDReconcile, "node-elsewhere"))
 	if !elected {
 		t.Fatal("could not take the drift lease for another node")
 	}

spinifex/domains/network/reconcile/lock.go

@@ -2,52 +2,100 @@ package reconcile
 
 import (
 	"context"
+	"errors"
 	"log/slog"
 	"time"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
-	"github.com/nats-io/nats.go"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	otelsetup "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
 )
 
-// Single CAS-elected leader key; TTL bounds crash-recovery.
-const (
-	KVBucketVPCDReconcile = "spinifex-vpcd-reconcile"
-	reconcileLeaderKey    = "leader"
-)
+// KeyVPCDReconcile is vpcd's network reconcile lease in the sync bucket.
+const KeyVPCDReconcile = "vpcd.reconcile"
 
-// Leader-key lifetime. Vars (not consts) so tests can shrink them.
-//
-// reconcileLeaderRenew is the gap between refreshes: a pass can outlive the TTL
-// on its own — one stalled gw-lrp DORA runs ~64s — so the key is renewed while
-// the pass runs rather than left to expire underneath it.
-var (
-	reconcileLeaderTTL   = 60 * time.Second
-	reconcileLeaderRenew = 20 * time.Second
-)
+// LeaderTTL is the per-pass leader leases' time to live. A pass can outlive it
+// on its own (one stalled gw-lrp DORA runs ~64s), so the lease renews itself
+// while the pass runs rather than lapsing underneath it.
+const LeaderTTL = 60 * time.Second
 
-// AcquireLeader elects one leader on the named lock bucket. Independent
-// reconcile loops pass distinct buckets so they never share a single mutex: the
-// gateway quota reconcile must not block vpcd's network reconcile, and vice
-// versa.
-func AcquireLeader(ctx context.Context, nc *nats.Conn, bucket, holder string) (func(), bool) {
-	lease, err := kvlease.New(kvlease.Config{
-		Name:   "reconcile/lock",
-		Bucket: kvlease.NATSBucket(nc, bucket, reconcileLeaderTTL),
-		Key:    reconcileLeaderKey,
-		Holder: holder,
-		Attrs:  []any{"bucket", bucket},
-		TTL:    reconcileLeaderTTL,
-		Renew:  reconcileLeaderRenew,
-	})
-	if err != nil {
-		slog.ErrorContext(ctx, "reconcile/lock: lease config invalid",
-			"node", holder, "bucket", bucket, "err", err)
-		return nil, false
-	}
-	if !lease.TryAcquire(ctx) {
-		slog.InfoContext(ctx, "reconcile/lock: not leader, skipping reconcile",
-			"node", holder, "bucket", bucket)
-		return nil, false
-	}
-	return func() { lease.Release(ctx) }, true
+// NewLeaderLease returns a per-pass leader lease on key. Each is built once per
+// process and shared by every site that reconciles under it, so those sites
+// exclude each other on one node as well as across nodes.
+func NewLeaderLease(store kvsync.KVStore, key, holder string) *kvsync.Lease {
+	return kvsync.NewLease(store, key, holder, LeaderTTL)
+}
+
+// takeoverRetry is when a pass that found the lease held asks to run again. A
+// dead holder's key is taken over once a follower has seen its revision sit for
+// a TTL, so retrying at the renewal interval takes over within about 4*TTL/3.
+const takeoverRetry = LeaderTTL / 3
+
+// Bounded wait for JetStream quorum on a cold multi-node start, for a one-shot
+// election that has no later pass to retry it. Vars so tests can shrink them.
+var (
+	startRetryFor  = 60 * time.Second
+	startRetryStep = time.Second
+)
+
+// ElectPass is AcquireLeader for a pass in a loop, adding a revisit deadline:
+// a pass that found the lease held asks to run again once a dead holder could
+// have been waited out, rather than waiting for the loop's next resync. The
+// release survives a cancelled ctx.
+func ElectPass(ctx context.Context, lease *kvsync.Lease) (release func(), revisit time.Duration, ok bool) {
+	tok, err := lease.TryLock(ctx)
+	if err != nil {
+		logFailure(ctx, lease, err)
+		if _, held := errors.AsType[*kvsync.HeldError](err); held {
+			return nil, takeoverRetry, false
+		}
+		return nil, 0, false
+	}
+	return func() { lease.Unlock(ctx, tok) }, 0, true
+}
+
+// AcquireLeaderAtStart is AcquireLeader for a one-shot election at start,
+// retrying until JetStream has formed quorum or the wait runs out, since a cold
+// cluster fails in many shapes. A held lease or a config fault answers at once.
+func AcquireLeaderAtStart(ctx context.Context, lease *kvsync.Lease) (func(), bool) {
+	deadline := time.Now().Add(startRetryFor)
+	for {
+		tok, err := lease.TryLock(ctx)
+		if err == nil {
+			return func() { lease.Unlock(ctx, tok) }, true
+		}
+		if _, held := errors.AsType[*kvsync.HeldError](err); held || clustersize.Permanent(err) {
+			logFailure(ctx, lease, err)
+			return nil, false
+		}
+		if !time.Now().Before(deadline) {
+			slog.WarnContext(ctx, "reconcile/lock: election failed until the start wait ran out, skipping reconcile",
+				"key", lease.Key(), "wait_ms", otelsetup.Millis(startRetryFor), "err", err)
+			return nil, false
+		}
+		slog.DebugContext(ctx, "reconcile/lock: election failed at start, retrying", "key", lease.Key(), "err", err)
+		select {
+		case <-ctx.Done():
+			return nil, false
+		case <-time.After(startRetryStep):
+		}
+	}
+}
+
+// AcquireLeader makes one attempt to take lease for a pass. Independent
+// reconcile loops hold distinct leases, so the gateway quota reconcile never
+// blocks vpcd's network reconcile. The release survives a cancelled ctx.
+func AcquireLeader(ctx context.Context, lease *kvsync.Lease) (func(), bool) {
+	release, _, ok := ElectPass(ctx, lease)
+	return release, ok
+}
+
+func logFailure(ctx context.Context, lease *kvsync.Lease, err error) {
+	if held, ok := errors.AsType[*kvsync.HeldError](err); ok {
+		slog.InfoContext(ctx, "reconcile/lock: not leader, skipping reconcile",
+			"key", lease.Key(), "holder", held.Current.Holder)
+		return
+	}
+	slog.WarnContext(ctx, "reconcile/lock: election failed, skipping reconcile",
+		"key", lease.Key(), "err", err)
 }

spinifex/domains/network/reconcile/lock_test.go

@@ -5,114 +5,83 @@ import (
 	"context"
 	"encoding/json"
 	"log/slog"
+	"sync/atomic"
 	"testing"
 	"time"
 
-	"github.com/mulgadc/spinifex/internal/testkit"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-// TestAcquireLeader_GetOrCreateAcrossCalls pins the get-or-create fix: first
-// acquire creates the bucket and wins; second must attach (not hang on "stream
-// name already in use") and lose; after release a new acquire wins.
-func TestAcquireLeader_GetOrCreateAcrossCalls(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
+// vpcdLease returns holder's vpcd lease on a store of its own, standing in for
+// a separate process.
+func vpcdLease(js jetstream.JetStream, holder string) *kvsync.Lease {
+	return NewLeaderLease(kvstore.OpenSyncBucket(js), KeyVPCDReconcile, holder)
+}
 
-	release, ok := AcquireLeader(t.Context(), nc, KVBucketVPCDReconcile, "node-1")
-	require.True(t, ok, "first acquire must win and create the bucket")
-	require.NotNil(t, release)
+func TestAcquireLeader_OneHolderAtATimeAcrossProcesses(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	node1, node2 := vpcdLease(js, "node-1"), vpcdLease(js, "node-2")
 
-	// Bucket now exists; the second acquire must reach the Create-key contention
-	// path (and lose) rather than dead-ending on CreateKeyValue.
-	loserRelease, ok := AcquireLeader(t.Context(), nc, KVBucketVPCDReconcile, "node-2")
-	assert.False(t, ok, "second acquire must lose while the lock is held")
+	release, ok := AcquireLeader(t.Context(), node1)
+	require.True(t, ok, "the first acquire on a free key must win")
+
+	loserRelease, ok := AcquireLeader(t.Context(), node2)
+	assert.False(t, ok, "a second process must lose while the lease is held")
 	assert.Nil(t, loserRelease)
 
-	// Releasing frees the key so a subsequent acquire can take over.
 	release()
-	release2, ok := AcquireLeader(t.Context(), nc, KVBucketVPCDReconcile, "node-3")
-	require.True(t, ok, "acquire after release must win")
-	require.NotNil(t, release2)
+	release2, ok := AcquireLeader(t.Context(), node2)
+	require.True(t, ok, "acquire after release must win at once")
 	release2()
 }
 
-// TestAcquireLeader_ReleaseSurvivesCancelledContext pins that the release runs
-// on a context of its own. Shutdown cancels the acquiring context first and then
-// releases, so a release bound to that context would silently no-op and park the
-// lock for the full TTL, blocking every other node's reconcile.
-func TestAcquireLeader_ReleaseSurvivesCancelledContext(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
+// The two vpcd sites share one Lease, so they exclude each other on one node
+// as well as across nodes.
+func TestAcquireLeader_SitesSharingALeaseExcludeEachOther(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	shared := vpcdLease(js, "node-1")
 
-	ctx, cancel := context.WithCancel(t.Context())
-	release, ok := AcquireLeader(ctx, nc, KVBucketVPCDReconcile, "node-1")
+	startup, ok := AcquireLeader(t.Context(), shared)
 	require.True(t, ok)
 
+	_, ok = AcquireLeader(t.Context(), shared)
+	assert.False(t, ok, "the drift pass ran while the startup pass held the lease")
+
+	startup()
+	drift, ok := AcquireLeader(t.Context(), shared)
+	require.True(t, ok, "the drift pass must run once the startup pass has released")
+	drift()
+}
+
+// Shutdown cancels the acquiring context first and then releases, so a release
+// bound to that context would no-op and leave the next leader waiting a TTL.
+func TestAcquireLeader_ReleaseSurvivesCancelledContext(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	node1, node2 := vpcdLease(js, "node-1"), vpcdLease(js, "node-2")
+
+	ctx, cancel := context.WithCancel(t.Context())
+	release, ok := AcquireLeader(ctx, node1)
+	require.True(t, ok)
 	cancel()
 	release()
 
-	release2, ok := AcquireLeader(t.Context(), nc, KVBucketVPCDReconcile, "node-2")
-	require.True(t, ok, "lock must be free immediately after release, not held until the TTL reaps it")
+	release2, ok := AcquireLeader(t.Context(), node2)
+	require.True(t, ok, "the lease must be free immediately after release")
 	release2()
 }
 
-// shrinkLeaderTiming compresses the leader key's lifetime so a test can outlast
-// the TTL in under a second rather than a minute.
-func shrinkLeaderTiming(t *testing.T, ttl, renew time.Duration) {
-	t.Helper()
-	oldTTL, oldRenew := reconcileLeaderTTL, reconcileLeaderRenew
-	reconcileLeaderTTL, reconcileLeaderRenew = ttl, renew
-	t.Cleanup(func() { reconcileLeaderTTL, reconcileLeaderRenew = oldTTL, oldRenew })
-}
+// A losing acquire logs who holds the key, not the local node as holder.
+func TestAcquireLeader_NotLeaderLogNamesTheHolder(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	node1, node2 := vpcdLease(js, "node-1"), vpcdLease(js, "node-2")
 
-// A pass can outlive the TTL on its own — one stalled gw-lrp DORA runs ~64s
-// against a 60s TTL — so the key must be renewed while the pass runs. Without
-// this a peer elects itself mid-pass and two reconcilers race the same OVN NB.
-func TestAcquireLeader_RenewsKeyBeyondTTL(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	shrinkLeaderTiming(t, 300*time.Millisecond, 100*time.Millisecond)
-
-	release, ok := AcquireLeader(t.Context(), nc, KVBucketVPCDReconcile, "node-1")
-	require.True(t, ok)
-	defer release()
-
-	// Well past the TTL: an unrenewed key would have expired by now.
-	time.Sleep(750 * time.Millisecond)
-
-	loserRelease, ok := AcquireLeader(t.Context(), nc, KVBucketVPCDReconcile, "node-2")
-	assert.False(t, ok, "leader key expired mid-pass: a peer can now reconcile concurrently")
-	assert.Nil(t, loserRelease)
-}
-
-// If the key was lost anyway, the stale holder's release must not delete the
-// successor's key — an unguarded delete hands the lock to a third node while
-// the successor believes it still holds it.
-func TestAcquireLeader_ReleaseDoesNotDeleteSuccessorKey(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
-
-	release, ok := AcquireLeader(t.Context(), nc, KVBucketVPCDReconcile, "node-1")
-	require.True(t, ok)
-
-	// Stand in for a successor that claimed the key after it expired: the write
-	// bumps the revision, so node-1's release is now CAS-stale.
-	kv, err := js.KeyValue(t.Context(), KVBucketVPCDReconcile)
-	require.NoError(t, err)
-	_, err = kv.Put(t.Context(), reconcileLeaderKey, []byte("node-2"))
-	require.NoError(t, err)
-
-	release()
-
-	entry, err := kv.Get(t.Context(), reconcileLeaderKey)
-	require.NoError(t, err, "stale release deleted the successor's leader key")
-	assert.Equal(t, "node-2", string(entry.Value()), "successor's key must survive a stale release")
-}
-
-// A node that loses the election logs itself as node: "holder=node-2" on
-// node-2's log reads as though node-2 holds the lock.
-func TestAcquireLeader_NotLeaderLogNamesLocalNode(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-
-	release, ok := AcquireLeader(t.Context(), nc, KVBucketVPCDReconcile, "node-1")
+	release, ok := AcquireLeader(t.Context(), node1)
 	require.True(t, ok)
 	defer release()
 
@@ -121,7 +90,7 @@ func TestAcquireLeader_NotLeaderLogNamesLocalNode(t *testing.T) {
 	slog.SetDefault(slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo})))
 	t.Cleanup(func() { slog.SetDefault(prev) })
 
-	_, ok = AcquireLeader(t.Context(), nc, KVBucketVPCDReconcile, "node-2")
+	_, ok = AcquireLeader(t.Context(), node2)
 	require.False(t, ok)
 
 	var line map[string]any
@@ -134,6 +103,148 @@ func TestAcquireLeader_NotLeaderLogNamesLocalNode(t *testing.T) {
 		}
 	}
 	require.NotNil(t, line, "a losing acquire must log that it is skipping")
-	assert.Equal(t, "node-2", line["node"])
-	assert.NotEqual(t, "node-2", line["holder"], "the local node is not the holder")
+	assert.Equal(t, "node-1", line["holder"])
+	assert.Equal(t, KeyVPCDReconcile, line["key"])
+}
+
+// coldStore answers every Load with ErrUnavailable until down reaches zero,
+// standing in for a JetStream still forming quorum.
+type coldStore struct {
+	kvsync.KVStore
+
+	down atomic.Int32
+}
+
+func (c *coldStore) Load(ctx context.Context, key string) ([]byte, uint64, error) {
+	if c.down.Add(-1) >= 0 {
+		return nil, 0, kvsync.ErrUnavailable
+	}
+	return c.KVStore.Load(ctx, key)
+}
+
+func shrinkStartRetry(t *testing.T, retryFor, step time.Duration) {
+	t.Helper()
+	prevFor, prevStep := startRetryFor, startRetryStep
+	startRetryFor, startRetryStep = retryFor, step
+	t.Cleanup(func() { startRetryFor, startRetryStep = prevFor, prevStep })
+}
+
+// A pass that loses to a peer asks to run again within a renewal interval, so a
+// follower that first saw a dead holder's revision sees it a TTL later and takes
+// over about 4*TTL/3 after the last renewal rather than up to 2*TTL.
+func TestElectPass_AHeldLeaseAsksForATakeoverRevisit(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	release, ok := AcquireLeader(t.Context(), vpcdLease(js, "node-1"))
+	require.True(t, ok)
+	defer release()
+
+	_, revisit, ok := ElectPass(t.Context(), vpcdLease(js, "node-2"))
+	assert.False(t, ok)
+	assert.Positive(t, revisit, "a held pass must ask to run again, not wait for a resync")
+	assert.LessOrEqual(t, revisit, LeaderTTL/3, "the retry is a renewal interval out, so takeover is not delayed a further TTL")
+}
+
+func TestElectPass_AWonPassAsksForNoRevisit(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	release, revisit, ok := ElectPass(t.Context(), vpcdLease(js, "node-1"))
+	require.True(t, ok)
+	release()
+	assert.Zero(t, revisit)
+}
+
+// A cold multi-node start has no quorum yet, and the one startup election
+// must wait for it rather than skip the startup reconcile.
+func TestAcquireLeaderAtStart_WaitsOutAnUnavailableStore(t *testing.T) {
+	shrinkStartRetry(t, 10*time.Second, 5*time.Millisecond)
+	_, _, js := testutil.StartTestJetStream(t)
+	cold := &coldStore{KVStore: kvstore.OpenSyncBucket(js)}
+	cold.down.Store(3)
+
+	release, ok := AcquireLeaderAtStart(t.Context(), NewLeaderLease(cold, KeyVPCDReconcile, "node-1"))
+	require.True(t, ok, "the election must be won once the store answers")
+	release()
+}
+
+// formingJS fails its first down bucket lookups and creates with err, as a
+// cluster still electing its meta leader or placing replicas does.
+type formingJS struct {
+	jetstream.JetStream
+
+	err  error
+	down atomic.Int32
+}
+
+func (f *formingJS) KeyValue(ctx context.Context, name string) (jetstream.KeyValue, error) {
+	if f.down.Add(-1) >= 0 {
+		return nil, f.err
+	}
+	return f.JetStream.KeyValue(ctx, name)
+}
+
+func (f *formingJS) CreateKeyValue(ctx context.Context, cfg jetstream.KeyValueConfig) (jetstream.KeyValue, error) {
+	if f.down.Add(-1) >= 0 {
+		return nil, f.err
+	}
+	return f.JetStream.CreateKeyValue(ctx, cfg)
+}
+
+// The startup election waits out each shape a forming cluster fails in, as the
+// real bucket reports it, not only the errors already classed as unavailable.
+func TestAcquireLeaderAtStart_WaitsOutAFormingCluster(t *testing.T) {
+	cases := map[string]error{
+		"no meta leader":       &jetstream.APIError{Code: 503, ErrorCode: 10008, Description: "JetStream system temporarily unavailable"},
+		"no suitable peers":    &jetstream.APIError{Code: 400, ErrorCode: 10005, Description: "no suitable peers for placement"},
+		"insufficient storage": &jetstream.APIError{Code: 503, ErrorCode: 10023, Description: "insufficient resources"},
+	}
+	for name, failure := range cases {
+		t.Run(name, func(t *testing.T) {
+			shrinkStartRetry(t, 10*time.Second, 5*time.Millisecond)
+			_, _, js := testutil.StartTestJetStream(t)
+			forming := &formingJS{JetStream: js, err: failure}
+			forming.down.Store(4)
+
+			release, ok := AcquireLeaderAtStart(t.Context(), NewLeaderLease(kvstore.OpenSyncBucket(forming), KeyVPCDReconcile, "node-1"))
+			require.True(t, ok, "the election must be won once the cluster has formed")
+			release()
+		})
+	}
+}
+
+func TestAcquireLeaderAtStart_GivesUpWhenTheWaitRunsOut(t *testing.T) {
+	shrinkStartRetry(t, 50*time.Millisecond, 5*time.Millisecond)
+	_, _, js := testutil.StartTestJetStream(t)
+	cold := &coldStore{KVStore: kvstore.OpenSyncBucket(js)}
+	cold.down.Store(1 << 30)
+
+	start := time.Now()
+	_, ok := AcquireLeaderAtStart(t.Context(), NewLeaderLease(cold, KeyVPCDReconcile, "node-1"))
+	assert.False(t, ok)
+	assert.Less(t, time.Since(start), 5*time.Second)
+}
+
+// A held lease is an answer, not an outage, so it is not waited on.
+func TestAcquireLeaderAtStart_AHeldLeaseAnswersAtOnce(t *testing.T) {
+	shrinkStartRetry(t, time.Minute, time.Second)
+	_, _, js := testutil.StartTestJetStream(t)
+	release, ok := AcquireLeader(t.Context(), vpcdLease(js, "node-1"))
+	require.True(t, ok)
+	defer release()
+
+	start := time.Now()
+	_, ok = AcquireLeaderAtStart(t.Context(), vpcdLease(js, "node-2"))
+	assert.False(t, ok)
+	assert.Less(t, time.Since(start), 5*time.Second, "a held lease must not consume the retry window")
+}
+
+// An undeclared cluster size never becomes declared by waiting, so it ends the
+// wait at once instead of holding the startup reconcile for the whole window.
+func TestAcquireLeaderAtStart_StopsOnAConfigFault(t *testing.T) {
+	shrinkStartRetry(t, time.Minute, time.Second)
+	_, _, js := testutil.StartTestJetStream(t)
+	clustersize.RedeclareForTest(t, 0)
+
+	start := time.Now()
+	_, ok := AcquireLeaderAtStart(t.Context(), vpcdLease(js, "node-1"))
+	assert.False(t, ok)
+	assert.Less(t, time.Since(start), 10*time.Second, "a permanent fault must not consume the retry window")
 }

spinifex/domains/ochre/reaper.go

@@ -8,16 +8,18 @@ import (
 	"log/slog"
 	"time"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
 )
 
-// Sweep and lease timing. The holder refreshes well inside the bucket's TTL,
-// so a leader that dies is replaced within one TTL rather than one refresh.
+// KeyReaper is the reaper's leader lease in the sync bucket.
+const KeyReaper = "ochre.reaper"
+
+// Sweep and lease timing. A leader that dies is replaced within about one TTL.
 const (
+	reaperLeaseTTL        = 60 * time.Second
 	defaultReaperInterval = 30 * time.Second
-	defaultLeaseRefresh   = 20 * time.Second
 	defaultIdleTTL        = 15 * time.Minute // How long an endpoint must have been idle before its GPU is taken back.
 	maxScrapeFailures     = 5                // How many consecutive failed scrapes mean the serving process is wedged rather than briefly busy.
 	defaultScrapeTimeout  = 5 * time.Second  // Per-scrape budget.
@@ -28,7 +30,6 @@ const (
 type ReaperDeps struct {
 	Interval      time.Duration
 	IdleTTL       time.Duration
-	LeaseRefresh  time.Duration
 	ScrapeTimeout time.Duration
 }
 
@@ -36,26 +37,19 @@ type ReaperDeps struct {
 // does the scraping and reaping; every node keeps serving the API, so a
 // leaderless gap delays a reclaim rather than failing a request.
 type Reaper struct {
-	svc      *Service
-	holder   string
-	deps     ReaperDeps
-	lease    *kvlease.Lease
-	leaseErr error
+	svc   *Service
+	deps  ReaperDeps
+	lease *kvsync.Lease
 }
 
-// NewReaper builds the reaper for svc; holder identifies this daemon in the lease.
-func NewReaper(svc *Service, holder string, deps ReaperDeps) *Reaper {
-	r := &Reaper{svc: svc, holder: holder, deps: deps}
-	r.lease, r.leaseErr = kvlease.New(kvlease.Config{
-		Name:   "bedrock/reaper",
-		Bucket: svc.leader.KV,
-		Key:    leaderKey,
-		Holder: holder,
-		TTL:    KVBucketLeaderTTL,
-		Renew:  r.leaseRefresh(),
-		Retry:  r.leaseRefresh(),
-	})
-	return r
+// NewReaperLease returns the reaper's leader lease for holder.
+func NewReaperLease(store kvsync.KVStore, holder string) *kvsync.Lease {
+	return kvsync.NewLease(store, KeyReaper, holder, reaperLeaseTTL)
+}
+
+// NewReaper builds the reaper for svc, which leads while it holds lease.
+func NewReaper(svc *Service, lease *kvsync.Lease, deps ReaperDeps) *Reaper {
+	return &Reaper{svc: svc, deps: deps, lease: lease}
 }
 
 func (r *Reaper) interval() time.Duration {
@@ -72,13 +66,6 @@ func (r *Reaper) idleTTL() time.Duration {
 	return defaultIdleTTL
 }
 
-func (r *Reaper) leaseRefresh() time.Duration {
-	if r.deps.LeaseRefresh > 0 {
-		return r.deps.LeaseRefresh
-	}
-	return defaultLeaseRefresh
-}
-
 func (r *Reaper) scrapeTimeout() time.Duration {
 	if r.deps.ScrapeTimeout > 0 {
 		return r.deps.ScrapeTimeout
@@ -89,17 +76,13 @@ func (r *Reaper) scrapeTimeout() time.Duration {
 // Run drives the leadership and sweep loop until ctx is cancelled. Intended as
 // a daemon-boot goroutine; panics are the caller's recover concern.
 func (r *Reaper) Run(ctx context.Context) {
-	if r.leaseErr != nil {
-		slog.ErrorContext(ctx, "bedrock reaper: lease config invalid", "holder", r.holder, "err", r.leaseErr)
-		return
-	}
 	sweepTicker := time.NewTicker(r.interval())
 	defer sweepTicker.Stop()
 
 	leaseDone := make(chan struct{})
 	go func() {
 		defer close(leaseDone)
-		r.lease.Run(ctx)
+		kvsync.RunElection(ctx, r.lease, nil, nil)
 	}()
 
 	for {
@@ -112,7 +95,7 @@ func (r *Reaper) Run(ctx context.Context) {
 				continue
 			}
 			if err := r.sweepOnce(ctx); err != nil {
-				slog.ErrorContext(ctx, "bedrock reaper: sweep failed", "holder", r.holder, "err", err)
+				slog.ErrorContext(ctx, "bedrock reaper: sweep failed", "key", r.lease.Key(), "err", err)
 			}
 		}
 	}

spinifex/domains/ochre/reaper_test.go

@@ -14,6 +14,9 @@ import (
 
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -95,13 +98,31 @@ func newReaperFixture(t *testing.T, deps ReaperDeps) *reaperFixture {
 	})
 	return &reaperFixture{
 		svc:       svc,
-		reaper:    NewReaper(svc, "node-a", deps),
+		reaper:    testReaper(t, svc, "node-a", deps),
 		harness:   h,
 		transport: transport,
 		store:     svc.store,
 	}
 }
 
+// testReaper is holder's reaper on a sync bucket of its own, as a separate
+// daemon's would be.
+func testReaper(t *testing.T, svc *Service, holder string, deps ReaperDeps) *Reaper {
+	t.Helper()
+	js, err := jetstream.New(svc.nc)
+	require.NoError(t, err)
+	return NewReaper(svc, NewReaperLease(kvstore.OpenSyncBucket(js), holder), deps)
+}
+
+// lead takes r's lease, and unlocks it when the test ends.
+func lead(t *testing.T, r *Reaper) kvsync.Token {
+	t.Helper()
+	tok, err := r.lease.TryLock(t.Context())
+	require.NoError(t, err)
+	t.Cleanup(func() { r.lease.Unlock(context.Background(), tok) })
+	return tok
+}
+
 // ready launches testModelID and waits for its record to reach READY.
 func (f *reaperFixture) ready(t *testing.T) EndpointRecord {
 	t.Helper()
@@ -393,31 +414,28 @@ func TestReaper_SkipsNonReadyRecords(t *testing.T) {
 
 func TestReaper_LeaseAdmitsOneLeader(t *testing.T) {
 	f := newReaperFixture(t, ReaperDeps{})
-	other := NewReaper(f.svc, "node-b", ReaperDeps{})
+	other := testReaper(t, f.svc, "node-b", ReaperDeps{})
 
-	require.True(t, f.reaper.lease.TryAcquire(t.Context()))
-	require.False(t, other.lease.TryAcquire(t.Context()),
+	lead(t, f.reaper)
+	_, err := other.lease.TryLock(t.Context())
+	require.ErrorAs(t, err, new(*kvsync.HeldError),
 		"two nodes must never sweep the same endpoints at once")
 
 	assert.True(t, f.reaper.IsLeader())
 	assert.False(t, other.IsLeader())
-
-	// A refresh by the holder keeps it, and does not hand it away.
-	require.True(t, f.reaper.lease.TryAcquire(t.Context()))
-	assert.True(t, f.reaper.IsLeader())
 }
 
 // Releasing on shutdown is what makes a rolling restart cost seconds rather
 // than a full lease TTL of no reclaim.
 func TestReaper_ReleaseFreesTheLeaseImmediately(t *testing.T) {
 	f := newReaperFixture(t, ReaperDeps{})
-	other := NewReaper(f.svc, "node-b", ReaperDeps{})
+	other := testReaper(t, f.svc, "node-b", ReaperDeps{})
 
-	f.reaper.lease.TryAcquire(t.Context())
+	tok := lead(t, f.reaper)
 	require.True(t, f.reaper.IsLeader())
 
-	f.reaper.lease.Release(t.Context())
-	other.lease.TryAcquire(t.Context())
+	f.reaper.lease.Unlock(t.Context(), tok)
+	lead(t, other)
 	assert.True(t, other.IsLeader())
 }
 
@@ -428,8 +446,8 @@ func TestReaper_NonLeaderSweepsNothing(t *testing.T) {
 	f.ready(t)
 	f.age(t, time.Hour, time.Hour)
 
-	holder := NewReaper(f.svc, "node-b", ReaperDeps{})
-	holder.lease.TryAcquire(t.Context())
+	holder := testReaper(t, f.svc, "node-b", ReaperDeps{})
+	lead(t, holder)
 	require.True(t, holder.IsLeader())
 
 	ctx, cancel := context.WithTimeout(t.Context(), 60*time.Millisecond)
@@ -457,7 +475,6 @@ func TestEndpointRecord_LastActive(t *testing.T) {
 // full TTL rather than freeing it immediately.
 func TestReaper_RunReleasesLeaseOnShutdown(t *testing.T) {
 	f := newReaperFixture(t, ReaperDeps{Interval: time.Hour})
-	require.NoError(t, f.reaper.leaseErr)
 
 	ctx, cancel := context.WithCancel(t.Context())
 	done := make(chan struct{})
@@ -467,7 +484,45 @@ func TestReaper_RunReleasesLeaseOnShutdown(t *testing.T) {
 	cancel()
 	<-done
 
-	other := NewReaper(f.svc, "node-b", ReaperDeps{})
-	assert.True(t, other.lease.TryAcquire(t.Context()),
-		"Run returned with the lease key still present")
+	other := testReaper(t, f.svc, "node-b", ReaperDeps{})
+	tok, err := other.lease.TryLock(t.Context())
+	require.NoError(t, err, "Run returned with the lease key still present")
+	other.lease.Unlock(t.Context(), tok)
+}
+
+// A reaper whose lease is taken over must stop reclaiming at once, or two
+// nodes would tear the same endpoint down; re-elected, it reclaims again.
+func TestReaper_LosingTheLeaseStopsReclaimUntilReElection(t *testing.T) {
+	deps := ReaperDeps{Interval: 5 * time.Millisecond, IdleTTL: time.Minute}
+	f := newReaperFixture(t, deps)
+	js, err := jetstream.New(f.svc.nc)
+	require.NoError(t, err)
+	store := kvstore.OpenSyncBucket(js)
+	f.reaper = NewReaper(f.svc, kvsync.NewLease(store, KeyReaper, "node-a", 300*time.Millisecond), deps)
+	f.ready(t)
+
+	ctx, cancel := context.WithCancel(t.Context())
+	done := make(chan struct{})
+	go func() { defer close(done); f.reaper.Run(ctx) }()
+	t.Cleanup(func() { cancel(); <-done })
+	require.Eventually(t, f.reaper.IsLeader, 5*time.Second, 5*time.Millisecond)
+
+	syncKV, err := store.KV(t.Context())
+	require.NoError(t, err)
+	_, err = syncKV.Put(t.Context(), KeyReaper, []byte(`{"holder":"node-b","generation":0}`))
+	require.NoError(t, err)
+	require.Eventually(t, func() bool { return !f.reaper.IsLeader() }, 5*time.Second, 5*time.Millisecond)
+
+	// Idle from here on, so only the lost lease keeps it from being reaped.
+	f.age(t, 10*time.Minute, 10*time.Minute)
+	time.Sleep(20 * time.Millisecond) // a sweep already running when the lease went
+	scrapes := f.transport.scrapeCount()
+	time.Sleep(100 * time.Millisecond)
+	_, found := f.current(t)
+	assert.True(t, found, "a reaper that lost the lease must not reclaim")
+	assert.Equal(t, scrapes, f.transport.scrapeCount(), "a reaper that lost the lease must not even scrape")
+
+	require.NoError(t, syncKV.Delete(t.Context(), KeyReaper))
+	require.Eventually(t, func() bool { _, found := f.current(t); return !found }, 5*time.Second, 5*time.Millisecond,
+		"re-elected, the reaper reclaims the idle endpoint")
 }

spinifex/domains/ochre/service.go

@@ -63,9 +63,6 @@ type Service struct {
 	nc    *nats.Conn
 	deps  ServiceDeps
 	store *endpointStore
-	// leader is the reaper's TTL-backed lease bucket, built alongside the
-	// endpoints store so both share this Service's one JetStream client.
-	leader *kvstore.Bucket
 
 	// ensureMu collapses concurrent Ensure calls FOR THE SAME KEY on THIS
 	// replica into one KV round trip: without it, N concurrent local callers
@@ -82,15 +79,14 @@ type Service struct {
 var _ EndpointService = (*Service)(nil)
 
 // NewService constructs a Service bound to nc, using deps for launch/capacity.
-// Both buckets are resolved lazily but memoised, so the get-or-create round
+// The endpoints bucket is resolved lazily but memoised, so the get-or-create round
 // trip is paid once for the daemon rather than once per KV call.
 func NewService(nc *nats.Conn, deps ServiceDeps) *Service {
 	js := serviceJetStream(nc)
 	return &Service{
-		nc:     nc,
-		deps:   deps,
-		store:  newEndpointStore(js),
-		leader: newLeaderBucket(js),
+		nc:    nc,
+		deps:  deps,
+		store: newEndpointStore(js),
 	}
 }
 

spinifex/domains/ochre/store.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/base64"
 	"errors"
-	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
@@ -21,30 +20,10 @@ const KVBucketEndpoints = "bedrock-endpoints"
 // not meaningful to retain, only the current one CAS-guards against races.
 const KVBucketEndpointsHistory = 1
 
-// KVBucketLeader holds the reaper's leader lease. Its TTL is what makes the
-// lease self-healing: a leader that dies without releasing it is replaced
-// within one TTL rather than never.
-const (
-	KVBucketLeader    = "bedrock-leader"
-	KVBucketLeaderTTL = 60 * time.Second
-	leaderKey         = "reaper"
-)
-
 // missingJetStream is the error every accessor reports when the Service was
 // built without a NATS connection to derive a JetStream client from.
 const missingJetStream = "bedrock service: nil nats connection"
 
-// newLeaderBucket returns the reaper's TTL-backed leader bucket. The lease
-// carries no value of its own, so it is a Bucket rather than a Store[T].
-func newLeaderBucket(js jetstream.JetStream) *kvstore.Bucket {
-	return kvstore.NewBucket(js, kvstore.Config{
-		Name:    KVBucketLeader,
-		History: 1,
-		TTL:     KVBucketLeaderTTL,
-		Missing: missingJetStream,
-	})
-}
-
 // EndpointKey returns the KV key for accountID's modelID endpoint record.
 // Model IDs contain ':' (e.g. "meta.llama3-2-1b-instruct-v1:0"), which NATS
 // rejects in a KV key, so the segment is base64url-encoded, mirroring

spinifex/domains/rds/instance/backup_test.go

@@ -823,7 +823,7 @@ func TestReconciler_RunsTheBackupWindowOnItsAccountPass(t *testing.T) {
 	rec.Agent = AgentState{InstanceID: testInstance, EngineHealth: EngineHealthHealthy, LastSeen: &now}
 	seedInstance(t, h.svc, rec)
 
-	require.NoError(t, onePass(t, NewReconciler(h.svc, "node-a")))
+	require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-a")))
 
 	assert.Len(t, h.snaps.created, 1)
 	assert.Len(t, h.automatedStamps(t, testDBID), 1)
@@ -840,7 +840,8 @@ func TestReconciler_ClusterLeaseFollowsLeadership(t *testing.T) {
 	require.NotNil(t, release)
 	assert.False(t, ok, "a node that does not hold the lease does not sweep")
 
-	require.True(t, h.rec.lease.TryAcquire(t.Context()))
+	_, err := lead(t, h.rec)
+	require.NoError(t, err)
 	release, ok = h.rec.AcquireClusterLease()
 	require.True(t, ok)
 

spinifex/domains/rds/instance/delete_test.go

@@ -202,7 +202,7 @@ func TestReconciler_AdoptsAnInterruptedFinalSnapshot(t *testing.T) {
 		},
 	}
 
-	require.NoError(t, onePass(t, NewReconciler(h.svc, "node-a")))
+	require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-a")))
 
 	assert.Empty(t, h.snaps.created, "the resumed teardown must adopt the cut snapshot, not create another")
 	stored, found := h.snapshotRecord(t, rec.FinalSnapshotIdentifier)
@@ -328,7 +328,7 @@ func TestReconciler_IsIdempotentAcrossARetriedTeardown(t *testing.T) {
 	rec := abandonedDelete(availableRecord())
 	rec.FinalSnapshotIdentifier = "orders-db-final"
 	seedInstance(t, h.svc, rec)
-	reconciler := NewReconciler(h.svc, "node-a")
+	reconciler := testReconciler(t, h.svc, "node-a")
 	require.NoError(t, onePass(t, reconciler))
 
 	// The record left behind by the interrupted attempt, replayed.
@@ -346,7 +346,7 @@ func TestReconciler_ResumesAnInterruptedDelete(t *testing.T) {
 	h := newLifecycleHarness(t, false)
 	seedInstance(t, h.svc, abandonedDelete(availableRecord()))
 
-	reconciler := NewReconciler(h.svc, "node-a")
+	reconciler := testReconciler(t, h.svc, "node-a")
 	require.NoError(t, onePass(t, reconciler))
 
 	assert.Equal(t, []string{testInstance}, h.launcher.terminated)
@@ -364,7 +364,7 @@ func TestReconciler_LeavesADeleteItsHandlerMayStillBeRunning(t *testing.T) {
 	rec.TransitionStartedAt = &started
 	seedInstance(t, h.svc, rec)
 
-	require.NoError(t, onePass(t, NewReconciler(h.svc, "node-a")))
+	require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-a")))
 
 	assert.Empty(t, h.launcher.terminated)
 	assert.True(t, h.recordExists(t, testDBID))
@@ -443,7 +443,7 @@ func TestReconciler_ResumesAnInterruptedStop(t *testing.T) {
 	rec.Status = StatusStopping
 	seedInstance(t, h.svc, rec)
 
-	reconciler := NewReconciler(h.svc, "node-a")
+	reconciler := testReconciler(t, h.svc, "node-a")
 	require.NoError(t, onePass(t, reconciler))
 
 	assert.Equal(t, []string{"stop:" + testInstance}, h.cmdr.calls)

spinifex/domains/rds/instance/lifecycle_test.go

@@ -578,7 +578,7 @@ func TestReconciler_DoesNotCallAStillRunningVMStopped(t *testing.T) {
 	rec.TransitionStartedAt = &started
 	seedInstance(t, h.svc, rec)
 
-	require.NoError(t, onePass(t, NewReconciler(h.svc, "node-a")))
+	require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-a")))
 
 	// Left stopping so the next pass retries; the bound is what ends it.
 	assert.Equal(t, StatusStopping, h.record(t).Status)
@@ -598,7 +598,7 @@ func TestReconciler_ResumesAStopTheLiveCallerIsAlreadyDriving(t *testing.T) {
 	rec.TransitionStartedAt = &started
 	seedInstance(t, h.svc, rec)
 
-	require.NoError(t, onePass(t, NewReconciler(h.svc, "node-a")))
+	require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-a")))
 
 	assert.Equal(t, StatusStopped, h.record(t).Status)
 }
@@ -770,7 +770,7 @@ func TestReconciler_CompletesARestartOnAHeartbeatFromTheRestartedEngine(t *testi
 			now := time.Now().UTC()
 			seedInstance(t, h.svc, restartingRecord(status, now.Add(-time.Minute), now))
 
-			require.NoError(t, onePass(t, NewReconciler(h.svc, "node-a")))
+			require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-a")))
 
 			rec := h.record(t)
 			assert.Equal(t, StatusAvailable, rec.Status)
@@ -789,7 +789,7 @@ func TestReconciler_CompletesARestartOnAPersistedHeartbeatInsideTheFloor(t *test
 	persisted := time.Now().UTC().Add(-HeartbeatStaleAfter - time.Minute)
 	seedInstance(t, h.svc, restartingRecord(StatusRebooting, persisted.Add(-time.Second), persisted))
 
-	require.NoError(t, onePass(t, NewReconciler(h.svc, "node-a")))
+	require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-a")))
 
 	assert.Equal(t, StatusAvailable, h.record(t).Status)
 }
@@ -803,7 +803,7 @@ func TestReconciler_IgnoresAHeartbeatPredatingTheRestart(t *testing.T) {
 	now := time.Now().UTC()
 	seedInstance(t, h.svc, restartingRecord(StatusRebooting, now, now.Add(-time.Minute)))
 
-	require.NoError(t, onePass(t, NewReconciler(h.svc, "node-a")))
+	require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-a")))
 
 	assert.Equal(t, StatusRebooting, h.record(t).Status)
 }
@@ -821,7 +821,7 @@ func TestReconciler_IgnoresAHeartbeatFromTheAgentTheRestartReplaces(t *testing.T
 	rec.Agent.StartedAt = &predecessor
 	seedInstance(t, h.svc, rec)
 
-	require.NoError(t, onePass(t, NewReconciler(h.svc, "node-a")))
+	require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-a")))
 	assert.Equal(t, StatusRebooting, h.record(t).Status)
 
 	restarted := started.Add(time.Second)
@@ -829,7 +829,7 @@ func TestReconciler_IgnoresAHeartbeatFromTheAgentTheRestartReplaces(t *testing.T
 	rec.Agent.StartedAt = &restarted
 	seedInstance(t, h.svc, rec)
 
-	require.NoError(t, onePass(t, NewReconciler(h.svc, "node-a")))
+	require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-a")))
 	assert.Equal(t, StatusAvailable, h.record(t).Status)
 }
 
@@ -842,7 +842,7 @@ func TestReconciler_MarksFailedWhenARestartOverrunsItsBound(t *testing.T) {
 	started := now.Add(-2 * transitionTimeout)
 	seedInstance(t, h.svc, restartingRecord(StatusStarting, started, started.Add(-time.Minute)))
 
-	require.NoError(t, onePass(t, NewReconciler(h.svc, "node-a")))
+	require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-a")))
 
 	rec := h.record(t)
 	assert.Equal(t, StatusFailed, rec.Status)
@@ -861,7 +861,7 @@ func TestReconciler_CarriesTheAgentsReasonIntoAFailedRestart(t *testing.T) {
 		`FATAL: invalid value for parameter "max_connections"`
 	seedInstance(t, h.svc, rec)
 
-	require.NoError(t, onePass(t, NewReconciler(h.svc, "node-a")))
+	require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-a")))
 
 	stored := h.record(t)
 	assert.Equal(t, StatusFailed, stored.Status)

spinifex/domains/rds/instance/modify_test.go

@@ -72,7 +72,7 @@ func newModifyHarnessWithAgent(t *testing.T, agentFails bool) *modifyHarness {
 		ServingCertKeyBits: testServingCertKeyBits,
 		ApplyParamsTimeout: testApplyParamsTimeout,
 	})
-	h.rec = NewReconciler(h.svc, "node-a")
+	h.rec = testReconciler(t, h.svc, "node-a")
 	return h
 }
 

spinifex/domains/rds/instance/reconciler.go

@@ -13,17 +13,19 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/reconciler"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/nats-io/nats.go/jetstream"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 )
 
-// Lease and sweep timing. The holder refreshes well inside the bucket's 60s TTL,
-// so a leader that dies is replaced within one TTL rather than one refresh.
+// KeyReconciler is the RDS reconciler's leader lease in the sync bucket.
+const KeyReconciler = "rds.reconciler"
+
+// Lease and sweep timing. A follower asks again at the lease's own renewal
+// rate, so a leader that dies is replaced within about one TTL.
 const (
-	reconcilerLeaderKey = "reconciler"
-	leaseRefresh        = 20 * time.Second
-	reconcileInterval   = 15 * time.Second
+	reconcilerLeaseTTL = 60 * time.Second
+	leaseRefresh       = reconcilerLeaseTTL / 3
+	reconcileInterval  = 15 * time.Second
 
 	// The drift backstop, and so also the cadence on which a backup or
 	// maintenance window is noticed. A window is a half-hour slot, so noticing
@@ -81,11 +83,8 @@ type InstanceStateResolver interface {
 // the failure classifier that gives a settled instance an honest health state.
 // The backup sweep extends the same loop.
 type Reconciler struct {
-	svc    *Service
-	holder string
-
-	lease    *kvlease.Lease
-	leaseErr error
+	svc   *Service
+	lease *kvsync.Lease
 
 	// The payloads already reported as stuck pending. The condition persists
 	// until an operator acts, so without this the event would be re-recorded
@@ -100,34 +99,24 @@ type Reconciler struct {
 	systemSGID string
 }
 
-// NewReconciler returns a Reconciler for svc; holder identifies this daemon in the lease. A bad lease
-// config is kept, and Run logs it and returns.
-func NewReconciler(svc *Service, holder string) *Reconciler {
-	r := &Reconciler{svc: svc, holder: holder, reportedPending: make(map[string]string)}
-	r.lease, r.leaseErr = kvlease.New(kvlease.Config{
-		Name:   "rds/reconciler",
-		Bucket: r.leaderBucket,
-		Key:    reconcilerLeaderKey,
-		Holder: holder,
-		TTL:    KVBucketRDSLeaderTTL,
-		Renew:  leaseRefresh,
-		Retry:  leaseRefresh,
-	})
-	return r
+// NewReconcilerLease returns the RDS reconciler's leader lease for holder.
+func NewReconcilerLease(store kvsync.KVStore, holder string) *kvsync.Lease {
+	return kvsync.NewLease(store, KeyReconciler, holder, reconcilerLeaseTTL)
+}
+
+// NewReconciler returns a Reconciler for svc that leads while it holds lease.
+func NewReconciler(svc *Service, lease *kvsync.Lease) *Reconciler {
+	return &Reconciler{svc: svc, lease: lease, reportedPending: make(map[string]string)}
 }
 
 // Run drives the leadership and reconcile loop until ctx is canceled. Intended as a daemon-boot
 // goroutine; panics are the caller's recover concern.
 func (r *Reconciler) Run(ctx context.Context) {
-	if r.leaseErr != nil {
-		slog.ErrorContext(ctx, "rds reconciler: lease config invalid", "holder", r.holder, "err", r.leaseErr)
-		return
-	}
 	leadershipCtx, cancelLeadership := context.WithCancel(ctx)
 	leadershipDone := make(chan struct{})
 	go func() {
 		defer close(leadershipDone)
-		r.lease.Run(leadershipCtx)
+		kvsync.RunElection(leadershipCtx, r.lease, nil, nil)
 	}()
 	defer func() {
 		cancelLeadership()
@@ -162,7 +151,7 @@ func (r *Reconciler) reconcilePass(ctx context.Context) (time.Duration, error) {
 	}
 	revisit, err := r.reconcileOnce(ctx)
 	if err != nil {
-		slog.ErrorContext(ctx, "rds reconciler: pass failed", "holder", r.holder, "err", err)
+		slog.ErrorContext(ctx, "rds reconciler: pass failed", "key", r.lease.Key(), "err", err)
 	}
 	// Already logged, so it is not returned again: the loop would log it a
 	// second time under its own message.
@@ -176,14 +165,6 @@ func (r *Reconciler) AcquireClusterLease() (func(), bool) {
 	return func() {}, r.lease.Held()
 }
 
-func (r *Reconciler) leaderBucket(ctx context.Context) (jetstream.KeyValue, error) {
-	js, err := r.svc.js()
-	if err != nil {
-		return nil, err
-	}
-	return InitLeaderBucket(ctx, js)
-}
-
 // One pass across every tenant. A bucket that cannot be read stops the pass
 // with an error rather than being skipped silently, so a partial view shows up
 // in the log instead of looking like a fleet with nothing to do.

spinifex/domains/rds/instance/reconciler_test.go

@@ -10,11 +10,32 @@ import (
 	"github.com/aws/aws-sdk-go/service/rds"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+// testReconciler is holder's reconciler on a sync bucket of its own, as a
+// separate daemon's would be.
+func testReconciler(t *testing.T, svc *Service, holder string) *Reconciler {
+	t.Helper()
+	js, err := svc.js()
+	require.NoError(t, err)
+	return NewReconciler(svc, NewReconcilerLease(kvstore.OpenSyncBucket(js), holder))
+}
+
+// lead takes r's lease, and unlocks it when the test ends.
+func lead(t *testing.T, r *Reconciler) (kvsync.Token, error) {
+	t.Helper()
+	tok, err := r.lease.TryLock(t.Context())
+	if err == nil {
+		t.Cleanup(func() { r.lease.Unlock(context.Background(), tok) })
+	}
+	return tok, err
+}
+
 // onePass runs one reconcile and keeps only its error, for the tests that assert
 // on what converged rather than on when the loop should next run. The tests that
 // do care about the deadline call reconcileOnce directly.
@@ -70,7 +91,7 @@ func newReconcileHarness(t *testing.T, opts ...func(*Deps)) *reconcileHarness {
 		opt(&deps)
 	}
 	svc := NewService(nc, testRegion).WithDeps(deps)
-	return &reconcileHarness{svc: svc, rec: NewReconciler(svc, "node-a"), state: state, nc: nc}
+	return &reconcileHarness{svc: svc, rec: testReconciler(t, svc, "node-a"), state: state, nc: nc}
 }
 
 // healthyRecord is a creating instance whose agent has just reported a healthy
@@ -377,18 +398,21 @@ func TestReconciler_RetriesTheSystemNICMoveAfterAFailure(t *testing.T) {
 func TestReconciler_ElectsASingleLeader(t *testing.T) {
 	t.Parallel()
 	h := newReconcileHarness(t)
-	other := NewReconciler(h.svc, "node-b")
+	other := testReconciler(t, h.svc, "node-b")
 
-	assert.True(t, h.rec.lease.TryAcquire(t.Context()))
-	assert.False(t, other.lease.TryAcquire(t.Context()))
-
-	// The holder reports the leasership it already holds, and does not hand it away.
-	assert.True(t, h.rec.lease.TryAcquire(t.Context()))
+	tok, err := lead(t, h.rec)
+	require.NoError(t, err)
+	_, err = lead(t, other)
+	assert.ErrorAs(t, err, new(*kvsync.HeldError))
+	assert.True(t, h.rec.lease.Held())
+	assert.False(t, other.lease.Held())
 
 	// Releasing on shutdown hands over immediately instead of after the TTL.
-	h.rec.lease.Release(t.Context())
-	assert.True(t, other.lease.TryAcquire(t.Context()))
-	assert.False(t, h.rec.lease.TryAcquire(t.Context()))
+	h.rec.lease.Unlock(t.Context(), tok)
+	_, err = lead(t, other)
+	require.NoError(t, err)
+	_, err = lead(t, h.rec)
+	assert.ErrorAs(t, err, new(*kvsync.HeldError))
 }
 
 func TestReconciler_RunHoldsLeadershipIndependentlyOfTheReconcileLoop(t *testing.T) {
@@ -404,21 +428,23 @@ func TestReconciler_RunHoldsLeadershipIndependentlyOfTheReconcileLoop(t *testing
 	cancel()
 	<-done
 
-	other := NewReconciler(h.svc, "node-b")
-	assert.True(t, other.lease.TryAcquire(t.Context()),
-		"Run returned with the lease key still present")
+	other := testReconciler(t, h.svc, "node-b")
+	_, err := lead(t, other)
+	assert.NoError(t, err, "Run returned with the lease key still present")
 }
 
 // A node that never won the lease must not delete the holder's key on shutdown.
 func TestReconciler_RelinquishOnlyReleasesItsOwnLease(t *testing.T) {
 	t.Parallel()
 	h := newReconcileHarness(t)
-	other := NewReconciler(h.svc, "node-b")
+	other := testReconciler(t, h.svc, "node-b")
 
-	require.True(t, h.rec.lease.TryAcquire(t.Context()))
-	other.lease.Release(t.Context())
+	tok, err := lead(t, h.rec)
+	require.NoError(t, err)
+	other.lease.Unlock(t.Context(), tok)
 
-	assert.False(t, other.lease.TryAcquire(t.Context()), "the original holder still owns the lease")
+	_, err = lead(t, other)
+	assert.ErrorAs(t, err, new(*kvsync.HeldError), "the original holder still owns the lease")
 }
 
 // An overrun modify failed on a fresh read must not fail a delete accepted since
@@ -440,3 +466,46 @@ func TestTransitionFresh_LeavesAnAcceptedDelete(t *testing.T) {
 	require.True(t, found)
 	assert.Equal(t, StatusDeleting, stored.Status)
 }
+
+// The retention sweep deletes customer snapshots under AcquireClusterLease, so
+// a node whose lease was taken over must stop sweeping and reconciling at once,
+// and only start again once it is re-elected.
+func TestReconciler_ATakenOverLeaseStopsTheSweepUntilReElection(t *testing.T) {
+	t.Parallel()
+	h := newReconcileHarness(t)
+	js, err := h.svc.js()
+	require.NoError(t, err)
+	store := kvstore.OpenSyncBucket(js)
+	h.rec.lease = kvsync.NewLease(store, KeyReconciler, "node-a", 300*time.Millisecond)
+	seedInstance(t, h.svc, healthyRecord())
+
+	_, err = lead(t, h.rec)
+	require.NoError(t, err)
+	_, ok := h.rec.AcquireClusterLease()
+	require.True(t, ok)
+
+	syncKV, err := store.KV(t.Context())
+	require.NoError(t, err)
+	_, err = syncKV.Put(t.Context(), KeyReconciler, []byte(`{"holder":"node-b","generation":0}`))
+	require.NoError(t, err)
+	require.Eventually(t, func() bool { return !h.rec.lease.Held() }, 5*time.Second, 10*time.Millisecond,
+		"the takeover must cost this node the lease")
+
+	_, ok = h.rec.AcquireClusterLease()
+	assert.False(t, ok, "a node that lost the lease must not sweep snapshots")
+	revisit, err := h.rec.reconcilePass(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, leaseRefresh, revisit)
+	status, _ := h.statusOf(t, testDBID)
+	assert.Equal(t, StatusCreating, status, "a node that lost the lease must not reconcile")
+
+	require.NoError(t, syncKV.Delete(t.Context(), KeyReconciler))
+	_, err = lead(t, h.rec)
+	require.NoError(t, err, "re-election")
+	_, ok = h.rec.AcquireClusterLease()
+	assert.True(t, ok)
+	_, err = h.rec.reconcilePass(t.Context())
+	require.NoError(t, err)
+	status, _ = h.statusOf(t, testDBID)
+	assert.NotEqual(t, StatusCreating, status, "the re-elected node reconciles again")
+}

spinifex/domains/rds/instance/recovery_test.go

@@ -211,7 +211,7 @@ func TestReconciler_FailsOnAClockStampedByAnEarlierLeader(t *testing.T) {
 	seedInstance(t, h.svc, rec)
 
 	// A different node holding the lease, with no beats of its own in memory.
-	other := NewReconciler(h.svc, "node-b")
+	other := testReconciler(t, h.svc, "node-b")
 	require.NoError(t, onePass(t, other))
 
 	stored := h.recordOf(t, testDBID)

spinifex/domains/rds/instance/snapshot_test.go

@@ -568,7 +568,7 @@ func TestProjectDBSnapshot_ReportsProgressOnlyWhenTheDataExists(t *testing.T) {
 func TestReconciler_AdoptsTheEC2SnapshotOfAnUnfinishedDBSnapshot(t *testing.T) {
 	t.Parallel()
 	h := newSnapshotHarness(t, false)
-	rec := NewReconciler(h.svc, "node-a")
+	rec := testReconciler(t, h.svc, "node-a")
 
 	kv, err := h.svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
@@ -601,7 +601,7 @@ func TestReconciler_AdoptsTheEC2SnapshotOfAnUnfinishedDBSnapshot(t *testing.T) {
 func TestReconciler_DoesNotAdoptAnotherAccountsSnapshot(t *testing.T) {
 	t.Parallel()
 	h := newSnapshotHarness(t, false)
-	reconciler := NewReconciler(h.svc, "node-a")
+	reconciler := testReconciler(t, h.svc, "node-a")
 
 	kv, err := h.svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
@@ -636,7 +636,7 @@ func TestReconciler_KeepsCreatingSnapshotWhenEC2LookupIsIncomplete(t *testing.T)
 	t.Parallel()
 	h := newSnapshotHarness(t, false)
 	h.snaps.describeErr = errors.New("snapshot metadata temporarily unavailable")
-	reconciler := NewReconciler(h.svc, "node-a")
+	reconciler := testReconciler(t, h.svc, "node-a")
 
 	kv, err := h.svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
@@ -660,7 +660,7 @@ func TestReconciler_KeepsCreatingSnapshotWhenEC2LookupIsIncomplete(t *testing.T)
 func TestReconciler_DoesNotDeleteAConcurrentlyCompletedSnapshot(t *testing.T) {
 	t.Parallel()
 	h := newSnapshotHarness(t, false)
-	reconciler := NewReconciler(h.svc, "node-a")
+	reconciler := testReconciler(t, h.svc, "node-a")
 
 	kv, err := h.svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
@@ -693,7 +693,7 @@ func TestReconciler_DoesNotDeleteAConcurrentlyCompletedSnapshot(t *testing.T) {
 func TestReconciler_RejectsAmbiguousEC2Snapshots(t *testing.T) {
 	t.Parallel()
 	h := newSnapshotHarness(t, false)
-	reconciler := NewReconciler(h.svc, "node-a")
+	reconciler := testReconciler(t, h.svc, "node-a")
 
 	kv, err := h.svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
@@ -728,7 +728,7 @@ func TestReconciler_RejectsAmbiguousEC2Snapshots(t *testing.T) {
 func TestReconciler_WithdrawsADBSnapshotWhoseDataWasNeverCut(t *testing.T) {
 	t.Parallel()
 	h := newSnapshotHarness(t, false)
-	rec := NewReconciler(h.svc, "node-a")
+	rec := testReconciler(t, h.svc, "node-a")
 
 	kv, err := h.svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
@@ -753,7 +753,7 @@ func TestReconciler_WithdrawsADBSnapshotWhoseDataWasNeverCut(t *testing.T) {
 func TestReconciler_LeavesAFreshCreatingSnapshotAlone(t *testing.T) {
 	t.Parallel()
 	h := newSnapshotHarness(t, false)
-	rec := NewReconciler(h.svc, "node-a")
+	rec := testReconciler(t, h.svc, "node-a")
 
 	kv, err := h.svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
@@ -780,7 +780,7 @@ func TestReconciler_ReturnsAnInstanceStuckInBackingUp(t *testing.T) {
 		t.Run(string(resume), func(t *testing.T) {
 			t.Parallel()
 			h := newSnapshotHarness(t, false)
-			reconciler := NewReconciler(h.svc, "node-a")
+			reconciler := testReconciler(t, h.svc, "node-a")
 
 			started := time.Now().UTC().Add(-2 * transitionTimeout)
 			rec := availableRecord()
@@ -807,7 +807,7 @@ func TestReconciler_ReturnsAnInstanceStuckInBackingUp(t *testing.T) {
 func TestReconciler_LeavesAnInFlightSnapshotAlone(t *testing.T) {
 	t.Parallel()
 	h := newSnapshotHarness(t, false)
-	reconciler := NewReconciler(h.svc, "node-a")
+	reconciler := testReconciler(t, h.svc, "node-a")
 
 	started := time.Now().UTC()
 	rec := availableRecord()

spinifex/domains/rds/instance/store.go

@@ -9,9 +9,7 @@ import (
 	"errors"
 	"fmt"
 	"strings"
-	"time"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
@@ -29,13 +27,6 @@ const (
 	KVBucketRDSSystem        = "rds-system"
 	KVBucketRDSSystemVersion = 1
 	KVBucketRDSSystemHistory = 1
-
-	// "spinifex-rds-leader" holds the single reconciler lease. Its TTL expires a
-	// lease whose holder died mid-cycle, so control work resumes without an
-	// operator.
-	KVBucketRDSLeader        = "spinifex-rds-leader"
-	KVBucketRDSLeaderVersion = 1
-	KVBucketRDSLeaderTTL     = 60 * time.Second
 )
 
 // Key-path helpers for the per-account bucket:
@@ -230,16 +221,6 @@ func GetOrCreateSystemBucket(ctx context.Context, js jetstream.JetStream) (jetst
 	return kv, nil
 }
 
-// InitLeaderBucket creates (or attaches to) the shared spinifex-rds-leader
-// bucket used for the reconciler's leader-lease CAS lock.
-func InitLeaderBucket(ctx context.Context, js jetstream.JetStream) (jetstream.KeyValue, error) {
-	return kvlease.OpenBucket(ctx, js, kvlease.BucketConfig{
-		Name:    KVBucketRDSLeader,
-		TTL:     KVBucketRDSLeaderTTL,
-		Version: KVBucketRDSLeaderVersion,
-	})
-}
-
 // AccountBucketNames lists every RDS per-account bucket in the cluster. The reconciler and the DNS
 // desired-set both need a cross-tenant view, which only this provides.
 func AccountBucketNames(ctx context.Context, js jetstream.JetStream) ([]string, error) {

spinifex/domains/route53/realizer.go

@@ -39,8 +39,9 @@ type Realizer struct {
 	store *Store
 	pub   Publisher
 
-	// Elect wins this pass for the calling node; nil means always.
-	Elect func(ctx context.Context) (release func(), ok bool)
+	// Elect wins this pass for the calling node; nil means always. A lost
+	// election's revisit is the pass's own.
+	Elect func(ctx context.Context) (release func(), revisit time.Duration, ok bool)
 	// Reserved reports platform zone names, which are never written or deleted.
 	Reserved func(name string) bool
 	// Now is the clock; tests replace it.
@@ -75,9 +76,9 @@ func (r *Realizer) Run(ctx context.Context) {
 // a prompt revisit, since nothing in KV changes when it does.
 func (r *Realizer) Reconcile(ctx context.Context) (time.Duration, error) {
 	if r.Elect != nil {
-		release, ok := r.Elect(ctx)
+		release, revisit, ok := r.Elect(ctx)
 		if !ok {
-			return 0, nil
+			return revisit, nil
 		}
 		defer release()
 	}

spinifex/domains/route53/realizer_test.go

@@ -279,13 +279,14 @@ func TestRealizerDoesNothingWhenNotElected(t *testing.T) {
 	svc := newService(t)
 	ns := newFakeNorthstar(1)
 	r := newRealizer(svc, ns)
-	r.Elect = func(context.Context) (func(), bool) { return nil, false }
+	r.Elect = func(context.Context) (func(), time.Duration, bool) { return nil, 61 * time.Second, false }
 	id := createZone(t, svc, acctA, "example.com")
 	setState(t, svc, acctA, id, route53.StateActive)
 
-	_, err := r.Reconcile(ctx)
+	revisit, err := r.Reconcile(ctx)
 	require.NoError(t, err)
 	assert.False(t, ns.has("example.com"))
+	assert.Equal(t, 61*time.Second, revisit, "a lost election's revisit is the pass's, so a dead leader is replaced")
 }
 
 func TestRealizerRendersEveryRecordType(t *testing.T) {

spinifex/domains/route53/store.go

@@ -14,10 +14,10 @@ import (
 const (
 	KVBucketRoute53        = "spinifex-route53"
 	KVBucketRoute53Version = 1
-	// KVBucketRoute53Realize elects the one node that publishes zones.
-	KVBucketRoute53Realize = "spinifex-route53-realize"
-	// KVBucketRoute53Verify elects the one node that checks delegations.
-	KVBucketRoute53Verify = "spinifex-route53-verify"
+	// KeyRealize is the sync-bucket lease electing the node that publishes zones.
+	KeyRealize = "route53.realize"
+	// KeyVerify is the sync-bucket lease electing the node that checks delegations.
+	KeyVerify = "route53.verify"
 
 	keyPrefixZone     = "zone."
 	keyPrefixRRSets   = "rrsets."

spinifex/domains/route53/verifier.go

@@ -45,8 +45,9 @@ type Verifier struct {
 	Platform func(ctx context.Context, name string) (d Delegation, ok bool, err error)
 	// Public asks the parent's authoritative servers, without recursion.
 	Public func(ctx context.Context, name string) (Delegation, error)
-	// Elect wins this pass for the calling node; nil means always.
-	Elect func(ctx context.Context) (release func(), ok bool)
+	// Elect wins this pass for the calling node; nil means always. A lost
+	// election's revisit is the pass's own.
+	Elect func(ctx context.Context) (release func(), revisit time.Duration, ok bool)
 	Now   func() time.Time
 	// Interval is the pass period; zero means one minute.
 	Interval time.Duration
@@ -72,9 +73,9 @@ func (v *Verifier) Run(ctx context.Context) {
 // Reconcile checks every zone that is due.
 func (v *Verifier) Reconcile(ctx context.Context) (time.Duration, error) {
 	if v.Elect != nil {
-		release, ok := v.Elect(ctx)
+		release, revisit, ok := v.Elect(ctx)
 		if !ok {
-			return 0, nil
+			return revisit, nil
 		}
 		defer release()
 	}

spinifex/domains/route53/verifier_test.go

@@ -12,6 +12,8 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/mulgadc/spinifex/internal/testkit"
+	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
 	"github.com/mulgadc/spinifex/spinifex/domains/route53"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 )
@@ -330,3 +332,35 @@ func TestDuplicateActiveZoneReturnsToPendingAndItsChangesSettle(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, "INSYNC", changeStatus(t, svc, acctB, aws.StringValue(out.ChangeInfo.Id)))
 }
+
+// Delegation checks are elected: while a peer holds the verify lease this node
+// looks nothing up and changes no zone, and it asks to come back once a dead
+// holder's lease could be taken over.
+func TestVerifierChecksNothingWhileAPeerHoldsTheLease(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	svc := route53.NewService(route53.NewStore(js))
+	svc.Nameservers = ours
+	dns := &fakeDNS{parents: map[string][]string{"example.com": ours()}}
+	v := route53.NewVerifier(svc.Store())
+	v.Public = dns.lookup
+	store := kvstore.OpenSyncBucket(js)
+	lease := reconcilelock.NewLeaderLease(store, route53.KeyVerify, "node-a")
+	v.Elect = func(ctx context.Context) (func(), time.Duration, bool) { return reconcilelock.ElectPass(ctx, lease) }
+	id := createZone(t, svc, acctA, "example.com")
+
+	syncKV, err := store.KV(ctx)
+	require.NoError(t, err)
+	_, err = syncKV.Put(ctx, route53.KeyVerify, []byte(`{"holder":"node-b","generation":1}`))
+	require.NoError(t, err)
+
+	revisit, err := v.Reconcile(ctx)
+	require.NoError(t, err)
+	assert.Zero(t, dns.lookups, "a node that does not hold the lease must not check delegations")
+	assert.Equal(t, route53.StatePendingNameservers, zone(t, svc, acctA, id).State)
+	assert.Positive(t, revisit, "a held pass comes back so a dead holder is replaced")
+	assert.LessOrEqual(t, revisit, reconcilelock.LeaderTTL/3, "the retry comes within a renewal interval")
+
+	require.NoError(t, syncKV.Delete(ctx, route53.KeyVerify))
+	pass(t, v)
+	assert.Equal(t, route53.StateActive, zone(t, svc, acctA, id).State, "elected, the node checks again")
+}

spinifex/foundation/state/kvlease/bucket.go

@@ -1,119 +0,0 @@
-package kvlease
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"log/slog"
-	"time"
-
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
-	telemetry "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
-	"github.com/nats-io/nats.go"
-	"github.com/nats-io/nats.go/jetstream"
-)
-
-// Bounded wait for JetStream quorum on cold multi-node start. Vars (not consts) so tests can shrink them.
-var (
-	bucketRetryFor  = 60 * time.Second
-	bucketRetryStep = 1 * time.Second
-)
-
-// BucketConfig describes a lease bucket. History is not a field: a lease key has
-// no history worth keeping, so every lease bucket is created with one.
-type BucketConfig struct {
-	Name string
-
-	// TTL expires a lease its holder died without releasing, so a crashed leader
-	// does not park the key forever. It must match the lease's own Config.TTL.
-	TTL time.Duration
-
-	// Version runs the registered schema migrations for the bucket after it is
-	// opened. Zero skips them, for a bucket holding nothing but lease keys.
-	Version int
-}
-
-// OpenBucket attaches to a lease bucket, creating it only when genuinely absent,
-// and runs any schema migration registered for it.
-//
-// Attach before create, because callers re-open this on every reconcile tick:
-// CreateKeyValue against a bucket that already exists is a STREAM.CREATE the
-// meta leader answers with an error, so creating first bills one per tick.
-//
-// A leader lease is the worst thing here to lose to a single node. Nobody can
-// acquire a lease whose bucket has no quorum, so every reconciler that shares
-// it stops at once, cluster-wide, at exactly the moment a node has failed.
-func OpenBucket(ctx context.Context, js jetstream.JetStream, cfg BucketConfig) (jetstream.KeyValue, error) {
-	replicas, err := clustersize.Replicas()
-	if err != nil {
-		return nil, fmt.Errorf("kvlease: open or create lease bucket %s: %w", cfg.Name, err)
-	}
-
-	kv, err := js.KeyValue(ctx, cfg.Name)
-	switch {
-	case errors.Is(err, jetstream.ErrBucketNotFound):
-		kv, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{
-			Bucket:   cfg.Name,
-			History:  1,
-			TTL:      cfg.TTL,
-			Replicas: replicas,
-		})
-	case err == nil:
-		// A failed raise must not fail the open. This runs on every reconcile
-		// tick, so refusing here would stop every reconciler sharing the bucket
-		// — the outage this replica count exists to prevent — over a bucket that
-		// is present and quorate at the count it already has.
-		kvstore.TryRaiseBucketReplicas(ctx, js, cfg.Name, replicas)
-	}
-	if err != nil {
-		return nil, fmt.Errorf("kvlease: open or create lease bucket %s: %w", cfg.Name, err)
-	}
-	if cfg.Version > 0 {
-		if err := migrate.DefaultRegistry.RunKV(ctx, cfg.Name, kv, cfg.Version); err != nil {
-			return nil, fmt.Errorf("migrate %s: %w", cfg.Name, err)
-		}
-	}
-	slog.DebugContext(ctx, "kvlease: lease bucket ready", "bucket", cfg.Name, "ttl_ms", telemetry.Millis(cfg.TTL))
-	return kv, nil
-}
-
-// NATSBucket is OpenBucket as a BucketFunc, retrying while JetStream is still
-// forming quorum on a cold multi-node start. Callers with their own bucket
-// initialiser should pass that instead.
-func NATSBucket(nc *nats.Conn, bucket string, ttl time.Duration) BucketFunc {
-	return func(ctx context.Context) (jetstream.KeyValue, error) {
-		js, err := jetstream.New(nc)
-		if err != nil {
-			return nil, fmt.Errorf("kvlease: JetStream unavailable: %w", err)
-		}
-		deadline := time.Now().Add(bucketRetryFor)
-		for {
-			kv, err := OpenBucket(ctx, js, BucketConfig{Name: bucket, TTL: ttl})
-			if err == nil {
-				return kv, nil
-			}
-			// An undeclared cluster size does not become declared by waiting, so
-			// it surfaces now rather than after the whole retry window.
-			if clustersize.Permanent(err) {
-				return nil, err
-			}
-			if time.Now().After(deadline) {
-				return nil, fmt.Errorf("kvlease: KV bucket %q unreachable after %s: %w", bucket, bucketRetryFor, err)
-			}
-			// A shutdown mid-wait must not sit out the remaining retry window.
-			select {
-			case <-ctx.Done():
-				return nil, ctx.Err()
-			case <-time.After(bucketRetryStep):
-			}
-		}
-	}
-}
-
-// StaticBucket adapts an already-open bucket to a BucketFunc, for callers handed
-// a KeyValue rather than opening one themselves.
-func StaticBucket(kv jetstream.KeyValue) BucketFunc {
-	return func(context.Context) (jetstream.KeyValue, error) { return kv, nil }
-}

spinifex/foundation/state/kvlease/bucket_test.go

@@ -1,76 +0,0 @@
-package kvlease_test
-
-import (
-	"testing"
-	"time"
-
-	"github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// TestOpenBucket_OpensALeaseBucketTheClusterCannotYetRaise is the assertion that
-// keeps this fix from becoming the outage it prevents.
-//
-// OpenBucket runs on every reconcile tick. If a raise it cannot place failed the
-// open, no reconciler could acquire its lease, and every one of them would stop
-// cluster-wide — over a lease bucket that is present and quorate at the replica
-// count it already has. The config naming a node that is not serving yet is the
-// documented way a cluster grows, so this is a state to tolerate, not to refuse.
-func TestOpenBucket_OpensALeaseBucketTheClusterCannotYetRaise(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
-	defer nc.Close()
-
-	cfg := kvlease.BucketConfig{Name: "lease-growing-cluster", TTL: time.Minute}
-	kv, err := kvlease.OpenBucket(t.Context(), js, cfg)
-	require.NoError(t, err)
-	_, err = kv.Put(t.Context(), "leader", []byte("node1"))
-	require.NoError(t, err)
-
-	// One embedded server cannot hold three replicas, so the raise must fail.
-	clustersize.RedeclareForTest(t, 3)
-
-	reopened, err := kvlease.OpenBucket(t.Context(), js, cfg)
-	require.NoError(t, err, "a reconciler must still be able to reach its lease bucket")
-	entry, err := reopened.Get(t.Context(), "leader")
-	require.NoError(t, err)
-	assert.Equal(t, "node1", string(entry.Value()))
-}
-
-// TestOpenBucket_RefusesWhenClusterSizeIsUndeclared pins the fail-closed half:
-// creating a lease bucket without knowing the cluster size must not guess at one
-// replica, which is the state that makes a lease unacquirable when a node fails.
-func TestOpenBucket_RefusesWhenClusterSizeIsUndeclared(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
-	defer nc.Close()
-
-	clustersize.RedeclareForTest(t, 0)
-
-	_, err := kvlease.OpenBucket(t.Context(), js, kvlease.BucketConfig{Name: "lease-undeclared", TTL: time.Minute})
-	require.ErrorIs(t, err, clustersize.ErrUndeclared)
-}
-
-// TestNATSBucket_StopsOnAConfigFault covers the retry loop rather than the open.
-//
-// The loop exists because a cold multi-node start has no JetStream quorum yet, and
-// waiting for one is right. An undeclared cluster size never becomes declared by
-// waiting, so a loop that cannot tell them apart holds a reconciler for its whole
-// retry window and then reports the bucket as unreachable — which was never the
-// problem.
-func TestNATSBucket_StopsOnAConfigFault(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	defer nc.Close()
-
-	clustersize.RedeclareForTest(t, 0)
-
-	start := time.Now()
-	_, err := kvlease.NATSBucket(nc, "lease-config-fault", time.Minute)(t.Context())
-	elapsed := time.Since(start)
-
-	require.ErrorIs(t, err, clustersize.ErrUndeclared,
-		"the error has to name the config fault, not the bucket it never reached")
-	require.Lessf(t, elapsed, 10*time.Second,
-		"returned after %s; a permanent fault must not consume the retry window", elapsed)
-}

spinifex/foundation/state/kvlease/lease.go

@@ -1,257 +0,0 @@
-// Package kvlease elects a single holder of a JetStream KV key. The key's TTL
-// bounds crash recovery; background renewal keeps it alive while the holder
-// works, so a pass that outlives the TTL does not lose the lease underneath it.
-package kvlease
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"log/slog"
-	"sync"
-	"time"
-
-	"github.com/nats-io/nats.go/jetstream"
-)
-
-// releaseTimeout bounds the delete on the way out, which runs on a detached
-// context and so cannot inherit the caller's deadline.
-const releaseTimeout = 5 * time.Second
-
-// heldError reports the node that owns the key when a claim loses to it.
-type heldError struct{ holder string }
-
-func (e *heldError) Error() string { return "kvlease: lease held by " + e.holder }
-
-// BucketFunc returns the KV bucket holding the lease key. It is injected because
-// each subsystem initialises its own bucket, some with migrations attached.
-type BucketFunc func(context.Context) (jetstream.KeyValue, error)
-
-// Config describes a single lease. Name appears in logs; Retry applies only to Run.
-type Config struct {
-	Name   string
-	Bucket BucketFunc
-	Key    string
-	Holder string
-	Attrs  []any
-
-	TTL   time.Duration // must match the bucket's TTL
-	Renew time.Duration // gap between refreshes
-	Retry time.Duration // session mode only: non-leader re-attempt interval
-
-	// Edges fire per transition, never under the lock
-	OnGained func(context.Context) error
-	OnLost   func()
-}
-
-// Lease is a CAS-guarded claim on a single key. The zero value is unusable; call New.
-type Lease struct {
-	cfg  Config
-	mu   sync.Mutex
-	kv   jetstream.KeyValue
-	rev  uint64
-	held bool
-	stop context.CancelFunc
-	lost chan struct{}
-}
-
-// New validates cfg and returns an unclaimed lease. Renew must leave room for a failed refresh,
-// so it may be at most half the TTL.
-func New(cfg Config) (*Lease, error) {
-	switch {
-	case cfg.Bucket == nil:
-		return nil, errors.New("kvlease: Bucket is required")
-	case cfg.Key == "":
-		return nil, errors.New("kvlease: Key is required")
-	case cfg.Holder == "":
-		return nil, errors.New("kvlease: Holder is required")
-	case cfg.TTL <= 0:
-		return nil, errors.New("kvlease: TTL must be positive")
-	case cfg.Renew <= 0 || cfg.Renew*2 > cfg.TTL:
-		return nil, fmt.Errorf("kvlease: Renew must be in (0, TTL/2]; got renew=%s ttl=%s", cfg.Renew, cfg.TTL)
-	}
-	if cfg.Name == "" {
-		cfg.Name = cfg.Key
-	}
-	if cfg.Retry <= 0 {
-		cfg.Retry = cfg.Renew
-	}
-	return &Lease{cfg: cfg, lost: make(chan struct{})}, nil
-}
-
-// Held reports whether this node currently holds the lease.
-func (l *Lease) Held() bool {
-	l.mu.Lock()
-	defer l.mu.Unlock()
-	return l.held
-}
-
-// logArgs prefixes the lease identity onto a log line's own fields. The local
-// node logs as "node"; "holder" is reserved for whoever owns the key.
-func (l *Lease) logArgs(extra ...any) []any {
-	args := make([]any, 0, 4+len(l.cfg.Attrs)+len(extra))
-	args = append(args, "lease", l.cfg.Name, "node", l.cfg.Holder)
-	args = append(args, l.cfg.Attrs...)
-	return append(args, extra...)
-}
-
-// TryAcquire makes one attempt to claim the key, starting background renewal and firing OnGained on
-// success. A node that already holds the lease is reported as still holding it without touching the key.
-func (l *Lease) TryAcquire(ctx context.Context) bool {
-	if l.Held() {
-		return true
-	}
-	kv, err := l.cfg.Bucket(ctx)
-	if err != nil {
-		slog.WarnContext(ctx, "kvlease: bucket unavailable", l.logArgs("err", err)...)
-		return false
-	}
-	rev, err := l.claim(ctx, kv)
-	if err != nil {
-		if held, ok := errors.AsType[*heldError](err); ok {
-			slog.DebugContext(ctx, "kvlease: held by another node", l.logArgs("holder", held.holder)...)
-		}
-		return false
-	}
-
-	renewCtx, stop := context.WithCancel(ctx)
-	l.mu.Lock()
-	if l.stop != nil {
-		l.stop()
-	}
-	l.lost = make(chan struct{})
-	l.kv, l.rev, l.held, l.stop = kv, rev, true, stop
-	l.mu.Unlock()
-	go l.renew(renewCtx)
-
-	slog.InfoContext(ctx, "kvlease: elected", l.logArgs()...)
-	if l.cfg.OnGained != nil {
-		if err := l.cfg.OnGained(ctx); err != nil {
-			// A leader that cannot set up its work is worse than no leader. Stand down so a healthy node wins.
-			slog.ErrorContext(ctx, "kvlease: OnGained failed, standing down", l.logArgs("err", err)...)
-			l.Release(ctx)
-			return false
-		}
-	}
-	return true
-}
-
-// claim creates the key or adopts one this holder left behind. A restarted
-// process finds its own key still present and must refresh it rather than
-// sit out the TTL with no leader elected.
-func (l *Lease) claim(ctx context.Context, kv jetstream.KeyValue) (uint64, error) {
-	rev, err := kv.Create(ctx, l.cfg.Key, []byte(l.cfg.Holder))
-	if err == nil {
-		return rev, nil
-	}
-	entry, gerr := kv.Get(ctx, l.cfg.Key)
-	if gerr != nil {
-		return 0, gerr
-	}
-	if string(entry.Value()) != l.cfg.Holder {
-		return 0, &heldError{holder: string(entry.Value())}
-	}
-	return kv.Update(ctx, l.cfg.Key, []byte(l.cfg.Holder), entry.Revision())
-}
-
-// renew refreshes the key at the revision last observed, until ctx is cancelled or a refresh
-// loses the CAS - which means the key expired and another node claimed it.
-func (l *Lease) renew(ctx context.Context) {
-	ticker := time.NewTicker(l.cfg.Renew)
-	defer ticker.Stop()
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-ticker.C:
-			l.mu.Lock()
-			kv, rev, held := l.kv, l.rev, l.held
-			l.mu.Unlock()
-			if !held {
-				return
-			}
-			next, err := kv.Update(ctx, l.cfg.Key, []byte(l.cfg.Holder), rev)
-			if err != nil {
-				slog.WarnContext(ctx, "kvlease: renewal lost the key, another node may take over mid-pass", l.logArgs("err", err)...)
-				l.markLost()
-				return
-			}
-			l.mu.Lock()
-			l.rev = next
-			l.mu.Unlock()
-		}
-	}
-}
-
-// markLost clears held state and fires OnLost. It reports whether this call was the transition,
-// so renewal and release never double-fire the edge.
-func (l *Lease) markLost() bool {
-	l.mu.Lock()
-	was := l.held
-	l.held = false
-	if was {
-		close(l.lost)
-	}
-	l.mu.Unlock()
-	if !was {
-		return false
-	}
-	if l.cfg.OnLost != nil {
-		l.cfg.OnLost()
-	}
-	return true
-}
-
-// Release deletes the key if this node still owns it at the revision it last observed.
-// It runs on a detached context: shutdown cancels ctx first, and skipping the delete would park
-// the lease for the full TTL.
-func (l *Lease) Release(ctx context.Context) {
-	l.mu.Lock()
-	kv, rev, stop := l.kv, l.rev, l.stop
-	l.kv, l.stop = nil, nil
-	l.mu.Unlock()
-	if stop != nil {
-		stop()
-	}
-	if kv == nil {
-		// Never acquired, or already released.
-		return
-	}
-	if !l.markLost() {
-		// Renewal already lost the key, so another node may hold it now. An unguarded delete here
-		// would drop that node's lease, not ours.
-		slog.WarnContext(ctx, "kvlease: already lost before release, not deleting", l.logArgs()...)
-		return
-	}
-	releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), releaseTimeout)
-	defer cancel()
-	if err := kv.Delete(releaseCtx, l.cfg.Key, jetstream.LastRevision(rev)); err != nil {
-		slog.WarnContext(releaseCtx, "kvlease: release failed, TTL will reap", l.logArgs("err", err)...)
-	}
-}
-
-// Run holds the lease across many passes, re-attempting on a ticker while this node is not the
-// holder. A holder does not re-attempt: renewal keeps the key.
-func (l *Lease) Run(ctx context.Context) {
-	ticker := time.NewTicker(l.cfg.Retry)
-	defer ticker.Stop()
-	l.TryAcquire(ctx)
-	for {
-		select {
-		case <-ctx.Done():
-			l.Release(ctx)
-			return
-		case <-ticker.C:
-			l.TryAcquire(ctx)
-		}
-	}
-}
-
-// Lost returns a channel closed when this lease stops being held, so work that
-// must not outlive the lease can select on it rather than poll Held. The channel
-// is per-acquisition: re-acquiring installs a fresh one.
-func (l *Lease) Lost() <-chan struct{} {
-	l.mu.Lock()
-	defer l.mu.Unlock()
-	return l.lost
-}

spinifex/foundation/state/kvlease/lease_test.go

@@ -1,437 +0,0 @@
-package kvlease_test
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"errors"
-	"log/slog"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
-	"github.com/nats-io/nats.go"
-	"github.com/nats-io/nats.go/jetstream"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-const (
-	testBucket = "kvlease-test"
-	testKey    = "leader"
-)
-
-// newTestLease builds a lease over a short-lived bucket so a test can outlast
-// the TTL in a second rather than a minute. Bucket is built after mut so a test
-// that lengthens the TTL gets a bucket that matches it.
-func newTestLease(t *testing.T, nc *nats.Conn, holder string, mut func(*kvlease.Config)) *kvlease.Lease {
-	t.Helper()
-	cfg := kvlease.Config{
-		Name:   "test",
-		Key:    testKey,
-		Holder: holder,
-		TTL:    time.Second,
-		Renew:  250 * time.Millisecond,
-		Retry:  100 * time.Millisecond,
-	}
-	if mut != nil {
-		mut(&cfg)
-	}
-	cfg.Bucket = kvlease.NATSBucket(nc, testBucket, cfg.TTL)
-	l, err := kvlease.New(cfg)
-	require.NoError(t, err)
-	return l
-}
-
-func testKV(t *testing.T, nc *nats.Conn, ttl time.Duration) jetstream.KeyValue {
-	t.Helper()
-	kv, err := kvlease.NATSBucket(nc, testBucket, ttl)(t.Context())
-	require.NoError(t, err)
-	return kv
-}
-
-// isOpen reports whether a Lost channel has not yet fired, without blocking.
-func isOpen(ch <-chan struct{}) bool {
-	select {
-	case <-ch:
-		return false
-	default:
-		return true
-	}
-}
-
-func TestTryAcquire_SecondHolderLoses(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	a := newTestLease(t, nc, "node-a", nil)
-	b := newTestLease(t, nc, "node-b", nil)
-
-	require.True(t, a.TryAcquire(t.Context()), "first acquire must win and create the bucket")
-	assert.False(t, b.TryAcquire(t.Context()), "second acquire must lose while the lease is held")
-
-	a.Release(t.Context())
-	assert.True(t, b.TryAcquire(t.Context()), "acquire after release must win")
-	b.Release(t.Context())
-}
-
-// A restarted process finds its own key still present. Refusing it would leave the subsystem leaderless for the rest of
-// the TTL for no reason.
-func TestTryAcquire_AdoptsOwnKeyAfterRestart(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	kv := testKV(t, nc, time.Second)
-
-	rev, err := kv.Create(t.Context(), testKey, []byte("node-a"))
-	require.NoError(t, err)
-
-	restarted := newTestLease(t, nc, "node-a", nil)
-	require.True(t, restarted.TryAcquire(t.Context()), "a holder must adopt the key it left behind")
-
-	entry, err := kv.Get(t.Context(), testKey)
-	require.NoError(t, err)
-	assert.Greater(t, entry.Revision(), rev, "adopting must refresh the key, resetting its TTL")
-	restarted.Release(t.Context())
-}
-
-// A pass can outlive the TTL on its own, so the key must be renewed while the pass runs. Without this a peer elects
-// itself mid-pass and both act at once.
-func TestRenew_KeySurvivesBeyondTTL(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	// Three renewals per TTL, and above the 100ms floor NATS puts on a bucket TTL.
-	short := func(c *kvlease.Config) { c.TTL, c.Renew = 300*time.Millisecond, 100*time.Millisecond }
-	a := newTestLease(t, nc, "node-a", short)
-	require.True(t, a.TryAcquire(t.Context()))
-	defer a.Release(t.Context())
-
-	// Well past the TTL: an unrenewed key would have expired by now.
-	time.Sleep(750 * time.Millisecond)
-
-	b := newTestLease(t, nc, "node-b", short)
-	assert.False(t, b.TryAcquire(t.Context()), "lease expired mid-pass: a peer can now act concurrently")
-	assert.True(t, a.Held())
-}
-
-// If the key was lost anyway, the stale holder's release must not delete the successor's key -
-// that hand the lease to a third node silently.
-func TestRelease_DoesNotDeleteSuccessorKey(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	a := newTestLease(t, nc, "node-a", func(c *kvlease.Config) {
-		c.TTL, c.Renew = time.Minute, 20*time.Second // no renewal during the test
-	})
-	require.True(t, a.TryAcquire(t.Context()))
-
-	kv := testKV(t, nc, time.Minute)
-	_, err := kv.Put(t.Context(), testKey, []byte("node-b"))
-	require.NoError(t, err)
-
-	a.Release(t.Context())
-
-	entry, err := kv.Get(t.Context(), testKey)
-	require.NoError(t, err, "stale release deleted the successor's key")
-	assert.Equal(t, "node-b", string(entry.Value()))
-}
-
-// Shutdown cancels the acquiring context first. A release bound to it would no-op and park the lease for the full TTL.
-func TestRelease_SurvivesCancelledContext(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	ctx, cancel := context.WithCancel(t.Context())
-	a := newTestLease(t, nc, "node-a", nil)
-	require.True(t, a.TryAcquire(ctx))
-
-	cancel()
-	a.Release(ctx)
-
-	b := newTestLease(t, nc, "node-b", nil)
-	assert.True(t, b.TryAcquire(t.Context()), "lease must be free immediately, not at the TTL")
-	b.Release(t.Context())
-}
-
-func TestEdges_FireOncePerTransition(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	var gained, lost atomic.Int32
-	a := newTestLease(t, nc, "node-a", func(c *kvlease.Config) {
-		c.OnGained = func(context.Context) error { gained.Add(1); return nil }
-		c.OnLost = func() { lost.Add(1) }
-	})
-
-	require.True(t, a.TryAcquire(t.Context()))
-	require.True(t, a.TryAcquire(t.Context()), "a holder re-attempting must stay held")
-	assert.EqualValues(t, 1, gained.Load(), "OnGained fired twice for one election")
-	assert.EqualValues(t, 0, lost.Load())
-
-	a.Release(t.Context())
-	assert.EqualValues(t, 1, lost.Load())
-	a.Release(t.Context())
-	assert.EqualValues(t, 1, lost.Load(), "a second release must not re-fire OnLost")
-}
-
-func TestRenew_LossFiresOnLost(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	lost := make(chan struct{}, 4)
-	a := newTestLease(t, nc, "node-a", func(c *kvlease.Config) {
-		c.OnLost = func() { lost <- struct{}{} }
-	})
-	require.True(t, a.TryAcquire(t.Context()))
-
-	// Stand in for the key expiring and a peer claiming it.
-	kv := testKV(t, nc, time.Second)
-	require.NoError(t, kv.Delete(t.Context(), testKey))
-	_, err := kv.Create(t.Context(), testKey, []byte("node-b"))
-	require.NoError(t, err)
-
-	select {
-	case <-lost:
-	case <-time.After(3 * time.Second):
-		t.Fatal("renewal lost the CAS but OnLost never fired")
-	}
-	assert.False(t, a.Held())
-}
-
-// A leader that cannot set up its own work is worse than no leader.
-func TestTryAcquire_OnGainedFailureStandsDown(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	var lost atomic.Int32
-	a := newTestLease(t, nc, "node-a", func(c *kvlease.Config) {
-		c.OnGained = func(context.Context) error { return errors.New("subscribe failed") }
-		c.OnLost = func() { lost.Add(1) }
-	})
-
-	assert.False(t, a.TryAcquire(t.Context()), "a holder that cannot set up its work must not report leadership")
-	assert.False(t, a.Held())
-	assert.EqualValues(t, 1, lost.Load(), "OnLost must fire so the caller tears down partial statea")
-
-	b := newTestLease(t, nc, "node-b", nil)
-	assert.True(t, b.TryAcquire(t.Context()), "standing down must free the key immediately")
-	b.Release(t.Context())
-}
-
-func TestNew_RejectUnworkableConfig(t *testing.T) {
-	base := func() kvlease.Config {
-		return kvlease.Config{
-			Bucket: func(context.Context) (jetstream.KeyValue, error) { return nil, nil },
-			Key:    testKey,
-			Holder: "node-a",
-			TTL:    time.Minute,
-			Renew:  20 * time.Second,
-		}
-	}
-	_, err := kvlease.New(base())
-	require.NoError(t, err)
-
-	cases := map[string]func(*kvlease.Config){
-		"no bucket":           func(c *kvlease.Config) { c.Bucket = nil },
-		"no key":              func(c *kvlease.Config) { c.Key = "" },
-		"no holder":           func(c *kvlease.Config) { c.Holder = "" },
-		"no ttl":              func(c *kvlease.Config) { c.TTL = 0 },
-		"renew equals ttl":    func(c *kvlease.Config) { c.Renew = c.TTL },
-		"renew over half ttl": func(c *kvlease.Config) { c.Renew = 31 * time.Second },
-	}
-	for name, mut := range cases {
-		t.Run(name, func(t *testing.T) {
-			cfg := base()
-			mut(&cfg)
-			_, err := kvlease.New(cfg)
-			assert.Error(t, err)
-		})
-	}
-}
-
-// Session mode: a loser must take over on its retry tick rather than wait out the TTL, which is what makes a rolling restart a sub-second gap.
-func TestRun_LoserTakesOverAfterLeaderStops(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	aCtx, stopA := context.WithCancel(t.Context())
-	a := newTestLease(t, nc, "node-a", nil)
-	go a.Run(aCtx)
-	require.Eventually(t, a.Held, 2*time.Second, 20*time.Millisecond)
-
-	b := newTestLease(t, nc, "node-b", nil)
-	go b.Run(t.Context())
-	time.Sleep(300 * time.Millisecond)
-	require.False(t, b.Held(), "two holders at once")
-
-	stopA()
-	require.Eventually(t, b.Held, 2*time.Second, 20*time.Millisecond, "loser must take over on its retry tick, not at the TTL")
-}
-
-// Work gated on Lose must run while the lease is held. A channel closed at construction or on acquire would abort every pass before it started.
-func TestLost_OpenWhileHeld(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	a := newTestLease(t, nc, "node-a", nil)
-
-	assert.True(t, isOpen(a.Lost()), "an unclaimed lease has lost nothing")
-
-	require.True(t, a.TryAcquire(t.Context()))
-	assert.True(t, isOpen(a.Lost()))
-
-	// Well past two renewal intervals: a renewing holder must stay held.
-	time.Sleep(600 * time.Millisecond)
-	assert.True(t, isOpen(a.Lost()), "renewal succeeded but lost fired anyway")
-
-	a.Release(t.Context())
-}
-
-// The signal callers select on to abandon work. Polling Held cannot express "stop what you are doing now", which is what a lost lease requires.
-func TestLost_ClosesOnRenewalLoss(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	a := newTestLease(t, nc, "node-a", nil)
-	require.True(t, a.TryAcquire(t.Context()))
-	lost := a.Lost()
-
-	// Stand in for the key expiring and a peer claiming it.
-	kv := testKV(t, nc, time.Second)
-	require.NoError(t, kv.Delete(t.Context(), testKey))
-	_, err := kv.Create(t.Context(), testKey, []byte("node-b"))
-	require.NoError(t, err)
-
-	select {
-	case <-lost:
-	case <-time.After(3 * time.Second):
-		t.Fatal("renewal lost the CAS but Lost never closed")
-	}
-	assert.False(t, a.Held())
-}
-
-func TestLost_ClosesOnRelease(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	a := newTestLease(t, nc, "node-a", nil)
-	require.True(t, a.TryAcquire(t.Context()))
-	lost := a.Lost()
-
-	a.Release(t.Context())
-	assert.False(t, isOpen(lost), "release must close Lost so gated work stops")
-
-	// A second release must not close an already-closed channel.
-	assert.NotPanics(t, func() { a.Release(t.Context()) })
-}
-
-// The channel is per-acquisition. Handing a re-elected holder the closed channel from its previous term would abort its new pass immediately.
-func TestLost_FreshChannelAfterReacquire(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	a := newTestLease(t, nc, "node-a", nil)
-
-	require.True(t, a.TryAcquire(t.Context()))
-	first := a.Lost()
-	a.Release(t.Context())
-	require.False(t, isOpen(first))
-
-	require.True(t, a.TryAcquire(t.Context()))
-	second := a.Lost()
-	assert.True(t, isOpen(second), "a re-elected holder got its old term's closed channel")
-	assert.NotEqual(t, first, second, "Lost must be replaced on acquire, not reused")
-
-	a.Release(t.Context())
-}
-
-func TestAttrs_AppearOnLeaseOwnedLogLines(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-
-	var buf bytes.Buffer
-	prev := slog.Default()
-	slog.SetDefault(slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo})))
-	t.Cleanup(func() { slog.SetDefault(prev) })
-
-	lease := newTestLease(t, nc, "node-a", func(cfg *kvlease.Config) {
-		cfg.Attrs = []any{"bucket", "quota-reconcile"}
-	})
-	require.True(t, lease.TryAcquire(t.Context()))
-	lease.Release(t.Context())
-
-	require.Contains(t, buf.String(), `"bucket":"quota-reconcile"`)
-}
-
-// A node that never won the lease still unwinds its Run loop through Release.
-// That is not a lost key, so it must not raise the warning that says one.
-func TestRelease_WithoutAcquireIsSilent(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-
-	holder := newTestLease(t, nc, "node-a", nil)
-	require.True(t, holder.TryAcquire(t.Context()))
-	t.Cleanup(func() { holder.Release(t.Context()) })
-
-	var buf bytes.Buffer
-	prev := slog.Default()
-	slog.SetDefault(slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo})))
-	t.Cleanup(func() { slog.SetDefault(prev) })
-
-	loser := newTestLease(t, nc, "node-b", nil)
-	require.False(t, loser.TryAcquire(t.Context()))
-	loser.Release(t.Context())
-
-	assert.NotContains(t, buf.String(), "already lost before release")
-}
-
-// The warning still has to fire where it means something: renewal lost the key,
-// so a delete here would drop the successor's lease rather than.
-func TestRelease_AfterRenewalLossWarns(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	lost := make(chan struct{}, 4)
-	a := newTestLease(t, nc, "node-a", func(c *kvlease.Config) {
-		c.OnLost = func() { lost <- struct{}{} }
-	})
-	require.True(t, a.TryAcquire(t.Context()))
-
-	kv := testKV(t, nc, time.Second)
-	require.NoError(t, kv.Delete(t.Context(), testKey))
-	_, err := kv.Create(t.Context(), testKey, []byte("node-b"))
-	require.NoError(t, err)
-
-	select {
-	case <-lost:
-	case <-time.After(3 * time.Second):
-		t.Fatal("renewal lost the CAS but OnLost never fired")
-	}
-
-	var buf bytes.Buffer
-	prev := slog.Default()
-	slog.SetDefault(slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo})))
-	t.Cleanup(func() { slog.SetDefault(prev) })
-
-	a.Release(t.Context())
-	assert.Contains(t, buf.String(), "already lost before release")
-}
-
-// logLines captures default-logger output as decoded JSON records.
-func logLines(t *testing.T) func() []map[string]any {
-	t.Helper()
-	var buf bytes.Buffer
-	prev := slog.Default()
-	slog.SetDefault(slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})))
-	t.Cleanup(func() { slog.SetDefault(prev) })
-	return func() []map[string]any {
-		var out []map[string]any
-		dec := json.NewDecoder(&buf)
-		for dec.More() {
-			var rec map[string]any
-			require.NoError(t, dec.Decode(&rec))
-			out = append(out, rec)
-		}
-		return out
-	}
-}
-
-// "holder" must name whoever owns the key. The local node logs as "node", so a
-// loser's log does not read as though it won.
-func TestLogFields_SeparateLocalNodeFromCurrentHolder(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	read := logLines(t)
-
-	a := newTestLease(t, nc, "node-a", nil)
-	require.True(t, a.TryAcquire(t.Context()))
-	t.Cleanup(func() { a.Release(t.Context()) })
-	b := newTestLease(t, nc, "node-b", nil)
-	require.False(t, b.TryAcquire(t.Context()))
-
-	var elected, refused map[string]any
-	for _, rec := range read() {
-		switch {
-		case rec["msg"] == "kvlease: elected" && rec["node"] == "node-a":
-			elected = rec
-		case rec["node"] == "node-b":
-			refused = rec
-		}
-	}
-	require.NotNil(t, elected, "the elected line must carry the local node as node")
-	assert.NotContains(t, elected, "holder")
-	require.NotNil(t, refused, "a refused acquire must log the local node as node")
-	assert.Equal(t, "node-a", refused["holder"], "a refused acquire must report the node that holds the key")
-}

spinifex/foundation/state/kvstore/bucket.go

@@ -53,8 +53,14 @@ type Bucket struct {
 	js  jetstream.JetStream
 	cfg Config
 
+	// mu guards the fields below and is never held across a network call.
 	mu sync.Mutex
 	kv jetstream.KeyValue
+	// opened is set once a handle has been held, after which a missing bucket
+	// is a lost one, and only RecreateIfMissing may create it again.
+	opened bool
+	// resolving is closed when the one open or reopen in flight finishes.
+	resolving chan struct{}
 }
 
 // NewBucket returns a Bucket over js. A nil js is permitted: every call to KV then
@@ -68,7 +74,7 @@ func NewBucket(js jetstream.JetStream, cfg Config) *Bucket {
 // rather than on first use. js is what recovery reopens against; pass nil to
 // keep the handle fixed for a caller not ready to recover.
 func NewOpenBucket(js jetstream.JetStream, kv jetstream.KeyValue, cfg Config) *Bucket {
-	return &Bucket{js: js, kv: kv, cfg: cfg}
+	return &Bucket{js: js, kv: kv, cfg: cfg, opened: kv != nil}
 }
 
 // Name returns the configured bucket name, for callers holding several buckets
@@ -84,27 +90,9 @@ func (b *Bucket) Configured() bool {
 }
 
 // KV opens the bucket on first use and returns the cached handle thereafter.
+// Concurrent callers share one open, and each stops waiting when its ctx ends.
 func (b *Bucket) KV(ctx context.Context) (jetstream.KeyValue, error) {
-	b.mu.Lock()
-	defer b.mu.Unlock()
-	if b.kv != nil {
-		return b.kv, nil
-	}
-	if b.js == nil {
-		if b.cfg.Missing == "" {
-			return nil, errors.New("kvstore: no JetStream client configured")
-		}
-		return nil, errors.New(b.cfg.Missing)
-	}
-	kv, err := b.open(ctx)
-	if err != nil {
-		return nil, err
-	}
-	if err := b.onOpen(ctx, kv); err != nil {
-		return nil, err
-	}
-	b.kv = kv
-	return kv, nil
+	return b.resolve(ctx, false)
 }
 
 // LastSequence reports the stream sequence of the newest message matching
@@ -140,41 +128,115 @@ func (b *Bucket) LastSequence(ctx context.Context, filter string) (uint64, error
 // caller that saw the stream go out from under it. It reconnects; whether it
 // may also recreate an absent bucket is Config.RecreateIfMissing.
 func (b *Bucket) Reopen(ctx context.Context) (jetstream.KeyValue, error) {
-	b.mu.Lock()
-	defer b.mu.Unlock()
+	return b.resolve(ctx, true)
+}
 
-	// Checked before the handle is discarded. A Bucket over a fixed handle has
-	// nothing to resolve against, so dropping it would turn one transient
-	// outage into a permanent failure for every later call.
-	if b.js == nil {
-		if b.cfg.Missing == "" {
-			return nil, errors.New("kvstore: cannot reopen without a JetStream client")
+// resolve returns the cached handle, or runs the one open in flight and
+// caches its result. A waiter for a reopen takes whatever handle the flight
+// left, since another goroutine resolving it is the repair this caller wanted.
+func (b *Bucket) resolve(ctx context.Context, reopen bool) (jetstream.KeyValue, error) {
+	for {
+		b.mu.Lock()
+		if !reopen && b.kv != nil {
+			kv := b.kv
+			b.mu.Unlock()
+			return kv, nil
 		}
-		return nil, errors.New(b.cfg.Missing)
+		// Checked before the handle is discarded. A Bucket over a fixed handle
+		// has nothing to resolve against, so dropping it would turn one
+		// transient outage into a permanent failure for every later call.
+		if b.js == nil {
+			b.mu.Unlock()
+			return nil, b.missingErr(reopen)
+		}
+		if wait := b.resolving; wait != nil {
+			b.mu.Unlock()
+			select {
+			case <-wait:
+			case <-ctx.Done():
+				return nil, fmt.Errorf("kvstore: open %s: %w", b.cfg.Name, ctx.Err())
+			}
+			if reopen {
+				b.mu.Lock()
+				kv := b.kv
+				b.mu.Unlock()
+				if kv != nil {
+					return kv, nil
+				}
+			}
+			continue
+		}
+		done := make(chan struct{})
+		b.resolving = done
+		if reopen {
+			b.kv = nil
+		}
+		opened := b.opened
+		b.mu.Unlock()
+
+		kv, err := b.connect(ctx, reopen, opened)
+
+		b.mu.Lock()
+		if err == nil {
+			b.kv, b.opened = kv, true
+		}
+		b.resolving = nil
+		close(done)
+		b.mu.Unlock()
+		return kv, err
 	}
-	b.kv = nil
+}
 
-	// Another goroutine may already have repaired it, so reconnecting comes
-	// first and is the only step that is always safe.
-	kv, err := b.js.KeyValue(ctx, b.cfg.Name)
-	if err != nil {
-		if !b.cfg.RecreateIfMissing {
+func (b *Bucket) missingErr(reopen bool) error {
+	switch {
+	case b.cfg.Missing != "":
+		return errors.New(b.cfg.Missing)
+	case reopen:
+		return errors.New("kvstore: cannot reopen without a JetStream client")
+	default:
+		return errors.New("kvstore: no JetStream client configured")
+	}
+}
+
+// connect does the network half of resolve, with no lock held. A bucket that
+// was open before is only attached to unless RecreateIfMissing allows more.
+func (b *Bucket) connect(ctx context.Context, reopen, opened bool) (jetstream.KeyValue, error) {
+	var (
+		kv  jetstream.KeyValue
+		err error
+	)
+	switch {
+	case reopen:
+		// Another goroutine may already have repaired it, so reconnecting comes
+		// first and is the only step that is always safe.
+		kv, err = b.js.KeyValue(ctx, b.cfg.Name)
+		if err != nil {
+			if !b.cfg.RecreateIfMissing {
+				return nil, fmt.Errorf("kvstore: reopen %s: %w", b.cfg.Name, err)
+			}
+			if !errors.Is(err, jetstream.ErrBucketNotFound) && !IsStreamUnavailable(err) {
+				return nil, fmt.Errorf("kvstore: reopen %s: %w", b.cfg.Name, err)
+			}
+			slog.WarnContext(ctx, "kvstore: bucket stream lost, recreating", "bucket", b.cfg.Name)
+			if kv, err = b.open(ctx); err != nil {
+				return nil, fmt.Errorf("kvstore: recreate %s: %w", b.cfg.Name, err)
+			}
+		}
+	case opened && !b.cfg.RecreateIfMissing:
+		if kv, err = b.js.KeyValue(ctx, b.cfg.Name); err != nil {
 			return nil, fmt.Errorf("kvstore: reopen %s: %w", b.cfg.Name, err)
 		}
-		if !errors.Is(err, jetstream.ErrBucketNotFound) && !IsStreamUnavailable(err) {
-			return nil, fmt.Errorf("kvstore: reopen %s: %w", b.cfg.Name, err)
-		}
-		slog.WarnContext(ctx, "kvstore: bucket stream lost, recreating", "bucket", b.cfg.Name)
+	default:
 		if kv, err = b.open(ctx); err != nil {
-			return nil, fmt.Errorf("kvstore: recreate %s: %w", b.cfg.Name, err)
+			return nil, err
 		}
 	}
-
 	if err := b.onOpen(ctx, kv); err != nil {
 		return nil, err
 	}
-	b.kv = kv
-	slog.InfoContext(ctx, "kvstore: bucket reopened", "bucket", b.cfg.Name)
+	if reopen {
+		slog.InfoContext(ctx, "kvstore: bucket reopened", "bucket", b.cfg.Name)
+	}
 	return kv, nil
 }
 
@@ -219,8 +281,8 @@ func (b *Bucket) Delete(ctx context.Context, key string) error {
 	return nil
 }
 
-// CompareAndDelete removes a key only if it is still at rev, returning
-// ErrConflict when it is not.
+// CompareAndDelete removes a key only if it is still at rev, returning an
+// error that is both ErrConflict and kvsync.ErrRevisionMismatch when it is not.
 //
 // Unlike Delete this is not idempotent: a guarded delete carries an expected
 // last sequence, and an absent, tombstoned or purged key fails that check, so
@@ -232,15 +294,12 @@ func (b *Bucket) Delete(ctx context.Context, key string) error {
 func (b *Bucket) CompareAndDelete(ctx context.Context, key string, rev uint64) error {
 	kv, err := b.KV(ctx)
 	if err != nil {
-		return err
+		return syncError("delete", key, err)
 	}
 	// Not through withKV: the revision is the caller's, so a replay against a
 	// reopened bucket would be guarded on a revision from the bucket it lost.
 	if err := kv.Delete(ctx, key, jetstream.LastRevision(rev)); err != nil {
-		if errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
-			return fmt.Errorf("%w: %s", ErrConflict, key)
-		}
-		return fmt.Errorf("kvstore: delete %s: %w", key, err)
+		return syncError("delete", key, err)
 	}
 	return nil
 }

spinifex/foundation/state/kvstore/bucket_creation_invariants_test.go

@@ -15,11 +15,9 @@ import (
 var createFuncs = []string{"CreateKeyValue(", "CreateOrUpdateKeyValue("}
 
 // exempt are the only files and packages allowed to call them: the chokepoint
-// itself, the lease-bucket opener it shares its rule with, and the test helpers
-// that build buckets for a single embedded server.
+// itself and the test helpers that build buckets for a single embedded server.
 var exempt = []string{
 	filepath.Join("spinifex", "foundation", "state", "kvstore", "buckets.go"),
-	filepath.Join("spinifex", "foundation", "state", "kvlease"),
 	filepath.Join("internal", "testkit"),
 }
 

spinifex/foundation/state/kvstore/leaderget_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/nats-io/nats-server/v2/server"
 	"github.com/nats-io/nats.go"
 	jsapi "github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -79,3 +80,26 @@ func TestAPIPrefixMatchesNatsGo(t *testing.T) {
 		require.Equal(t, sent, kvstore.APIPrefix(js.Options())+"INFO", name)
 	}
 }
+
+// LeaderGet returns the newest value and its revision, and a deleted or
+// absent key as ErrNotFound rather than the tombstone's empty payload.
+func TestStore_LeaderGetReadsTheNewestValueAndTombstones(t *testing.T) {
+	t.Parallel()
+	store := newStore(t)
+
+	_, _, err := store.LeaderGet(t.Context(), "k")
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+
+	mustCreate(t, store, "k", record{Name: "a", Count: 1})
+	seedRaw(t, store, "k", record{Name: "b", Count: 2})
+	got, rev, err := store.LeaderGet(t.Context(), "k")
+	require.NoError(t, err)
+	assert.Equal(t, record{Name: "b", Count: 2}, *got)
+	_, wantRev, err := store.Get(t.Context(), "k")
+	require.NoError(t, err)
+	assert.Equal(t, wantRev, rev)
+
+	require.NoError(t, store.Delete(t.Context(), "k"))
+	_, _, err = store.LeaderGet(t.Context(), "k")
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+}

spinifex/foundation/state/kvstore/recovery_test.go

@@ -133,6 +133,10 @@ func TestStore_RecreateIfMissingDefaultsOff(t *testing.T) {
 
 	err := store.Set(t.Context(), "acct-a/one", &record{Name: "two"})
 	require.Error(t, err, "recovery must not silently recreate a bucket the caller did not opt into")
+	// The failed recovery dropped the handle, so this one reaches the first-use
+	// open, which must not create either.
+	err = store.Set(t.Context(), "acct-a/one", &record{Name: "three"})
+	require.Error(t, err, "a later operation must not recreate it through the first-use open")
 
 	_, err = js.KeyValue(t.Context(), recoveryBucket)
 	require.Error(t, err, "the bucket must still be absent")

spinifex/foundation/state/kvstore/store.go

@@ -43,6 +43,35 @@ func (s *Store[T]) Get(ctx context.Context, key string) (*T, uint64, error) {
 	return v, rev, err
 }
 
+// LeaderGet is Get answered by the stream leader, for a caller that must not
+// act on a lagging replica's older value. The revision is the stream sequence.
+func (s *Store[T]) LeaderGet(ctx context.Context, key string) (*T, uint64, error) {
+	if _, err := s.KV(ctx); err != nil {
+		return nil, 0, err
+	}
+	s.mu.Lock()
+	js, name := s.js, s.cfg.Name
+	s.mu.Unlock()
+	msg, err := LeaderMsgGet(ctx, js, "KV_"+name, MsgGetRequest{LastFor: "$KV." + name + "." + key})
+	if errors.Is(err, jetstream.ErrMsgNotFound) {
+		return nil, 0, fmt.Errorf("%w: %s", ErrNotFound, key)
+	}
+	if err != nil {
+		return nil, 0, fmt.Errorf("kvstore: leader get %s: %w", key, err)
+	}
+	if op := msg.Header.Get(kvOperationHeader); op == "DEL" || op == "PURGE" {
+		return nil, 0, fmt.Errorf("%w: %s", ErrNotFound, key)
+	}
+	var v T
+	if err := json.Unmarshal(msg.Data, &v); err != nil {
+		return nil, 0, fmt.Errorf("kvstore: decode %s: %w", key, err)
+	}
+	return &v, msg.Sequence, nil
+}
+
+// kvOperationHeader marks a KV stream message that deletes or purges its key.
+const kvOperationHeader = "KV-Operation"
+
 // Create writes a record only if the key is absent, returning ErrExists when
 // it is not. The create-only write is the single-writer claim, and the
 // returned revision is the CAS precondition for the claim's own first update.

spinifex/foundation/state/kvstore/sync.go

@@ -0,0 +1,110 @@
+package kvstore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+var _ kvsync.KVStore = (*Bucket)(nil)
+
+// syncBucket holds every kvsync lock key, namespaced by the key itself.
+const (
+	syncBucket        = "spinifex-sync"
+	syncBucketVersion = 1
+)
+
+// OpenSyncBucket returns the Bucket every kvsync lock in a process is built on:
+// file storage, one revision of history and no TTL, since a lease's expiry is
+// observed by its contenders rather than kept by the store. It opens on first use.
+//
+// RecreateIfMissing stays false: a recreated stream restarts its revisions, and
+// a generation must only grow.
+func OpenSyncBucket(js jetstream.JetStream) *Bucket {
+	return NewBucket(js, Config{
+		Name:        syncBucket,
+		History:     1,
+		Description: "kvsync claims and leases",
+		Missing:     "kvstore: sync bucket has no JetStream client",
+		OnOpen: func(ctx context.Context, kv jetstream.KeyValue) error {
+			return WriteVersion(ctx, kv, syncBucketVersion)
+		},
+	})
+}
+
+// Load returns the value and revision of key, with an error that is both
+// ErrNotFound and kvsync.ErrNotFound when the key is absent.
+func (b *Bucket) Load(ctx context.Context, key string) ([]byte, uint64, error) {
+	var (
+		value []byte
+		rev   uint64
+	)
+	err := b.withKV(ctx, func(kv jetstream.KeyValue) error {
+		entry, err := kv.Get(ctx, key)
+		if err != nil {
+			return err
+		}
+		value, rev = entry.Value(), entry.Revision()
+		return nil
+	})
+	if err != nil {
+		return nil, 0, syncError("load", key, err)
+	}
+	return value, rev, nil
+}
+
+// CompareAndSwap writes value only while key is at rev, where rev 0 means the
+// key must be absent or deleted, and returns the new revision. A lost race is
+// both ErrConflict and kvsync.ErrRevisionMismatch.
+func (b *Bucket) CompareAndSwap(ctx context.Context, key string, value []byte, rev uint64) (uint64, error) {
+	kv, err := b.KV(ctx)
+	if err != nil {
+		return 0, syncError("compare-and-swap", key, err)
+	}
+	// Not through withKV: the revision is the caller's, as in CompareAndDelete.
+	var next uint64
+	if rev == 0 {
+		next, err = kv.Create(ctx, key, value)
+	} else {
+		next, err = kv.Update(ctx, key, value, rev)
+	}
+	if err != nil {
+		return 0, syncError("compare-and-swap", key, err)
+	}
+	return next, nil
+}
+
+// syncError classifies a JetStream failure into both packages' sentinels.
+func syncError(op, key string, err error) error {
+	switch {
+	case errors.Is(err, jetstream.ErrKeyNotFound):
+		return fmt.Errorf("%w: %w: %s", ErrNotFound, kvsync.ErrNotFound, key)
+	case IsRevisionConflict(err):
+		return fmt.Errorf("%w: %w: %s", ErrConflict, kvsync.ErrRevisionMismatch, key)
+	case isUnreachable(err):
+		return fmt.Errorf("kvstore: %s %s: %w: %w", op, key, kvsync.ErrUnavailable, err)
+	default:
+		return fmt.Errorf("kvstore: %s %s: %w", op, key, err)
+	}
+}
+
+// isUnreachable is IsStreamUnavailable widened to a connection that is down, a
+// request that went unanswered, or a server answering 503 (10008 while JetStream
+// has no meta leader), since a lock caller retries all alike.
+func isUnreachable(err error) bool {
+	if apiErr, ok := errors.AsType[*jetstream.APIError](err); ok && apiErr.Code == http.StatusServiceUnavailable {
+		return true
+	}
+	return IsStreamUnavailable(err) ||
+		errors.Is(err, context.DeadlineExceeded) ||
+		errors.Is(err, nats.ErrTimeout) ||
+		errors.Is(err, nats.ErrConnectionClosed) ||
+		errors.Is(err, nats.ErrConnectionReconnecting) ||
+		errors.Is(err, nats.ErrNoServers) ||
+		errors.Is(err, nats.ErrDisconnected)
+}

spinifex/foundation/state/kvstore/sync_test.go

@@ -0,0 +1,217 @@
+package kvstore_test
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSyncBucket_OpensWithOneRevisionNoTTLAndASchemaVersion(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	b := kvstore.OpenSyncBucket(js)
+	kv, err := b.KV(t.Context())
+	require.NoError(t, err)
+
+	status, err := kv.Status(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, "spinifex-sync", status.Bucket())
+	assert.Equal(t, int64(1), status.History())
+	assert.Zero(t, status.TTL(), "a lease's expiry is observed by contenders, never kept by the store")
+	info := status.(*jetstream.KeyValueBucketStatus).StreamInfo()
+	assert.Equal(t, jetstream.FileStorage, info.Config.Storage)
+
+	version, err := kvstore.ReadVersion(t.Context(), kv)
+	require.NoError(t, err)
+	assert.Equal(t, 1, version)
+}
+
+func TestSyncBucket_ErrorsCarryBothPackagesSentinels(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	b := kvstore.OpenSyncBucket(js)
+
+	_, _, err := b.Load(t.Context(), "k")
+	assert.ErrorIs(t, err, kvsync.ErrNotFound)
+	assert.ErrorIs(t, err, kvstore.ErrNotFound)
+
+	rev, err := b.CompareAndSwap(t.Context(), "k", []byte("a"), 0)
+	require.NoError(t, err)
+
+	mismatches := map[string]func() error{
+		"create over a live key": func() error {
+			_, err := b.CompareAndSwap(t.Context(), "k", []byte("b"), 0)
+			return err
+		},
+		"update at a stale revision": func() error {
+			_, err := b.CompareAndSwap(t.Context(), "k", []byte("b"), rev+100)
+			return err
+		},
+		"delete at a stale revision": func() error {
+			return b.CompareAndDelete(t.Context(), "k", rev+100)
+		},
+	}
+	for name, op := range mismatches {
+		t.Run(name, func(t *testing.T) {
+			err := op()
+			assert.ErrorIs(t, err, kvsync.ErrRevisionMismatch)
+			assert.ErrorIs(t, err, kvstore.ErrConflict)
+		})
+	}
+
+	value, got, err := b.Load(t.Context(), "k")
+	require.NoError(t, err)
+	assert.Equal(t, rev, got)
+	assert.Equal(t, []byte("a"), value)
+}
+
+func TestSyncBucket_RevisionGrowsAcrossADelete(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	b := kvstore.OpenSyncBucket(js)
+
+	first, err := b.CompareAndSwap(t.Context(), "k", []byte("a"), 0)
+	require.NoError(t, err)
+	require.NoError(t, b.CompareAndDelete(t.Context(), "k", first))
+
+	// Rev 0 creates over the delete marker, at a revision above the first.
+	second, err := b.CompareAndSwap(t.Context(), "k", []byte("b"), 0)
+	require.NoError(t, err)
+	assert.Greater(t, second, first)
+
+	third, err := b.CompareAndSwap(t.Context(), "k", []byte("c"), second)
+	require.NoError(t, err)
+	assert.Greater(t, third, second)
+}
+
+func TestSyncBucket_StoppedJetStreamIsUnavailable(t *testing.T) {
+	stops := map[string]func(t *testing.T, ns *server.Server){
+		"jetstream disabled": func(t *testing.T, ns *server.Server) { require.NoError(t, ns.DisableJetStream()) },
+		"server shut down":   func(_ *testing.T, ns *server.Server) { ns.Shutdown(); ns.WaitForShutdown() },
+	}
+	for name, stop := range stops {
+		t.Run(name, func(t *testing.T) {
+			ns, _, js := testutil.StartTestJetStream(t)
+			b := kvstore.OpenSyncBucket(js)
+			rev, err := b.CompareAndSwap(t.Context(), "k", []byte("a"), 0)
+			require.NoError(t, err)
+			stop(t, ns)
+
+			ops := map[string]func(context.Context) error{
+				"load": func(ctx context.Context) error { _, _, err := b.Load(ctx, "k"); return err },
+				"swap": func(ctx context.Context) error {
+					_, err := b.CompareAndSwap(ctx, "k", []byte("b"), rev)
+					return err
+				},
+				"delete": func(ctx context.Context) error { return b.CompareAndDelete(ctx, "k", rev) },
+			}
+			for op, call := range ops {
+				// Bounded because a gone server answers nothing at all.
+				ctx, cancel := context.WithTimeout(t.Context(), 500*time.Millisecond)
+				err := call(ctx)
+				cancel()
+				assert.ErrorIs(t, err, kvsync.ErrUnavailable, op)
+				assert.NotErrorIs(t, err, kvsync.ErrRevisionMismatch, op)
+			}
+		})
+	}
+}
+
+// A recreated sync stream restarts its revisions, so a stale holder's revision
+// could match a new key. Losing the stream must leave every write failing.
+func TestSyncBucket_LostStreamIsNeverRecreated(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	b := kvstore.OpenSyncBucket(js)
+	rev, err := b.CompareAndSwap(t.Context(), "k", []byte("a"), 0)
+	require.NoError(t, err)
+	require.NoError(t, js.DeleteKeyValue(t.Context(), "spinifex-sync"))
+
+	for range 3 {
+		_, err = b.CompareAndSwap(t.Context(), "k", []byte("b"), 0)
+		require.Error(t, err)
+		_, err = b.CompareAndSwap(t.Context(), "k", []byte("b"), rev)
+		require.Error(t, err)
+		_, _, err = b.Load(t.Context(), "k")
+		require.Error(t, err)
+	}
+	_, err = js.KeyValue(t.Context(), "spinifex-sync")
+	require.ErrorIs(t, err, jetstream.ErrBucketNotFound, "the sync bucket must still be absent")
+}
+
+// stallingJS holds every bucket lookup until gate closes, standing in for an
+// open stuck on a server that has not formed quorum.
+type stallingJS struct {
+	jetstream.JetStream
+
+	entered chan struct{}
+	once    sync.Once
+	gate    chan struct{}
+}
+
+func (s *stallingJS) KeyValue(ctx context.Context, name string) (jetstream.KeyValue, error) {
+	s.once.Do(func() { close(s.entered) })
+	<-s.gate
+	return s.JetStream.KeyValue(ctx, name)
+}
+
+// One caller's stuck open must not hold another past its own context, or a
+// lease renewal waiting behind it could outlive its deadline.
+func TestSyncBucket_AWaiterOnAStuckOpenStopsAtItsContext(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	stall := &stallingJS{JetStream: js, entered: make(chan struct{}), gate: make(chan struct{})}
+	t.Cleanup(func() { close(stall.gate) })
+	b := kvstore.OpenSyncBucket(stall)
+
+	go func() { _, _, _ = b.Load(context.Background(), "k") }()
+	<-stall.entered
+
+	done := make(chan error, 1)
+	go func() {
+		ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
+		defer cancel()
+		_, err := b.CompareAndSwap(ctx, "k", []byte("a"), 0)
+		done <- err
+	}()
+	select {
+	case err := <-done:
+		require.ErrorIs(t, err, kvsync.ErrUnavailable)
+	case <-time.After(5 * time.Second):
+		t.Fatal("the swap waited on another caller's open past its own context")
+	}
+}
+
+// leaderlessJS answers every bucket lookup and create with the 503 a cluster
+// gives while JetStream has no meta leader.
+type leaderlessJS struct {
+	jetstream.JetStream
+}
+
+func errNoMetaLeader() error {
+	return &jetstream.APIError{Code: 503, ErrorCode: 10008, Description: "JetStream system temporarily unavailable"}
+}
+
+func (leaderlessJS) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
+	return nil, errNoMetaLeader()
+}
+
+func (leaderlessJS) CreateKeyValue(context.Context, jetstream.KeyValueConfig) (jetstream.KeyValue, error) {
+	return nil, errNoMetaLeader()
+}
+
+// A cold cluster still electing its meta leader is an outage a lock caller
+// waits out, not a fault that ends its election.
+func TestSyncBucket_ALeaderlessJetStreamIsUnavailable(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	b := kvstore.OpenSyncBucket(leaderlessJS{JetStream: js})
+
+	_, _, err := b.Load(t.Context(), "k")
+	require.ErrorIs(t, err, kvsync.ErrUnavailable)
+	_, err = b.CompareAndSwap(t.Context(), "k", []byte("a"), 0)
+	require.ErrorIs(t, err, kvsync.ErrUnavailable)
+}

spinifex/foundation/state/kvsync/claim.go

@@ -0,0 +1,131 @@
+package kvsync
+
+import (
+	"context"
+	"errors"
+	"fmt"
+)
+
+// Claim is ownership of a key with no expiry: it is held until its holder
+// unlocks it or another holder takes it over. It keeps no state, so it gives no
+// exclusion between goroutines of one holder; its callers serialise their use.
+type Claim struct {
+	store  KVStore
+	key    string
+	holder string
+}
+
+// NewClaim returns the claim on key for holder. It panics if key is invalid or
+// already constructed against store, or if holder is empty.
+func NewClaim(store KVStore, key, holder string) *Claim {
+	mustWire(errors.Join(validHolder(holder), registerKey(store, key)))
+	return &Claim{store: store, key: key, holder: holder}
+}
+
+// TryLock takes the key when it is absent, and adopts it when it already names
+// this holder, keeping its generation. Otherwise it returns *HeldError naming
+// the holder.
+func (c *Claim) TryLock(ctx context.Context) (Token, error) {
+	var err error
+	for range casAttempts {
+		var rev uint64
+		rev, err = c.store.CompareAndSwap(ctx, c.key, encode(c.holder, 0), 0)
+		if err == nil {
+			return Token{Holder: c.holder, Generation: rev}, nil
+		}
+		if !errors.Is(err, ErrRevisionMismatch) {
+			return Token{}, fmt.Errorf("kvsync: claim %s: %w", c.key, err)
+		}
+		cur, rev, ok, rerr := read(ctx, c.store, c.key)
+		if rerr != nil {
+			return Token{}, rerr
+		}
+		if !ok {
+			// Unlocked between the create and the read; try again.
+			continue
+		}
+		if cur.Holder != c.holder {
+			return Token{}, &HeldError{Current: cur}
+		}
+		// A read can be answered by a lagging replica, so adoption is confirmed
+		// by a write at the revision read, which only the leader accepts.
+		_, err = c.store.CompareAndSwap(ctx, c.key, encode(c.holder, cur.Generation), rev)
+		if err == nil {
+			return cur, nil
+		}
+		if !errors.Is(err, ErrRevisionMismatch) {
+			return Token{}, fmt.Errorf("kvsync: claim %s: %w", c.key, err)
+		}
+	}
+	return Token{}, fmt.Errorf("kvsync: claim %s: %w", c.key, err)
+}
+
+// Unlock deletes the key while it still holds held, and returns ErrLost when
+// it does not.
+func (c *Claim) Unlock(ctx context.Context, held Token) error {
+	for range casAttempts {
+		cur, rev, ok, err := read(ctx, c.store, c.key)
+		if err != nil {
+			return err
+		}
+		if !ok || cur != held {
+			return ErrLost
+		}
+		err = c.store.CompareAndDelete(ctx, c.key, rev)
+		if err == nil {
+			return nil
+		}
+		if !errors.Is(err, ErrRevisionMismatch) {
+			return fmt.Errorf("kvsync: unlock %s: %w", c.key, err)
+		}
+	}
+	return ErrLost
+}
+
+// TakeOver moves the key to this holder only while it still holds from, the
+// token the caller judged dead from its own evidence. It returns *HeldError
+// when the key no longer holds from.
+func (c *Claim) TakeOver(ctx context.Context, from Token) (Token, error) {
+	cur, rev, ok, err := read(ctx, c.store, c.key)
+	if err != nil {
+		return Token{}, err
+	}
+	if !ok || cur != from {
+		return Token{}, &HeldError{Current: cur}
+	}
+	next, err := c.store.CompareAndSwap(ctx, c.key, encode(c.holder, 0), rev)
+	if errors.Is(err, ErrRevisionMismatch) {
+		return Token{}, heldAfterConflict(ctx, c.store, c.key)
+	}
+	if err != nil {
+		return Token{}, fmt.Errorf("kvsync: take over %s: %w", c.key, err)
+	}
+	return Token{Holder: c.holder, Generation: next}, nil
+}
+
+// Holder reports the token of whoever holds the key, and false when nobody does.
+func (c *Claim) Holder(ctx context.Context) (Token, bool, error) {
+	cur, _, ok, err := read(ctx, c.store, c.key)
+	return cur, ok, err
+}
+
+// ClaimGroup hands out per-resource claims under one registered prefix.
+type ClaimGroup struct {
+	store  KVStore
+	prefix string
+	holder string
+}
+
+// NewClaimGroup registers prefix, which ends in ".", against store. It panics
+// on an invalid or overlapping prefix, or an empty holder.
+func NewClaimGroup(store KVStore, prefix, holder string) *ClaimGroup {
+	mustWire(errors.Join(validHolder(holder), registerPrefix(store, prefix)))
+	return &ClaimGroup{store: store, prefix: prefix, holder: holder}
+}
+
+// Get returns the claim on prefix+name. A Claim keeps no state, so a fresh
+// value for the same name is equivalent to an earlier one.
+func (g *ClaimGroup) Get(name string) *Claim {
+	mustWire(validName("name", name))
+	return &Claim{store: g.store, key: g.prefix + name, holder: g.holder}
+}

spinifex/foundation/state/kvsync/claim_test.go

@@ -0,0 +1,160 @@
+package kvsync_test
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestClaim_SecondHolderIsRefusedAndSameHolderAdoptsWithoutAWrite(t *testing.T) {
+	s := stores(t, 3)
+	a := kvsync.NewClaim(s[0], "claim.x", "node-a")
+	b := kvsync.NewClaim(s[1], "claim.x", "node-b")
+	restarted := kvsync.NewClaim(s[2], "claim.x", "node-a")
+
+	held, err := a.TryLock(t.Context())
+	require.NoError(t, err)
+
+	_, err = b.TryLock(t.Context())
+	he, ok := asHeld(err)
+	require.True(t, ok, "want *HeldError, got %v", err)
+	assert.Equal(t, held, he.Current, "the refusal must name the holder and its generation")
+
+	adopted, err := restarted.TryLock(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, held, adopted, "adoption keeps the generation")
+	cur, ok, err := a.Holder(t.Context())
+	require.NoError(t, err)
+	require.True(t, ok)
+	assert.Equal(t, held, cur, "the adoption's confirming write keeps the generation in the key")
+}
+
+// A replica that has not seen a take-over still names the old holder. Its read
+// alone must not let that holder adopt the key back from the new one.
+func TestClaim_AStaleReadCannotAdoptATakenOverKey(t *testing.T) {
+	s := stores(t, 3)
+	a := kvsync.NewClaim(s[0], "claim.x", "node-a")
+	b := kvsync.NewClaim(s[1], "claim.x", "node-b")
+
+	first, err := a.TryLock(t.Context())
+	require.NoError(t, err)
+	value, rev, err := s[0].Load(t.Context(), "claim.x")
+	require.NoError(t, err)
+	second, err := b.TakeOver(t.Context(), first)
+	require.NoError(t, err)
+
+	stale := &staleStore{KVStore: s[2], value: value, rev: rev}
+	restarted := kvsync.NewClaim(stale, "claim.x", "node-a")
+	_, err = restarted.TryLock(t.Context())
+	_, held := asHeld(err)
+	assert.True(t, held, "a stale read must not adopt the key, got %v", err)
+	cur, ok, err := b.Holder(t.Context())
+	require.NoError(t, err)
+	require.True(t, ok)
+	assert.Equal(t, second, cur, "the key must still name the holder that took it over")
+}
+
+func TestClaim_TakeOverNeedsTheCurrentToken(t *testing.T) {
+	s := stores(t, 3)
+	a := kvsync.NewClaim(s[0], "claim.x", "node-a")
+	b := kvsync.NewClaim(s[1], "claim.x", "node-b")
+	c := kvsync.NewClaim(s[2], "claim.x", "node-c")
+
+	first, err := a.TryLock(t.Context())
+	require.NoError(t, err)
+
+	second, err := b.TakeOver(t.Context(), first)
+	require.NoError(t, err)
+	assert.Equal(t, "node-b", second.Holder)
+	assert.Greater(t, second.Generation, first.Generation)
+
+	// c judged node-a dead, but node-b has moved the key on since.
+	_, err = c.TakeOver(t.Context(), first)
+	he, ok := asHeld(err)
+	require.True(t, ok, "want *HeldError, got %v", err)
+	assert.Equal(t, second, he.Current)
+}
+
+func TestClaim_UnlockAfterTakeOverIsLostAndLeavesTheNewHolder(t *testing.T) {
+	s := stores(t, 2)
+	a := kvsync.NewClaim(s[0], "claim.x", "node-a")
+	b := kvsync.NewClaim(s[1], "claim.x", "node-b")
+
+	first, err := a.TryLock(t.Context())
+	require.NoError(t, err)
+	second, err := b.TakeOver(t.Context(), first)
+	require.NoError(t, err)
+
+	require.ErrorIs(t, a.Unlock(t.Context(), first), kvsync.ErrLost)
+
+	cur, ok, err := a.Holder(t.Context())
+	require.NoError(t, err)
+	require.True(t, ok, "the stale unlock deleted the new holder's key")
+	assert.Equal(t, second, cur)
+}
+
+func TestClaim_RelockAfterUnlockGetsAHigherGeneration(t *testing.T) {
+	s := stores(t, 1)
+	a := kvsync.NewClaim(s[0], "claim.x", "node-a")
+
+	first, err := a.TryLock(t.Context())
+	require.NoError(t, err)
+	require.NoError(t, a.Unlock(t.Context(), first))
+
+	_, ok, err := a.Holder(t.Context())
+	require.NoError(t, err)
+	require.False(t, ok, "unlock must free the key")
+
+	second, err := a.TryLock(t.Context())
+	require.NoError(t, err)
+	assert.Greater(t, second.Generation, first.Generation, "a generation must outrank every earlier one, across a delete")
+}
+
+func TestClaimGroup_HandsOutKeysUnderItsPrefix(t *testing.T) {
+	s := stores(t, 2)
+	ga := kvsync.NewClaimGroup(s[0], "uefi.", "node-a")
+	gb := kvsync.NewClaimGroup(s[1], "uefi.", "node-b")
+
+	_, err := ga.Get("i-1").TryLock(t.Context())
+	require.NoError(t, err)
+	_, err = gb.Get("i-1").TryLock(t.Context())
+	_, ok := asHeld(err)
+	assert.True(t, ok, "the same name in two groups is the same key")
+	_, err = gb.Get("i-2").TryLock(t.Context())
+	assert.NoError(t, err, "a different name is a different key")
+	assert.NotZero(t, revision(t, s[0], "uefi.i-1"))
+}
+
+// A peer that moves the key between the take-over's read and its write wins,
+// and the refusal names that peer rather than the holder judged dead.
+func TestClaim_TakeOverThatLosesTheRaceNamesTheWinner(t *testing.T) {
+	s := stores(t, 3)
+	a := kvsync.NewClaim(s[0], "claim.x", "node-a")
+	b := kvsync.NewClaim(s[1], "claim.x", "node-b")
+
+	first, err := a.TryLock(t.Context())
+	require.NoError(t, err)
+	var winner kvsync.Token
+	racing := &racingStore{KVStore: s[2], race: func() {
+		winner, err = b.TakeOver(t.Context(), first)
+		require.NoError(t, err)
+	}}
+	c := kvsync.NewClaim(racing, "claim.x", "node-c")
+
+	_, err = c.TakeOver(t.Context(), first)
+	he, ok := asHeld(err)
+	require.True(t, ok, "want *HeldError, got %v", err)
+	assert.Equal(t, winner, he.Current)
+}
+
+func TestClaim_UnlockOfAnAbsentKeyIsLost(t *testing.T) {
+	s := stores(t, 1)
+	a := kvsync.NewClaim(s[0], "claim.x", "node-a")
+
+	first, err := a.TryLock(t.Context())
+	require.NoError(t, err)
+	require.NoError(t, a.Unlock(t.Context(), first))
+	require.ErrorIs(t, a.Unlock(t.Context(), first), kvsync.ErrLost, "a key already gone is not this holder's to free")
+}