refactor: port the ELBv2 data-path leader to kvsync

The data-path reconciler that landed on dev elected its leader through
kvlease in its own spinifex-elbv2-leader bucket. It now holds the
elbv2.datapath lease in the sync bucket, constructed with the daemon's
other leases, and the old bucket joins the set deleted at start.

The RDS delete tests that landed alongside it build their reconciler
through the lease-taking constructor.

spinifex/domains/admission/quota/quota.go

@@ -22,10 +22,10 @@ const ReconcileInterval = 30 * time.Second
 // dimension; the live-counted dimensions need no stored state.
 const KVBucketAccountUsage = "spinifex-account-usage"
 
-// KVBucketQuotaReconcile is the leader-lock bucket for the gateway quota
+// KeyReconcile is the sync-bucket leader lease for the gateway quota reconcile.
-// reconcile. It is deliberately distinct from vpcd's network-reconcile lock so
+// It is deliberately distinct from vpcd's network-reconcile lease so the two
-// the two independent loops never share one mutex and block each other.
+// independent loops never share one mutex and block each other.
-const KVBucketQuotaReconcile = "spinifex-quota-reconcile"
+const KeyReconcile = "quota.reconcile"
 
 // KVBucketAccountQuota holds the per-account limit overrides, keyed by
 // accountID. It is separate from the accounts bucket so a quota write never

spinifex/domains/dns/delegation.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	nsconfig "github.com/mulgadc/northstar/pkg/config"
-	"github.com/nats-io/nats.go"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 
 	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
 )
@@ -48,7 +48,7 @@ func (w *Writer) PlatformDelegation(name string) (servers []string, zone string,
 // Delegate writes, or with remove withdraws, an NS set in the base zone that
 // delegates each name to this cluster's nameservers. It takes the zone-writer
 // election first, so it never races the reconcile's own writes to the zone.
-func (w *Writer) Delegate(ctx context.Context, nc *nats.Conn, holder string, names []string, remove bool) (*ChangeResult, error) {
+func (w *Writer) Delegate(ctx context.Context, lease *kvsync.Lease, names []string, remove bool) (*ChangeResult, error) {
 	if !w.Enabled() {
 		return nil, errors.New("northstar is not configured on this node")
 	}
@@ -77,7 +77,7 @@ func (w *Writer) Delegate(ctx context.Context, nc *nats.Conn, holder string, nam
 	if len(batch.Changes) == 0 {
 		return &ChangeResult{}, nil
 	}
-	release, err := w.awaitWriterElection(ctx, nc, holder)
+	release, err := w.awaitWriterElection(ctx, lease)
 	if err != nil {
 		return nil, err
 	}
@@ -85,19 +85,19 @@ func (w *Writer) Delegate(ctx context.Context, nc *nats.Conn, holder string, nam
 	return w.ApplyBatch(batch)
 }
 
-func (w *Writer) awaitWriterElection(ctx context.Context, nc *nats.Conn, holder string) (func(), error) {
+func (w *Writer) awaitWriterElection(ctx context.Context, lease *kvsync.Lease) (func(), error) {
-	if nc == nil {
+	if lease == nil {
 		return func() {}, nil
 	}
 	tick := time.NewTicker(delegateLockPoll)
 	defer tick.Stop()
 	for {
-		if release, ok := reconcilelock.AcquireLeader(ctx, nc, KVBucketDNSReconcile, holder); ok {
+		if release, ok := reconcilelock.AcquireLeader(ctx, lease); ok {
 			return release, nil
 		}
 		select {
 		case <-ctx.Done():
-			return nil, fmt.Errorf("another node is writing the zone: %w", ctx.Err())
+			return nil, fmt.Errorf("another writer has the zone: %w", ctx.Err())
 		case <-tick.C:
 		}
 	}

spinifex/domains/dns/delegation_test.go

@@ -14,7 +14,7 @@ import (
 
 func TestDelegateWritesAndWithdrawsPlatformDelegations(t *testing.T) {
 	w, _ := newMultiNodeTestWriter(t)
-	_, err := w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net", "B.r53e2e.spx3.net."}, false)
+	_, err := w.Delegate(t.Context(), nil, []string{"a.r53e2e.spx3.net", "B.r53e2e.spx3.net."}, false)
 	require.NoError(t, err)
 
 	for _, name := range []string{"a.r53e2e.spx3.net.", "b.r53e2e.spx3.net"} {
@@ -29,7 +29,7 @@ func TestDelegateWritesAndWithdrawsPlatformDelegations(t *testing.T) {
 	assert.True(t, ok, "a name below the base domain is decided by the base zone")
 	assert.Empty(t, servers)
 
-	_, err = w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net"}, true)
+	_, err = w.Delegate(t.Context(), nil, []string{"a.r53e2e.spx3.net"}, true)
 	require.NoError(t, err)
 	servers, _, _, err = w.PlatformDelegation("a.r53e2e.spx3.net")
 	require.NoError(t, err)
@@ -43,11 +43,11 @@ func TestDelegateWritesAndWithdrawsPlatformDelegations(t *testing.T) {
 // servers written while the cluster published a larger set.
 func TestDelegateRemoveWithdrawsServersFromAnEarlierSet(t *testing.T) {
 	w, _ := newMultiNodeTestWriter(t)
-	_, err := w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net"}, false)
+	_, err := w.Delegate(t.Context(), nil, []string{"a.r53e2e.spx3.net"}, false)
 	require.NoError(t, err)
 
 	w.nameservers = w.nameservers[:1]
-	_, err = w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net"}, true)
+	_, err = w.Delegate(t.Context(), nil, []string{"a.r53e2e.spx3.net"}, true)
 	require.NoError(t, err)
 	servers, _, _, err := w.PlatformDelegation("a.r53e2e.spx3.net")
 	require.NoError(t, err)
@@ -66,7 +66,7 @@ func TestPlatformDelegationIgnoresNamesOutsideTheBaseDomain(t *testing.T) {
 func TestDelegateRefusesPlatformAndForeignNames(t *testing.T) {
 	w, _ := newMultiNodeTestWriter(t)
 	for _, name := range []string{"ns1.spx3.net", "x.elb.spx3.net", "compute.spx3.net", "example.com", "spx3.net"} {
-		_, err := w.Delegate(t.Context(), nil, "node1", []string{"ok.spx3.net", name}, false)
+		_, err := w.Delegate(t.Context(), nil, []string{"ok.spx3.net", name}, false)
 		require.Error(t, err, name)
 	}
 	servers, _, _, err := w.PlatformDelegation("ok.spx3.net")
@@ -76,22 +76,45 @@ func TestDelegateRefusesPlatformAndForeignNames(t *testing.T) {
 
 func TestDelegateWaitsForTheZoneWriterElection(t *testing.T) {
 	w, _ := newMultiNodeTestWriter(t)
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	prev := delegateLockPoll
 	delegateLockPoll = 10 * time.Millisecond
 	t.Cleanup(func() { delegateLockPoll = prev })
 
-	release, ok := reconcilelock.AcquireLeader(t.Context(), nc, KVBucketDNSReconcile, "node2")
+	release, ok := reconcilelock.AcquireLeader(t.Context(), dnsLease(js, "node2"))
 	require.True(t, ok)
 	ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
 	defer cancel()
-	_, err := w.Delegate(ctx, nc, "node1", []string{"a.spx3.net"}, false)
+	node1 := dnsLease(js, "node1")
-	require.ErrorContains(t, err, "another node is writing the zone")
+	_, err := w.Delegate(ctx, node1, []string{"a.spx3.net"}, false)
+	require.ErrorContains(t, err, "another writer has the zone")
 
 	release()
-	_, err = w.Delegate(t.Context(), nc, "node1", []string{"a.spx3.net"}, false)
+	_, err = w.Delegate(t.Context(), node1, []string{"a.spx3.net"}, false)
 	require.NoError(t, err)
 	servers, _, _, err := w.PlatformDelegation("a.spx3.net")
 	require.NoError(t, err)
 	assert.Len(t, servers, 2)
 }
+
+// The reconcile pass and a delegation on one node share the DNS lease, so a
+// delegation waits for this node's own pass as it would for another node's.
+func TestDelegateWaitsForThisNodesOwnReconcilePass(t *testing.T) {
+	w, _ := newMultiNodeTestWriter(t)
+	_, _, js := testutil.StartTestJetStream(t)
+	prev := delegateLockPoll
+	delegateLockPoll = 10 * time.Millisecond
+	t.Cleanup(func() { delegateLockPoll = prev })
+
+	shared := dnsLease(js, "node1")
+	release, ok := reconcilelock.AcquireLeader(t.Context(), shared)
+	require.True(t, ok, "the reconcile pass holds the lease")
+	ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
+	defer cancel()
+	_, err := w.Delegate(ctx, shared, []string{"a.spx3.net"}, false)
+	require.ErrorContains(t, err, "another writer has the zone")
+
+	release()
+	_, err = w.Delegate(t.Context(), shared, []string{"a.spx3.net"}, false)
+	require.NoError(t, err)
+}

spinifex/domains/dns/reconcile.go

@@ -11,7 +11,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/reconciler"
-	"github.com/nats-io/nats.go"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 )
 
 // DefaultReconcileInterval is how often the drift backstop rebuilds managed
@@ -33,6 +33,12 @@ const DefaultReconcileInterval = reconciler.DefaultResync
 // costs a cycle for nothing.
 const DefaultComputeTimeout = 2 * time.Minute
 
+// NewReconcileLease returns the zone writer's election lease, shared by the
+// drift reconcile and Delegate so the two never write the zone together.
+func NewReconcileLease(store kvsync.KVStore, holder string) *kvsync.Lease {
+	return reconcilelock.NewLeaderLease(store, KeyReconcile, holder)
+}
+
 // DesiredFunc returns the full desired managed record set built from the live
 // resource inventory across all tenants. The daemon supplies it by enumerating
 // instances, load balancers, and EKS clusters.
@@ -102,7 +108,8 @@ type Reconciler struct {
 	// lands in. Distinct from both baseDomain and internalDomain, so pruning it
 	// needs its own read the same way internalDomain does for EC2.
 	serviceZone string
-	nc          *nats.Conn
+	// lease elects the one node that writes the zones; nil skips the election.
+	lease *kvsync.Lease
 	// writer applies the converged batch. The reconcile owns the zone object on
 	// the node that won the election, so the batch never leaves this process.
 	writer   *Writer
@@ -111,7 +118,6 @@ type Reconciler struct {
 	// computeTimeout bounds how long one pass spends building the desired set.
 	// A field rather than a constant so a test can shrink it.
 	computeTimeout time.Duration
-	holder         string
 	sources        []reconciler.Source
 }
 
@@ -121,18 +127,15 @@ type Reconciler struct {
 // changes should wake the loop. They are optional: with none supplied the loop
 // falls back to the interval alone, which is the behaviour that predates the
 // watch.
-func NewReconciler(cfg *config.Config, cluster *config.ClusterConfig, nc *nats.Conn, writer *Writer, desired DesiredFunc, sources ...reconciler.Source) *Reconciler {
+func NewReconciler(cfg *config.Config, cluster *config.ClusterConfig, lease *kvsync.Lease, writer *Writer, desired DesiredFunc, sources ...reconciler.Source) *Reconciler {
 	r := &Reconciler{
-		nc:             nc,
+		lease:          lease,
 		writer:         writer,
 		desired:        desired,
 		interval:       DefaultReconcileInterval,
 		computeTimeout: DefaultComputeTimeout,
 		sources:        sources,
 	}
-	if cfg != nil {
-		r.holder = cfg.Node
-	}
 	zoneCfg, ok := zoneS3Config(cfg)
 	if !ok || desired == nil || !writer.Enabled() {
 		return r
@@ -166,25 +169,24 @@ func (r *Reconciler) Run(ctx context.Context) {
 		// No revisit deadline: the desired set is a function of the buckets
 		// being watched, so a change is the only reason to run early.
 		Reconcile: func(ctx context.Context) (time.Duration, error) {
-			r.reconcileOnce(ctx)
+			return r.reconcileOnce(ctx), nil
-			return 0, nil
 		},
 		Resync: r.interval,
 	})
 }
 
 // reconcileOnce computes the converging batch and applies it, on whichever node
-// wins this cycle's election. The read that finds the stale records and the
+// wins this cycle's election, and returns the election's revisit when it lost. The read that finds the stale records and the
 // write that removes them now happen in one process, so a record re-added
 // between them is no longer deleted by a batch that predates it.
-func (r *Reconciler) reconcileOnce(ctx context.Context) {
+func (r *Reconciler) reconcileOnce(ctx context.Context) time.Duration {
 	if !r.enabled {
-		return
+		return 0
 	}
-	if r.nc != nil {
+	if r.lease != nil {
-		release, elected := reconcilelock.AcquireLeader(ctx, r.nc, KVBucketDNSReconcile, r.holder)
+		release, retry, elected := reconcilelock.ElectPass(ctx, r.lease)
 		if !elected {
-			return
+			return retry
 		}
 		defer release()
 	}
@@ -193,18 +195,19 @@ func (r *Reconciler) reconcileOnce(ctx context.Context) {
 		// A corrupt zone is handled in readZone; anything reaching here left the
 		// desired state unapplied, so surface it rather than burying it in WARN.
 		slog.Error("dns reconcile: compute batch failed, retrying next cycle", "error", err)
-		return
+		return 0
 	}
 	if len(batch) == 0 {
-		return
+		return 0
 	}
 	slog.Debug("dns reconcile: converging", "changes", len(batch))
 	res, err := r.writer.ApplyBatch(&ChangeBatch{Changes: batch})
 	if err != nil {
 		slog.Warn("dns reconcile: apply failed, retrying next cycle", "changes", len(batch), "error", err)
-		return
+		return 0
 	}
 	slog.Debug("dns reconcile: converged", "changes", res.Applied, "zones", res.Zones)
+	return 0
 }
 
 // computeBatchBounded runs computeBatch with a deadline, so a pass that never

spinifex/domains/dns/reconcile_election_test.go

@@ -8,41 +8,50 @@ import (
 
 	"github.com/mulgadc/spinifex/internal/testkit"
 	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
-	"github.com/nats-io/nats.go"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-// electingReconcilerFor is reconcilerFor with a live connection, so a pass runs
+// electingReconcilerFor is reconcilerFor with a lease, so a pass runs the
-// the election instead of skipping it.
+// election instead of skipping it.
-func electingReconcilerFor(w *Writer, nc *nats.Conn, holder string, desired func() DesiredSet) *Reconciler {
+func electingReconcilerFor(w *Writer, lease *kvsync.Lease, desired func() DesiredSet) *Reconciler {
 	r := reconcilerFor(w, desired)
-	r.nc = nc
+	r.lease = lease
-	r.holder = holder
 	return r
 }
 
+// dnsLease returns holder's DNS lease on a store of its own, standing in for
+// a separate process.
+func dnsLease(js jetstream.JetStream, holder string) *kvsync.Lease {
+	return NewReconcileLease(kvstore.OpenSyncBucket(js), holder)
+}
+
 // The zone lock is gone, and what replaces it is that a zone has one writer:
 // the node holding the reconcile lease. A node that loses the election must
 // therefore not touch the object at all — if it did, two nodes could be
 // read-modify-writing the same zone with nothing serialising them.
 func TestReconcileOnce_ALoserOfTheElectionDoesNotWriteTheZone(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	w, objects := newTestWriter(t)
 	before := objects[testBase+".toml"]
 	require.NotEmpty(t, before)
 
 	// A peer holds the lease for the whole pass, so this node cannot win it.
-	release, elected := reconcilelock.AcquireLeader(t.Context(), nc, KVBucketDNSReconcile, "peer")
+	release, elected := reconcilelock.AcquireLeader(t.Context(), dnsLease(js, "peer"))
 	require.True(t, elected, "the peer has to hold the lease or this test proves nothing")
 
-	r := electingReconcilerFor(w, nc, "node1", func() DesiredSet {
+	r := electingReconcilerFor(w, dnsLease(js, "node1"), func() DesiredSet {
 		return DesiredSet{Changes: []Change{upsert("lb-1.elb."+testBase, "10.200.1.9")}}
 	})
-	r.reconcileOnce(t.Context())
+	revisit := r.reconcileOnce(t.Context())
 
 	assert.Equal(t, before, objects[testBase+".toml"],
 		"a node that did not win the election must leave the zone object exactly as it found it")
+	assert.Positive(t, revisit, "the loser asks to retry so a dead holder is taken over")
+	assert.LessOrEqual(t, revisit, reconcilelock.LeaderTTL/3, "the retry comes within a renewal interval")
 
 	release()
 }
@@ -50,10 +59,10 @@ func TestReconcileOnce_ALoserOfTheElectionDoesNotWriteTheZone(t *testing.T) {
 // The counterpart, so the test above is not passing because the pass was broken
 // for some other reason: with the lease free, the same reconciler writes.
 func TestReconcileOnce_TheWinnerOfTheElectionWritesTheZone(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	w, objects := newTestWriter(t)
 
-	r := electingReconcilerFor(w, nc, "node1", func() DesiredSet {
+	r := electingReconcilerFor(w, dnsLease(js, "node1"), func() DesiredSet {
 		return DesiredSet{Changes: []Change{upsert("lb-1.elb."+testBase, "10.200.1.9")}}
 	})
 	r.reconcileOnce(t.Context())

spinifex/domains/dns/reconcile_test.go

@@ -224,7 +224,7 @@ func TestReconciler_RunReturnsWhenDisabled(t *testing.T) {
 // path of Run: the startup pass must fire before any watch or resync activity,
 // and cancelling ctx must still let the loop return.
 func TestReconciler_RunPerformsStartupPassThenStopsOnCancel(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	endpoint, _ := fakeS3(t, "northstar")
 
 	var passes atomic.Int64
@@ -235,8 +235,7 @@ func TestReconciler_RunPerformsStartupPassThenStopsOnCancel(t *testing.T) {
 			Endpoint: endpoint, Bucket: "northstar", Region: "us-east-1",
 			AccessKey: "SYSTEM", SecretKey: "SYSTEMSECRET",
 		},
-		nc:     nc,
+		lease: dnsLease(js, "test-node"),
-		holder: "test-node",
 		desired: func() DesiredSet {
 			passes.Add(1)
 			return DesiredSet{}
@@ -268,7 +267,7 @@ func TestReconciler_RunPerformsStartupPassThenStopsOnCancel(t *testing.T) {
 // acquirable afterwards, not merely on the call returning, since it is the
 // leadership and not the pass that the rest of the cluster waits on.
 func TestReconcileOnce_AStuckPassSurrendersLeadership(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	endpoint, _ := fakeS3(t, "northstar")
 
 	blocked := make(chan struct{})
@@ -281,8 +280,7 @@ func TestReconcileOnce_AStuckPassSurrendersLeadership(t *testing.T) {
 			Endpoint: endpoint, Bucket: "northstar", Region: "us-east-1",
 			AccessKey: "SYSTEM", SecretKey: "SYSTEMSECRET",
 		},
-		nc:             nc,
+		lease:          dnsLease(js, "stuck-node"),
-		holder:         "stuck-node",
 		computeTimeout: 100 * time.Millisecond,
 		desired: func() DesiredSet {
 			<-blocked
@@ -302,7 +300,7 @@ func TestReconcileOnce_AStuckPassSurrendersLeadership(t *testing.T) {
 		t.Fatal("reconcileOnce never returned: a stuck pass still holds the lease forever")
 	}
 
-	release, elected := reconcilelock.AcquireLeader(t.Context(), nc, KVBucketDNSReconcile, "next-node")
+	release, elected := reconcilelock.AcquireLeader(t.Context(), dnsLease(js, "next-node"))
 	require.True(t, elected, "another node must be able to take the next cycle")
 	release()
 }

spinifex/domains/dns/types.go

@@ -11,9 +11,9 @@ const (
 	// PUT so every northstar instance reloads just that zone immediately, instead
 	// of waiting for the next S3 sync poll. No queue group: all servers consume.
 	SubjectZoneReload = "dns.zone.reload"
-	// KVBucketDNSReconcile elects the one node that writes the zones for a drift
+	// KeyReconcile elects the one node that writes the zones for a drift cycle.
-	// cycle. Its own bucket, so the election never blocks another reconcile loop.
+	// Its own key, so the election never blocks another reconcile loop.
-	KVBucketDNSReconcile = "spinifex-dns-reconcile"
+	KeyReconcile = "dns.reconcile"
 	// PrivateZone is the fixed AWS-parity private DNS zone (IMDS synthHostname).
 	PrivateZone = "compute.internal"
 	// DefaultTTL is applied when a change omits a TTL.

spinifex/domains/ecs/eni_test.go

@@ -322,7 +322,7 @@ func TestReaper_Awsvpc_ReleasesENI(t *testing.T) {
 	rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
 	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
 
-	sc := NewScheduler(nc, svc, "test-holder")
+	sc := testScheduler(t, nc, svc, "test-holder")
 	sc.reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
 
 	assert.Equal(t, 1, eni.releaseCalls)

spinifex/domains/ecs/enumeration_failure_test.go

@@ -27,7 +27,7 @@ func dropRevisit(pass func(context.Context) (time.Duration, error)) func(context
 // drained, services never relaunched, stopped tasks never pruned.
 func TestSchedulerPasses_EnumerationFailureIsReported(t *testing.T) {
 	svc, nc := newTestService(t)
-	sc := NewScheduler(nc, svc, "test-holder")
+	sc := testScheduler(t, nc, svc, "test-holder")
 	_, err := svc.CreateCluster(context.Background(), &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
 	require.NoError(t, err)
 

spinifex/domains/ecs/instance_recovery_test.go

@@ -49,7 +49,7 @@ func TestRegister_RestoresReapedInstanceToActive(t *testing.T) {
 	rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
 	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
 
-	NewScheduler(nc, svc, "test-holder").reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
+	testScheduler(t, nc, svc, "test-holder").reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
 
 	reaped := instanceStatus(t, svc, "web", "i-1")
 	require.Equal(t, InstanceStatusDraining, reaped.Status)
@@ -89,7 +89,7 @@ func TestReaper_ReleasesCapacityAndRecoversClean(t *testing.T) {
 	seeded.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
 	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &seeded))
 
-	NewScheduler(nc, svc, "test-holder").reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
+	testScheduler(t, nc, svc, "test-holder").reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
 
 	reaped := instanceStatus(t, svc, "web", "i-1")
 	require.Equal(t, InstanceStatusDraining, reaped.Status)

spinifex/domains/ecs/revisit_test.go

@@ -50,7 +50,7 @@ func schedulerRig(t *testing.T) (*Scheduler, *Service, jetstream.KeyValue) {
 	registerInstance(t, svc, "web", "i-1", 4096, 8192)
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
-	return NewScheduler(nc, svc, "test-holder"), svc, kv
+	return testScheduler(t, nc, svc, "test-holder"), svc, kv
 }
 
 // runOneTask places a task and returns its ID, read back from the record rather

spinifex/domains/ecs/scheduler.go

@@ -13,17 +13,19 @@ import (
 
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/reconciler"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
-// Scheduler timing. The leader lease lives in spinifex-ecs-leader (60s TTL); the
+// KeyScheduler is the ECS scheduler's leader lease in the sync bucket.
-// holder refreshes well inside that. The reaper marks an instance DRAINING after
+const KeyScheduler = "ecs.scheduler"
-// three missed 30s heartbeats.
+
+// Scheduler timing. A follower asks again at the lease's renewal rate. The
+// reaper marks an instance DRAINING after three missed 30s heartbeats.
 const (
-	schedulerLeaderKey  = "scheduler"
+	schedulerLeaseTTL   = 60 * time.Second
-	leaseRefresh        = 20 * time.Second
+	leaseRefresh        = schedulerLeaseTTL / 3
 	reaperInterval      = 30 * time.Second
 	heartbeatTimeout    = 90 * time.Second
 	stoppedReasonReaped = "ContainerInstance disconnected"
@@ -59,17 +61,14 @@ const (
 	reconcileResync = 5 * time.Minute
 )
 
-// Scheduler is the per-daemon ECS control loop. A single leader (elected via the
+// Scheduler is the per-daemon ECS control loop. A single leader (elected on its
-// shared leader bucket) owns the Layer-2 bus subscriptions and the heartbeat
+// lease) owns the Layer-2 bus subscriptions and the heartbeat
 // reaper; losers idle and retry. Placement itself happens synchronously in the
 // RunTask handler, so a brief leaderless gap never blocks RunTask.
 type Scheduler struct {
-	nc     *nats.Conn
+	nc    *nats.Conn
-	svc    *Service
+	svc   *Service
-	holder string
+	lease *kvsync.Lease
-
-	lease    *kvlease.Lease
-	leaseErr error
 
 	// wake carries a bus message that changed something the service reconcile
 	// reads. Fed by the task-state subscription and not by the heartbeat one:
@@ -89,47 +88,28 @@ type Scheduler struct {
 	subs []*nats.Subscription
 }
 
-// NewScheduler constructs a Scheduler. holder identifies this daemon in the lease.
+// NewSchedulerLease returns the ECS scheduler's leader lease for holder.
-func NewScheduler(nc *nats.Conn, svc *Service, holder string) *Scheduler {
+func NewSchedulerLease(store kvsync.KVStore, holder string) *kvsync.Lease {
-	// One slot: it means "something the reconcile reads has changed", and a
+	return kvsync.NewLease(store, KeyScheduler, holder, schedulerLeaseTTL)
-	// second arriving before the pass runs says nothing the first did not.
-	sc := &Scheduler{nc: nc, svc: svc, holder: holder, wake: make(chan struct{}, 1)}
-	sc.lease, sc.leaseErr = kvlease.New(kvlease.Config{
-		Name:   "ecs/scheduler",
-		Bucket: sc.leaderBucket,
-		Key:    schedulerLeaderKey,
-		Holder: holder,
-		TTL:    KVBucketECSLeaderTTL,
-		Renew:  leaseRefresh,
-		Retry:  leaseRefresh,
-		// The leader owns the Layer-2 bus subscriptions and the instance-role
-		// converge pass. A node that can do neither stands down rather than hold
-		// the lease and process nothing.
-		OnGained: sc.onGained,
-		OnLost:   sc.unsubscribeBus,
-	})
-	return sc
 }
 
-func (sc *Scheduler) leaderBucket(ctx context.Context) (jetstream.KeyValue, error) {
+// NewScheduler constructs a Scheduler that leads while it holds lease.
-	js, err := jetstream.New(sc.nc)
+func NewScheduler(nc *nats.Conn, svc *Service, lease *kvsync.Lease) *Scheduler {
-	if err != nil {
+	// One slot: it means "something the reconcile reads has changed", and a
-		return nil, err
+	// second arriving before the pass runs says nothing the first did not.
-	}
+	return &Scheduler{nc: nc, svc: svc, lease: lease, wake: make(chan struct{}, 1)}
-	return InitLeaderBucket(ctx, js)
 }
 
 // Run drives the leadership + reaper loop until ctx is cancelled. It is intended
 // to run as a daemon-boot goroutine; panics are the caller's recover concern.
 func (sc *Scheduler) Run(ctx context.Context) {
-	if sc.leaseErr != nil {
-		slog.ErrorContext(ctx, "ECS scheduler: lease config invalid", "holder", sc.holder, "err", sc.leaseErr)
-		return
-	}
 	leaseDone := make(chan struct{})
 	go func() {
 		defer close(leaseDone)
-		sc.lease.Run(ctx)
+		// The leader owns the Layer-2 bus subscriptions and the instance-role
+		// converge pass. A node that can do neither stands down rather than hold
+		// the lease and process nothing.
+		kvsync.RunElection(ctx, sc.lease, sc.onGained, sc.unsubscribeBus)
 	}()
 
 	// Two loops rather than four tickers, split by what wakes them. Services are

spinifex/domains/ecs/scheduler_test.go

@@ -6,7 +6,9 @@ package ecs
 
 import (
 	"context"
+	"sync"
 	"testing"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ecs"
@@ -14,26 +16,67 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/nats-io/nats.go"
 )
 
+// testScheduler is holder's scheduler on a sync bucket of its own, as a
+// separate daemon's would be.
+func testScheduler(t *testing.T, nc *nats.Conn, svc *Service, holder string) *Scheduler {
+	t.Helper()
+	return NewScheduler(nc, svc, NewSchedulerLease(kvstore.OpenSyncBucket(testutil.NewJetStream(t, nc)), holder))
+}
+
+// electOnce runs sc's election with the scheduler's own callbacks and returns
+// what its first onGained returned. stop ends the election and waits for it.
+func electOnce(t *testing.T, sc *Scheduler) (stop func(), gainedErr error) {
+	t.Helper()
+	ctx, cancel := context.WithCancel(t.Context())
+	gained := make(chan error, 1)
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		kvsync.RunElection(ctx, sc.lease, func(ctx context.Context) error {
+			err := sc.onGained(ctx)
+			gained <- err
+			return err
+		}, sc.unsubscribeBus)
+	}()
+	stop = func() { cancel(); <-done }
+	t.Cleanup(stop)
+	select {
+	case gainedErr = <-gained:
+	case <-time.After(10 * time.Second):
+		t.Fatal("the election never gained the lease")
+	}
+	return sync.OnceFunc(stop), gainedErr
+}
+
+// requireLeaseKeyAbsent fails unless no node holds the scheduler's key.
+func requireLeaseKeyAbsent(t *testing.T, nc *nats.Conn, msg string) {
+	t.Helper()
+	_, _, err := kvstore.OpenSyncBucket(testutil.NewJetStream(t, nc)).Load(t.Context(), KeyScheduler)
+	require.ErrorIs(t, err, kvsync.ErrNotFound, msg)
+}
+
 // TestScheduler_StandsDownWithoutIAM is the fleet outage this guards: a leader
 // that cannot converge instance roles holds the lease, writes no sts:AssumeRole
 // grant, and every agent's credentials lapse an hour later.
 func TestScheduler_StandsDownWithoutIAM(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
+	_, nc, _ := testutil.StartTestJetStream(t)
 	svc := NewService(nc, testRegion, "")
-	sc := NewScheduler(nc, svc, "holder-a")
+	sc := testScheduler(t, nc, svc, "holder-a")
-	require.NoError(t, sc.leaseErr)
 
-	assert.False(t, sc.lease.TryAcquire(t.Context()), "a node without IAM must refuse the lease")
+	stop, err := electOnce(t, sc)
+	require.Error(t, err, "a node without IAM must refuse the lease")
+	stop()
 	assert.False(t, sc.lease.Held())
 	assert.Zero(t, sc.subCount())
 
 	// The key must be free, or a healthy node waits out the full TTL.
-	kv, err := InitLeaderBucket(t.Context(), js)
+	requireLeaseKeyAbsent(t, nc, "a node that stood down left its key")
-	require.NoError(t, err)
-	_, err = kv.Get(t.Context(), schedulerLeaderKey)
-	require.Error(t, err)
 }
 
 // TestConvergeInstanceRoles_RewritesProvisionedAccounts pins the pass that
@@ -49,7 +92,7 @@ func TestConvergeInstanceRoles_RewritesProvisionedAccounts(t *testing.T) {
 	require.NoError(t, err)
 
 	iam.RolePolicies[ecsInstanceRoleName] = `{"Version":"2012-10-17","Statement":[]}`
-	sc := NewScheduler(nc, svc, "holder-a")
+	sc := testScheduler(t, nc, svc, "holder-a")
 	require.NoError(t, sc.convergeInstanceRoles(t.Context()))
 
 	assert.Contains(t, iam.RolePolicies[ecsInstanceRoleName], "sts:AssumeRole")
@@ -65,7 +108,7 @@ func TestConvergeInstanceRoles_SkipsAccountsWithoutTheRole(t *testing.T) {
 	_, err := svc.CreateCluster(context.Background(), &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
 	require.NoError(t, err)
 
-	sc := NewScheduler(nc, svc, "holder-a")
+	sc := testScheduler(t, nc, svc, "holder-a")
 	require.NoError(t, sc.convergeInstanceRoles(t.Context()))
 
 	assert.Zero(t, iam.CreateRoleCalls)
@@ -77,7 +120,7 @@ func TestConvergeInstanceRoles_SkipsAccountsWithoutTheRole(t *testing.T) {
 // caller that reaches the pass without going through leader election.
 func TestConvergeInstanceRoles_NilIAMIsAnError(t *testing.T) {
 	_, nc, _ := testutil.StartTestJetStream(t)
-	sc := NewScheduler(nc, NewService(nc, testRegion, ""), "holder-a")
+	sc := testScheduler(t, nc, NewService(nc, testRegion, ""), "holder-a")
 
 	err := sc.convergeInstanceRoles(t.Context())
 	require.Error(t, err)
@@ -94,3 +137,59 @@ func TestECSInstanceRolePolicyDoc_GrantsAssumeRole(t *testing.T) {
 	assert.Contains(t, doc, `"ecs:*"`)
 	assert.Contains(t, doc, `"ecr:GetAuthorizationToken"`)
 }
+
+// containerInstanceStatus is i-1's status in the web cluster.
+func containerInstanceStatus(t *testing.T, svc *Service) string {
+	t.Helper()
+	di, err := svc.DescribeContainerInstances(context.Background(), &ecs.DescribeContainerInstancesInput{
+		Cluster: aws.String("web"), ContainerInstances: []*string{aws.String("i-1")},
+	}, testAccountID)
+	require.NoError(t, err)
+	return aws.StringValue(di.ContainerInstances[0].Status)
+}
+
+// A leader that loses the lease must drop the bus and stop reaping, or two
+// nodes would mark instances DRAINING and stop their tasks at once. Re-elected,
+// it takes both back up.
+func TestScheduler_LosingTheLeaseDropsTheBusAndTheReaperUntilReElection(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	svc := NewService(nc, testRegion, "").WithDeps(Deps{IAM: newStubIAM()})
+	_, err := svc.CreateCluster(context.Background(), &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
+	require.NoError(t, err)
+	registerInstance(t, svc, "web", "i-1", 1024, 2048)
+	kv, err := svc.bucket(t.Context(), testAccountID)
+	require.NoError(t, err)
+	var rec InstanceRecord
+	_, err = taskdefinition.GetJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec)
+	require.NoError(t, err)
+	rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
+
+	store := kvstore.OpenSyncBucket(testutil.NewJetStream(t, nc))
+	sc := NewScheduler(nc, svc, kvsync.NewLease(store, KeyScheduler, "holder-a", 300*time.Millisecond))
+	ctx, cancel := context.WithCancel(t.Context())
+	done := make(chan struct{})
+	go func() { defer close(done); kvsync.RunElection(ctx, sc.lease, sc.onGained, sc.unsubscribeBus) }()
+	t.Cleanup(func() { cancel(); <-done })
+
+	held := func() bool { return sc.lease.Held() && sc.subCount() == 4 }
+	require.Eventually(t, held, 5*time.Second, 10*time.Millisecond, "the leader subscribes to the four bus subjects")
+
+	syncKV, err := store.KV(t.Context())
+	require.NoError(t, err)
+	_, err = syncKV.Put(t.Context(), KeyScheduler, []byte(`{"holder":"holder-b","generation":0}`))
+	require.NoError(t, err)
+	require.Eventually(t, func() bool { return !sc.lease.Held() && sc.subCount() == 0 }, 5*time.Second, 10*time.Millisecond,
+		"a lost lease must drop every bus subscription")
+
+	revisit, err := sc.timerPass(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, leaseRefresh, revisit)
+	assert.Equal(t, "ACTIVE", containerInstanceStatus(t, svc), "a node that lost the lease must not reap")
+
+	require.NoError(t, syncKV.Delete(t.Context(), KeyScheduler))
+	require.Eventually(t, held, 5*time.Second, 10*time.Millisecond, "re-elected, the bus comes back")
+	_, err = sc.timerPass(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, "DRAINING", containerInstanceStatus(t, svc), "the re-elected leader reaps again")
+}

spinifex/domains/ecs/service_test.go

@@ -14,6 +14,7 @@ import (
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -571,7 +572,7 @@ func TestScheduler_ReapBucket_StopsStaleInstanceTasks(t *testing.T) {
 	rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
 	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
 
-	sc := NewScheduler(nc, svc, "test-holder")
+	sc := testScheduler(t, nc, svc, "test-holder")
 	sc.reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
 
 	di, err := svc.DescribeContainerInstances(context.Background(), &ecs.DescribeContainerInstancesInput{
@@ -586,32 +587,38 @@ func TestScheduler_ReapBucket_StopsStaleInstanceTasks(t *testing.T) {
 	assert.Equal(t, stoppedReasonReaped, aws.StringValue(dt.Tasks[0].StoppedReason))
 }
 
-// Leader election: first holder wins Create; a second holder is rejected.
+// Leader election: the first holder wins; a second node, and a second attempt
+// in the same process, are both refused while it leads.
 func TestScheduler_AcquireLease_SingleLeader(t *testing.T) {
 	_, nc, _ := testutil.StartTestJetStream(t)
 	svc := NewService(nc, testRegion, "").WithDeps(Deps{IAM: newStubIAM()})
-	a := NewScheduler(nc, svc, "holder-a")
+	a := testScheduler(t, nc, svc, "holder-a")
-	b := NewScheduler(nc, svc, "holder-b")
+	b := testScheduler(t, nc, svc, "holder-b")
-	require.True(t, a.lease.TryAcquire(t.Context()))
+	tok, err := a.lease.TryLock(t.Context())
-	assert.False(t, b.lease.TryAcquire(t.Context()))
+	require.NoError(t, err)
-	assert.True(t, a.lease.TryAcquire(t.Context()), "a holder re-attempting keeps leadership")
+	t.Cleanup(func() { a.lease.Unlock(context.Background(), tok) })
+
+	_, err = b.lease.TryLock(t.Context())
+	assert.ErrorAs(t, err, new(*kvsync.HeldError))
+	_, err = a.lease.TryLock(t.Context())
+	assert.ErrorAs(t, err, new(*kvsync.HeldError), "a second attempt in the leading process is refused")
+	assert.True(t, a.lease.Held(), "a refused second attempt keeps leadership")
 }
 
-// Leadership owns the Layer-2 bus subscriptions. This pins the wiring — gaining
+// Leadership owns the Layer-2 bus subscriptions. This pins the wiring: gaining
-// the lease subscribes, losing it unsubscribes — which is the regression this
+// the lease subscribes, losing it unsubscribes.
-// conversion could introduce and the one thing kvlease cannot check itself.
 func TestScheduler_LeadershipWiresBusSubscriptions(t *testing.T) {
 	_, nc, _ := testutil.StartTestJetStream(t)
 	svc := NewService(nc, testRegion, "").WithDeps(Deps{IAM: newStubIAM()})
-	sc := NewScheduler(nc, svc, "holder-a")
+	sc := testScheduler(t, nc, svc, "holder-a")
-	require.NoError(t, sc.leaseErr)
 
 	require.Zero(t, sc.subCount(), "subscriptions must not exist before election")
 
-	require.True(t, sc.lease.TryAcquire(t.Context()))
+	stop, err := electOnce(t, sc)
+	require.NoError(t, err)
 	assert.Equal(t, 4, sc.subCount(), "leader must own the register, heartbeat, task-state and reconcile-wake subscriptions")
 
-	sc.lease.Release(t.Context())
+	stop()
 	assert.Zero(t, sc.subCount(), "a released leader must drop its subscriptions")
 }
 
@@ -620,20 +627,21 @@ func TestScheduler_LeadershipWiresBusSubscriptions(t *testing.T) {
 func TestScheduler_LoserHoldsNoSubscriptions(t *testing.T) {
 	_, nc, _ := testutil.StartTestJetStream(t)
 	svc := NewService(nc, testRegion, "").WithDeps(Deps{IAM: newStubIAM()})
-	a := NewScheduler(nc, svc, "holder-a")
+	a := testScheduler(t, nc, svc, "holder-a")
-	b := NewScheduler(nc, svc, "holder-b")
+	b := testScheduler(t, nc, svc, "holder-b")
 
-	require.True(t, a.lease.TryAcquire(t.Context()))
+	_, err := electOnce(t, a)
-	require.False(t, b.lease.TryAcquire(t.Context()))
+	require.NoError(t, err)
+	_, err = b.lease.TryLock(t.Context())
+	require.ErrorAs(t, err, new(*kvsync.HeldError))
 	assert.Zero(t, b.subCount())
 	assert.Equal(t, 4, a.subCount())
 }
 
 func TestScheduler_RunReleasesLeaseOnShutdown(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
+	_, nc, _ := testutil.StartTestJetStream(t)
 	svc := NewService(nc, testRegion, "").WithDeps(Deps{IAM: newStubIAM()})
-	sc := NewScheduler(nc, svc, "holder-a")
+	sc := testScheduler(t, nc, svc, "holder-a")
-	require.NoError(t, sc.leaseErr)
 
 	ctx, cancel := context.WithCancel(t.Context())
 	done := make(chan struct{})
@@ -643,10 +651,7 @@ func TestScheduler_RunReleasesLeaseOnShutdown(t *testing.T) {
 	cancel()
 	<-done
 
-	kv, err := InitLeaderBucket(t.Context(), js)
+	requireLeaseKeyAbsent(t, nc, "Run returned with the lease key still present")
-	require.NoError(t, err)
-	_, err = kv.Get(t.Context(), schedulerLeaderKey)
-	require.Error(t, err, "Run returned with the lease key still present")
 }
 
 // --- GPU placement dimension (Epic C2) ---

spinifex/domains/ecs/store.go

@@ -4,9 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"time"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
@@ -19,18 +17,11 @@ import (
 // cluster, task-definition, task, service, and container-instance state. It is
 // created lazily on first cluster touch (no daemon-boot pre-creation) so accounts
 // without any ECS clusters never grow a bucket (ecs-v1.md Q2).
-//
+
-// Shared leader bucket "spinifex-ecs-leader" holds 60s-TTL CAS locks keyed by
-// "{accountID}/{clusterName}" for the per-cluster scheduler leader election
-// (Q3). Created once at daemon boot.
 const (
 	KVBucketECSAccountPrefix  = "ecs-account-"
 	KVBucketECSAccountVersion = 1
 	KVBucketECSAccountHistory = 1
-
-	KVBucketECSLeader        = "spinifex-ecs-leader"
-	KVBucketECSLeaderVersion = 1
-	KVBucketECSLeaderTTL     = 60 * time.Second
 )
 
 // Key-path helpers for the per-account bucket. The layout matches ecs-v1.md Q2.
@@ -119,15 +110,15 @@ func CapacityProviderKey(name string) string {
 	return CapacityProvidersPrefix() + name
 }
 
-// Store is the per-daemon ECS KV handle. Per-account and leader buckets are
+// Store is the per-daemon ECS KV handle. Per-account buckets are accessed via
-// accessed via the package-level factories below.
+// the package-level factories below.
 type Store struct {
 	nc *nats.Conn
 }
 
 // NewStore constructs a Store bound to the supplied NATS connection. It does not
 // touch JetStream — per-account buckets are created lazily by
-// GetOrCreateAccountBucket and the leader bucket by InitLeaderBucket.
+// GetOrCreateAccountBucket.
 func NewStore(nc *nats.Conn) (*Store, error) {
 	if nc == nil {
 		return nil, errors.New("ecs store: nats connection is nil")
@@ -185,13 +176,3 @@ func GetOrCreateAccountBucket(ctx context.Context, js jetstream.JetStream, accou
 	}
 	return kv, nil
 }
-
-// InitLeaderBucket creates (or attaches to) the shared spinifex-ecs-leader bucket
-// used for per-cluster scheduler leader-lease CAS locks.
-func InitLeaderBucket(ctx context.Context, js jetstream.JetStream) (jetstream.KeyValue, error) {
-	return kvlease.OpenBucket(ctx, js, kvlease.BucketConfig{
-		Name:    KVBucketECSLeader,
-		TTL:     KVBucketECSLeaderTTL,
-		Version: KVBucketECSLeaderVersion,
-	})
-}

spinifex/domains/ecs/store_test.go

@@ -1,13 +1,9 @@
 package ecs
 
 import (
-	"encoding/json"
-	"strings"
 	"testing"
-	"time"
 
 	"github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -46,71 +42,6 @@ func TestGetOrCreateAccountBucket_Idempotent(t *testing.T) {
 	assert.Equal(t, kv1.Bucket(), kv2.Bucket())
 }
 
-func TestInitLeaderBucket_Idempotent(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	js := testutil.NewJetStream(t, nc)
-
-	kv1, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
-	require.NotNil(t, kv1)
-	assert.Equal(t, KVBucketECSLeader, kv1.Bucket())
-
-	kv2, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
-	require.NotNil(t, kv2)
-	assert.Equal(t, KVBucketECSLeader, kv2.Bucket())
-}
-
-// The scheduler calls InitLeaderBucket on every tick. Creating first and falling
-// back on already-exists reaches the same handle, so idempotency alone cannot
-// tell the two orders apart — only the API traffic can.
-//
-// On a multi-node cluster the stored bucket is replicated while this call asks
-// for the default, and JetStream answers the mismatch with an error rather than
-// a no-op: 10058, once per tick, forever. That is invisible from the caller and
-// showed up only as a 16% JetStream API error rate on the meta leader.
-func TestInitLeaderBucket_AttachesWithoutCreating(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
-
-	// Watch from before the first call. The server queues the API advisory after
-	// the reply the caller waits on, so subscribing once the bucket exists can
-	// still catch the create that first call legitimately made.
-	creates := make(chan string, 8)
-	sub, err := nc.Subscribe("$JS.EVENT.ADVISORY.API", func(m *nats.Msg) {
-		var adv struct {
-			Subject string `json:"subject"`
-		}
-		if json.Unmarshal(m.Data, &adv) == nil &&
-			strings.HasPrefix(adv.Subject, "$JS.API.STREAM.CREATE.") {
-			creates <- adv.Subject
-		}
-	})
-	require.NoError(t, err)
-	defer func() { _ = sub.Unsubscribe() }()
-	require.NoError(t, nc.Flush())
-
-	_, err = InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
-
-	// Take the first, legitimate create off the channel before the call under
-	// test, so anything left there afterwards can only have come from it.
-	select {
-	case <-creates:
-	case <-time.After(5 * time.Second):
-		t.Fatal("the first InitLeaderBucket must create the bucket")
-	}
-
-	_, err = InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
-	require.NoError(t, nc.Flush())
-
-	select {
-	case subject := <-creates:
-		t.Fatalf("InitLeaderBucket issued %s on an existing bucket; attach before create", subject)
-	case <-time.After(250 * time.Millisecond):
-	}
-}
-
 // Key-path helpers must produce the ecs-v1.md Q2 layout exactly: prefixes are
 // what the List* enumerations watch, so a drift here silently breaks listing.
 func TestKeyPaths(t *testing.T) {

spinifex/domains/eks/awsapi/gateway_wire_test.go

@@ -14,6 +14,7 @@ import (
 	ekscluster "github.com/mulgadc/spinifex/spinifex/domains/eks/cluster"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -55,7 +56,7 @@ func TestEKSRequest_InternalAddonsWireBody(t *testing.T) {
 		ControlPlaneNodes: []ekscluster.ControlPlaneNode{{InstanceID: cpInstanceID}},
 	}))
 
-	svc, err := ekscluster.NewEKSServiceImpl(ekscluster.EKSServiceDeps{NATSConn: nc, Region: authzRegion})
+	svc, err := ekscluster.NewEKSServiceImpl(ekscluster.EKSServiceDeps{NATSConn: nc, Region: authzRegion, Leases: ekscluster.NewClusterLeases(kvstore.OpenSyncBucket(js), "test")})
 	require.NoError(t, err)
 	sub, err := nc.Subscribe("eks.ListStagedAddonManifests", func(m *nats.Msg) {
 		acct := natsmsg.AccountIDFromMsg(m)
@@ -112,7 +113,7 @@ func TestEKSRequest_InternalWorkersWireBody(t *testing.T) {
 		require.NoError(t, ekscluster.PutNodegroupRecord(t.Context(), kv, rec))
 	}
 
-	svc, err := ekscluster.NewEKSServiceImpl(ekscluster.EKSServiceDeps{NATSConn: nc, Region: authzRegion})
+	svc, err := ekscluster.NewEKSServiceImpl(ekscluster.EKSServiceDeps{NATSConn: nc, Region: authzRegion, Leases: ekscluster.NewClusterLeases(kvstore.OpenSyncBucket(js), "test")})
 	require.NoError(t, err)
 	sub, err := nc.Subscribe("eks.ListManagedWorkers", func(m *nats.Msg) {
 		acct := natsmsg.AccountIDFromMsg(m)

spinifex/domains/eks/cluster/cluster_reconciler.go

@@ -16,8 +16,8 @@ import (
 	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/reconciler"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
 
 	"github.com/nats-io/nats.go"
@@ -25,8 +25,7 @@ import (
 )
 
 const (
-	defaultReconcileLeaseRefresh = 20 * time.Second
+	defaultReconcileInterval = 30 * time.Second
-	defaultReconcileInterval     = 30 * time.Second
 	// reconcileResync is the drift backstop for a loop whose signals now arrive as
 	// watch updates and state reports. It also bounds every deadline below it, so
 	// the 15-minute create timeout is served by the resync rather than by a timer.
@@ -123,6 +122,13 @@ type natsSubscriber interface {
 // ErrReconcilerLeaseLost is returned by Run when the leader lease is lost mid-loop.
 var ErrReconcilerLeaseLost = errors.New("eks: reconciler lease lost")
 
+// closedLost stands in for the Lost channel of a reconciler that never acquired.
+var closedLost = func() <-chan struct{} {
+	ch := make(chan struct{})
+	close(ch)
+	return ch
+}()
+
 // ErrReconcilerClusterDeleting is returned by Run when meta.Status is DELETING;
 // DeleteCluster owns teardown so the reconciler exits.
 var ErrReconcilerClusterDeleting = errors.New("eks: cluster deleting")
@@ -138,15 +144,12 @@ type HTTPDoer interface {
 // ClusterReconciler drives one cluster's CREATING → ACTIVE → DELETING lifecycle
 // under a leader lease. One instance per (accountID, clusterName).
 type ClusterReconciler struct {
-	leaderKV    jetstream.KeyValue
 	acctKV      jetstream.KeyValue
 	accountID   string
 	clusterName string
-	holderID    string
 	healthURL   string
 
-	lease         *kvlease.Lease
+	lease         *kvsync.Lease
-	leaseRefresh  time.Duration
 	interval      time.Duration
 	healthTimeout time.Duration
 	createTimeout time.Duration
@@ -166,6 +169,9 @@ type ClusterReconciler struct {
 	// terminal records why the loop is ending — a terminal cluster status or a
 	// lost lease — and stopRun cancels it. reconciler.Run has no exit condition
 	// of its own, so ending it is cancelling it, and this is what Run reports.
+	// lost is the Lost channel of the acquisition AcquireLease took.
+	lost <-chan struct{}
+
 	terminalMu sync.Mutex
 	terminal   error
 	stopRun    context.CancelFunc
@@ -226,11 +232,6 @@ func WithReconcileInterval(d time.Duration) ReconcilerOption {
 	return func(r *ClusterReconciler) { r.interval = d }
 }
 
-// WithLeaseRefresh overrides the leader-lease refresh period (should be < KVBucketEKSLeaderTTL).
-func WithLeaseRefresh(d time.Duration) ReconcilerOption {
-	return func(r *ClusterReconciler) { r.leaseRefresh = d }
-}
-
 // WithHealthzTimeout overrides the per-probe HTTP timeout.
 func WithHealthzTimeout(d time.Duration) ReconcilerOption {
 	return func(r *ClusterReconciler) { r.healthTimeout = d }
@@ -335,12 +336,12 @@ func WithEtcdResetRecovery(grace, backoff time.Duration, maxAttempts int) Reconc
 	}
 }
 
-// NewClusterReconciler constructs a ClusterReconciler. healthURL is the
+// NewClusterReconciler constructs a ClusterReconciler that leads while it holds
-// fully-qualified probe target (e.g. https://nlb-dns:443/healthz) or empty
+// lease, the cluster's lease from ClusterLeases. healthURL is the probe target
-// to skip /healthz probing (CREATING transitions on bootstrap state alone).
+// (e.g. https://nlb-dns:443/healthz) or empty to skip /healthz probing.
-func NewClusterReconciler(leaderKV, acctKV jetstream.KeyValue, accountID, clusterName, holderID, healthURL string, opts ...ReconcilerOption) (*ClusterReconciler, error) {
+func NewClusterReconciler(lease *kvsync.Lease, acctKV jetstream.KeyValue, accountID, clusterName, healthURL string, opts ...ReconcilerOption) (*ClusterReconciler, error) {
-	if leaderKV == nil {
+	if lease == nil {
-		return nil, errors.New("eks: NewClusterReconciler nil leaderKV")
+		return nil, errors.New("eks: NewClusterReconciler nil lease")
 	}
 	if acctKV == nil {
 		return nil, errors.New("eks: NewClusterReconciler nil acctKV")
@@ -351,17 +352,12 @@ func NewClusterReconciler(leaderKV, acctKV jetstream.KeyValue, accountID, cluste
 	if clusterName == "" {
 		return nil, errors.New("eks: NewClusterReconciler empty clusterName")
 	}
-	if holderID == "" {
-		return nil, errors.New("eks: NewClusterReconciler empty holderID")
-	}
 	r := &ClusterReconciler{
-		leaderKV:        leaderKV,
 		acctKV:          acctKV,
 		accountID:       accountID,
 		clusterName:     clusterName,
-		holderID:        holderID,
 		healthURL:       healthURL,
-		leaseRefresh:    defaultReconcileLeaseRefresh,
+		lease:           lease,
 		interval:        defaultReconcileInterval,
 		healthTimeout:   defaultHealthzTimeout,
 		createTimeout:   defaultCreateTimeout,
@@ -390,42 +386,23 @@ func NewClusterReconciler(leaderKV, acctKV jetstream.KeyValue, accountID, cluste
 	for _, o := range opts {
 		o(r)
 	}
-	lease, err := kvlease.New(kvlease.Config{
-		Name:   "eks/cluster-reconciler",
-		Bucket: kvlease.StaticBucket(leaderKV),
-		Key:    reconcilerLeaderKey(accountID, clusterName),
-		Holder: holderID,
-		Attrs:  []any{"account", accountID, "cluster", clusterName},
-		TTL:    KVBucketEKSLeaderTTL,
-		Renew:  r.leaseRefresh,
-	})
-	if err != nil {
-		return nil, fmt.Errorf("cluster reconciler lease: %w", err)
-	}
-	r.lease = lease
 	return r, nil
 }
 
-func reconcilerLeaderKey(accountID, clusterName string) string {
+// AcquireLease takes the cluster's lease. Returns (release, true) on success;
-	return accountID + "/" + clusterName
+// (nil, false) when another holder, or another run in this process, has it.
-}
-
-// AcquireLease attempts a CAS Create on the leader bucket. Returns
-// (release, true) on success; (nil, false) otherwise. The bucket TTL reaps
-// stale leases when release does not run.
 func (r *ClusterReconciler) AcquireLease(ctx context.Context) (func(), bool) {
-	if !r.lease.TryAcquire(ctx) {
+	release, lost, ok := tryLease(ctx, r.lease)
-		return nil, false
+	if ok {
+		r.lost = lost
 	}
-	return func() { r.lease.Release(ctx) }, true
+	return release, ok
 }
 
 // Run drives reconcile until ctx is cancelled, the lease is lost, or the cluster
-// reaches a terminal state. Caller must AcquireLease first.
+// reaches a terminal state. Caller must AcquireLease first; without it Run
+// returns ErrReconcilerLeaseLost at once.
 func (r *ClusterReconciler) Run(ctx context.Context) error {
-	if r.lease == nil {
-		return errors.New("ClusterReconciler: Run called without AcquireLease")
-	}
 	runCtx, stop := context.WithCancel(ctx)
 	defer stop()
 	r.stopRun = stop
@@ -463,11 +440,16 @@ func (r *ClusterReconciler) Run(ctx context.Context) error {
 	}
 
 	// The lease turning over is not a KV write this loop watches, so losing it has
-	// to end the loop from outside it.
+	// to end the loop from outside it. The channel is the acquisition's own, taken
+	// with its token, so a later acquisition's cannot stand in for it.
+	lost := r.lost
+	if lost == nil {
+		lost = closedLost
+	}
 	go func() {
 		select {
 		case <-runCtx.Done():
-		case <-r.lease.Lost():
+		case <-lost:
 			r.endRun(ErrReconcilerLeaseLost)
 		}
 	}()

spinifex/domains/eks/cluster/cluster_reconciler_state_test.go

@@ -21,14 +21,12 @@ func newStateReconcilerHarness(t *testing.T, opts ...ReconcilerOption) (*Cluster
 	t.Helper()
 	_, nc, _ := testutil.StartTestJetStream(t)
 	js := testutil.NewJetStream(t, nc)
-	leaderKV, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
 	acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
 	require.NoError(t, err)
 	require.NoError(t, PutClusterMeta(t.Context(), acctKV, sampleClusterMeta("alpha")))
 
 	base := []ReconcilerOption{WithStateSource(nc, StateSubject(testAccountID, "alpha"))}
-	r, err := NewClusterReconciler(leaderKV, acctKV, testAccountID, "alpha", "holder-1", "", append(base, opts...)...)
+	r, err := NewClusterReconciler(otherNodeLease(js, "holder-1"), acctKV, testAccountID, "alpha", "", append(base, opts...)...)
 	require.NoError(t, err)
 	return r, nc, acctKV
 }
@@ -146,7 +144,6 @@ func TestClusterReconciler_StateReportSubscriptionDrivesActive(t *testing.T) {
 	t.Parallel()
 	r, nc, acctKV := newStateReconcilerHarness(t,
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 	freshenClusterCreatedAt(t, acctKV)
 	seedBootstrapState(t, acctKV)

spinifex/domains/eks/cluster/cluster_reconciler_test.go

@@ -12,6 +12,8 @@ import (
 	"time"
 
 	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -54,39 +56,47 @@ func onePass(t *testing.T, r *ClusterReconciler) error {
 	return err
 }
 
-func newReconcilerHarness(t *testing.T, healthURL string, opts ...ReconcilerOption) (*ClusterReconciler, jetstream.KeyValue, jetstream.KeyValue) {
+func newReconcilerHarness(t *testing.T, healthURL string, opts ...ReconcilerOption) (*ClusterReconciler, jetstream.JetStream, jetstream.KeyValue) {
+	t.Helper()
+	return newReconcilerHarnessTTL(t, clusterLeaseTTL, healthURL, opts...)
+}
+
+// newReconcilerHarnessTTL is newReconcilerHarness with a lease of the given ttl,
+// for a test that has to see a renewal.
+func newReconcilerHarnessTTL(t *testing.T, ttl time.Duration, healthURL string, opts ...ReconcilerOption) (*ClusterReconciler, jetstream.JetStream, jetstream.KeyValue) {
 	t.Helper()
 	_, nc, _ := testutil.StartTestJetStream(t)
 	js := testutil.NewJetStream(t, nc)
-	leaderKV, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
 	acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
 	require.NoError(t, err)
 	require.NoError(t, PutClusterMeta(t.Context(), acctKV, sampleClusterMeta("alpha")))
 
-	r, err := NewClusterReconciler(leaderKV, acctKV, testAccountID, "alpha", "holder-1", healthURL, opts...)
+	leases := kvsync.NewLeaseGroup(kvstore.OpenSyncBucket(js), KeyPrefixCluster, "holder-1", ttl)
+	r, err := NewClusterReconciler(leases.Get(clusterLeaseName(testAccountID, "alpha")), acctKV, testAccountID, "alpha", healthURL, opts...)
 	require.NoError(t, err)
-	return r, leaderKV, acctKV
+	return r, js, acctKV
+}
+
+// otherNodeLease is the cluster's reconciler lease as another node sees it.
+func otherNodeLease(js jetstream.JetStream, holder string) *kvsync.Lease {
+	return NewClusterLeases(kvstore.OpenSyncBucket(js), holder).Get(clusterLeaseName(testAccountID, "alpha"))
 }
 
 func TestNewClusterReconciler_EmptyInputsRejected(t *testing.T) {
 	t.Parallel()
 	_, nc, _ := testutil.StartTestJetStream(t)
 	js := testutil.NewJetStream(t, nc)
-	leaderKV, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
 	acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
 	require.NoError(t, err)
+	lease := otherNodeLease(js, "h")
 
-	_, err = NewClusterReconciler(nil, acctKV, testAccountID, "alpha", "h", "")
+	_, err = NewClusterReconciler(nil, acctKV, testAccountID, "alpha", "")
 	require.Error(t, err)
-	_, err = NewClusterReconciler(leaderKV, nil, testAccountID, "alpha", "h", "")
+	_, err = NewClusterReconciler(lease, nil, testAccountID, "alpha", "")
 	require.Error(t, err)
-	_, err = NewClusterReconciler(leaderKV, acctKV, "", "alpha", "h", "")
+	_, err = NewClusterReconciler(lease, acctKV, "", "alpha", "")
 	require.Error(t, err)
-	_, err = NewClusterReconciler(leaderKV, acctKV, testAccountID, "", "h", "")
+	_, err = NewClusterReconciler(lease, acctKV, testAccountID, "", "")
-	require.Error(t, err)
-	_, err = NewClusterReconciler(leaderKV, acctKV, testAccountID, "alpha", "", "")
 	require.Error(t, err)
 }
 
@@ -99,21 +109,21 @@ func TestClusterReconciler_AcquireLeaseFirstHolderWins(t *testing.T) {
 	require.NotNil(t, release)
 	defer release()
 
-	// Re-acquiring from the same holder succeeds: the lease is already held.
+	// A second run in the same process is refused while the first holds it.
 	release2, ok2 := r.AcquireLease(t.Context())
-	assert.True(t, ok2)
+	assert.False(t, ok2)
-	assert.NotNil(t, release2)
+	assert.Nil(t, release2)
 }
 
 func TestClusterReconciler_AcquireLeaseSecondHolderLoses(t *testing.T) {
 	t.Parallel()
-	r1, leaderKV, acctKV := newReconcilerHarness(t, "")
+	r1, js, acctKV := newReconcilerHarness(t, "")
 
 	release, ok := r1.AcquireLease(t.Context())
 	require.True(t, ok)
 	defer release()
 
-	r2, err := NewClusterReconciler(leaderKV, acctKV, testAccountID, "alpha", "holder-2", "")
+	r2, err := NewClusterReconciler(otherNodeLease(js, "holder-2"), acctKV, testAccountID, "alpha", "")
 	require.NoError(t, err)
 	release2, ok2 := r2.AcquireLease(t.Context())
 	assert.False(t, ok2)
@@ -154,7 +164,6 @@ func TestClusterReconciler_CreatingTransitionsToActiveOnReadyAndHealthz(t *testi
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 	release, ok := r.AcquireLease(t.Context())
 	require.True(t, ok)
@@ -193,7 +202,6 @@ func TestClusterReconciler_CreatingStaysWhenBootstrapMissing(t *testing.T) {
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 	release, ok := r.AcquireLease(t.Context())
 	require.True(t, ok)
@@ -228,7 +236,6 @@ func TestClusterReconciler_CreatingStaysWhenJWKSUnverified(t *testing.T) {
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 	release, ok := r.AcquireLease(t.Context())
 	require.True(t, ok)
@@ -262,7 +269,6 @@ func TestClusterReconciler_CreatingStaysWhenHealthzFails(t *testing.T) {
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 	release, ok := r.AcquireLease(t.Context())
 	require.True(t, ok)
@@ -287,7 +293,6 @@ func TestClusterReconciler_CreatingTimesOutToFailed(t *testing.T) {
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 		WithCreateTimeout(1*time.Millisecond),
 	)
 	release, ok := r.AcquireLease(t.Context())
@@ -320,7 +325,6 @@ func TestClusterReconciler_CreatingWithinDeadlineStaysCreating(t *testing.T) {
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 		WithCreateTimeout(time.Hour),
 	)
 	release, ok := r.AcquireLease(t.Context())
@@ -345,7 +349,6 @@ func TestClusterReconciler_DeletingExitsLoop(t *testing.T) {
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 	release, ok := r.AcquireLease(t.Context())
 	require.True(t, ok)
@@ -363,11 +366,10 @@ func TestClusterReconciler_DeletingExitsLoop(t *testing.T) {
 func TestClusterReconciler_LostLeaseExitsLoop(t *testing.T) {
 	t.Parallel()
 	stub := &stubHTTPDoer{status: http.StatusOK}
-	r, leaderKV, acctKV := newReconcilerHarness(t,
+	r, js, acctKV := newReconcilerHarnessTTL(t, 300*time.Millisecond,
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Second),
-		WithLeaseRefresh(20*time.Millisecond),
 	)
 	release, ok := r.AcquireLease(t.Context())
 	require.True(t, ok)
@@ -381,16 +383,17 @@ func TestClusterReconciler_LostLeaseExitsLoop(t *testing.T) {
 	runErr := make(chan error, 1)
 	go func() { runErr <- r.Run(ctx) }()
 
-	// Steal the lease in another holder's name. The renewal loses its CAS at the
+	// Overwrite the key in another holder's name. The next renewal loses its
-	// stale revision, which closes Lost and exits Run.
+	// CAS at the stale revision, which closes Lost and exits Run.
-	time.Sleep(40 * time.Millisecond)
+	syncKV, err := kvstore.OpenSyncBucket(js).KV(t.Context())
-	_, err := leaderKV.Put(t.Context(), reconcilerLeaderKey(testAccountID, "alpha"), []byte("holder-2"))
+	require.NoError(t, err)
+	_, err = syncKV.Put(t.Context(), KeyPrefixCluster+clusterLeaseName(testAccountID, "alpha"), []byte(`{"holder":"holder-2","generation":0}`))
 	require.NoError(t, err)
 
 	select {
 	case err := <-runErr:
 		assert.ErrorIs(t, err, ErrReconcilerLeaseLost)
-	case <-time.After(2 * time.Second):
+	case <-time.After(5 * time.Second):
 		t.Fatal("Run did not exit after lease loss")
 	}
 }
@@ -402,7 +405,6 @@ func TestClusterReconciler_ActiveProbesAndWarnsOnHealthzFail(t *testing.T) {
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 	release, ok := r.AcquireLease(t.Context())
 	require.True(t, ok)
@@ -438,7 +440,6 @@ func TestClusterReconciler_ActiveHealthRecoversClearsIssue(t *testing.T) {
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 	release, ok := r.AcquireLease(t.Context())
 	require.True(t, ok)
@@ -468,9 +469,111 @@ func TestClusterReconciler_ProbeHealthzEmptyURLNoop(t *testing.T) {
 		"",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 
 	require.NoError(t, r.probeHealthz(context.Background()))
 	assert.EqualValues(t, 0, stub.calls.Load(), "no HTTP call when URL empty")
 }
+
+// putForeignHolder writes the cluster's lease key in another holder's name and
+// never renews it, standing in for a node that died holding it.
+func putForeignHolder(t *testing.T, js jetstream.JetStream) {
+	t.Helper()
+	syncKV, err := kvstore.OpenSyncBucket(js).KV(t.Context())
+	require.NoError(t, err)
+	_, err = syncKV.Put(t.Context(), KeyPrefixCluster+clusterLeaseName(testAccountID, "alpha"), []byte(`{"holder":"holder-2","generation":0}`))
+	require.NoError(t, err)
+}
+
+func requireStillRunning(t *testing.T, finished <-chan struct{}) {
+	t.Helper()
+	select {
+	case <-finished:
+		t.Fatal("the reconciler goroutine exited while its cluster still wants one")
+	default:
+	}
+}
+
+// A dead node's key is not cleared by the store, so the reconciler has to keep
+// asking until it has sat a TTL; one attempt would leave the cluster unreconciled.
+func TestRunClusterReconciler_TakesOverADeadHoldersKey(t *testing.T) {
+	t.Parallel()
+	r, js, acctKV := newReconcilerHarnessTTL(t, 300*time.Millisecond, "https://nlb.example/healthz",
+		WithHTTPClient(&stubHTTPDoer{status: http.StatusOK}), WithReconcileInterval(10*time.Second))
+	freshenClusterCreatedAt(t, acctKV)
+	seedBootstrapState(t, acctKV)
+	putForeignHolder(t, js)
+
+	release, finished, err := RunClusterReconciler(t.Context(), r.lease, acctKV, testAccountID, "alpha", "https://nlb.example/healthz",
+		WithHTTPClient(&stubHTTPDoer{status: http.StatusOK}), WithReconcileInterval(10*time.Second))
+	require.NoError(t, err, "a held lease is contended for, not an error")
+	defer release()
+
+	require.Eventually(t, r.lease.Held, 5*time.Second, 10*time.Millisecond, "the dead holder's key must be taken over")
+	requireStillRunning(t, finished)
+}
+
+// Losing the lease ends a Run, not the reconciler: it contends again and
+// resumes once it holds the lease once more.
+func TestRunClusterReconciler_ResumesAfterALostLease(t *testing.T) {
+	t.Parallel()
+	r, js, acctKV := newReconcilerHarnessTTL(t, 300*time.Millisecond, "https://nlb.example/healthz")
+	freshenClusterCreatedAt(t, acctKV)
+	seedBootstrapState(t, acctKV)
+	release, finished, err := RunClusterReconciler(t.Context(), r.lease, acctKV, testAccountID, "alpha", "https://nlb.example/healthz",
+		WithHTTPClient(&stubHTTPDoer{status: http.StatusOK}), WithReconcileInterval(10*time.Second))
+	require.NoError(t, err)
+	defer release()
+	require.Eventually(t, r.lease.Held, 5*time.Second, 10*time.Millisecond)
+
+	putForeignHolder(t, js)
+	require.Eventually(t, func() bool { return !r.lease.Held() }, 5*time.Second, 10*time.Millisecond, "the overwrite must cost the lease")
+	require.Eventually(t, r.lease.Held, 5*time.Second, 10*time.Millisecond, "the reconciler must win the lease back")
+	requireStillRunning(t, finished)
+}
+
+// A cluster that has left CREATING and ACTIVE wants no reconciler, so one that
+// cannot get the lease stops asking.
+func TestRunClusterReconciler_StopsContendingForADeletingCluster(t *testing.T) {
+	t.Parallel()
+	r, js, acctKV := newReconcilerHarnessTTL(t, 300*time.Millisecond, "https://nlb.example/healthz")
+	meta, err := GetClusterMeta(t.Context(), acctKV, "alpha")
+	require.NoError(t, err)
+	meta.Status = ClusterStatusDeleting
+	require.NoError(t, PutClusterMeta(t.Context(), acctKV, meta))
+	putForeignHolder(t, js)
+
+	release, finished, err := RunClusterReconciler(t.Context(), r.lease, acctKV, testAccountID, "alpha", "")
+	require.NoError(t, err)
+	defer release()
+	select {
+	case <-finished:
+	case <-time.After(5 * time.Second):
+		t.Fatal("the reconciler kept contending for a DELETING cluster")
+	}
+	assert.False(t, r.lease.Held())
+}
+
+// Run watches the Lost channel of the acquisition AcquireLease took. A later
+// acquisition of the same lease must not keep a Run alive whose own was lost.
+func TestClusterReconciler_RunWatchesItsOwnAcquisition(t *testing.T) {
+	t.Parallel()
+	r, js, acctKV := newReconcilerHarnessTTL(t, 300*time.Millisecond, "https://nlb.example/healthz",
+		WithHTTPClient(&stubHTTPDoer{status: http.StatusOK}), WithReconcileInterval(10*time.Second))
+	_, ok := r.AcquireLease(t.Context())
+	require.True(t, ok)
+
+	putForeignHolder(t, js)
+	require.Eventually(t, func() bool { return !r.lease.Held() }, 5*time.Second, 10*time.Millisecond)
+	syncKV, err := kvstore.OpenSyncBucket(js).KV(t.Context())
+	require.NoError(t, err)
+	require.NoError(t, syncKV.Delete(t.Context(), KeyPrefixCluster+clusterLeaseName(testAccountID, "alpha")))
+	tok, err := r.lease.TryLock(t.Context())
+	require.NoError(t, err, "a later acquisition of the same lease")
+	defer r.lease.Unlock(t.Context(), tok)
+
+	seedBootstrapState(t, acctKV)
+	ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+	defer cancel()
+	assert.ErrorIs(t, r.Run(ctx), ErrReconcilerLeaseLost, "the acquisition this Run was given is gone")
+}

spinifex/domains/eks/cluster/cluster_teardown.go

@@ -7,7 +7,6 @@ import (
 	"log/slog"
 	"time"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	telemetry "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
 	"github.com/nats-io/nats.go/jetstream"
@@ -22,12 +21,6 @@ var teardownConcurrency = 4
 // writer added refs to the record while it was tearing down.
 const maxTeardownPasses = 5
 
-// Teardown lease timing. TTL must match the leader bucket's TTL; vars for tests.
-var (
-	teardownLeaseTTL   = KVBucketEKSLeaderTTL
-	teardownLeaseRenew = defaultReconcileLeaseRefresh
-)
-
 var (
 	errTeardownNotRecorded = errors.New("eks: cluster is not DELETING, teardown refused")
 	errTeardownLeaseLost   = errors.New("eks: teardown lease lost")
@@ -35,45 +28,17 @@ var (
 	errLaunchInFlight      = errors.New("eks: cluster create launch still running")
 )
 
-// teardownLeaderKey is the per-cluster cross-node gate for a DELETING
-// teardown. It is distinct from reconcilerLeaderKey so a delete never contends
-// with the still-live reconciler, which holds that lease until it reads DELETING.
-func teardownLeaderKey(accountID, clusterName string) string {
-	return reconcilerLeaderKey(accountID, clusterName) + "/teardown"
-}
-
 // teardownLease is a held teardown lease: lost closes if renewal loses the key.
 type teardownLease struct {
 	release func()
 	lost    <-chan struct{}
 }
 
-// acquireTeardownLease takes the per-cluster teardown lease, renewing it on
+// acquireTeardownLease takes the per-cluster teardown lease, renewing it until
-// ctx. The lease re-adopts a key its own holder already has, so it serialises
+// released. It refuses when another node holds it or another goroutine here does.
-// nodes, not goroutines; startTeardown's in-process gate covers the latter. A
-// nil leaderKV (single-node/test) skips gating.
 func (s *EKSServiceImpl) acquireTeardownLease(ctx context.Context, accountID, clusterName string) (teardownLease, bool) {
-	if s.leaderKV == nil {
+	release, lost, ok := tryLease(ctx, s.leases.Get(teardownLeaseName(accountID, clusterName)))
-		return teardownLease{release: func() {}}, true
+	return teardownLease{release: release, lost: lost}, ok
-	}
-	key := teardownLeaderKey(accountID, clusterName)
-	lease, err := kvlease.New(kvlease.Config{
-		Name:   "eks/teardown",
-		Bucket: kvlease.StaticBucket(s.leaderKV),
-		Key:    key,
-		Holder: s.deps.HolderID,
-		Attrs:  []any{"account", accountID, "cluster", clusterName},
-		TTL:    teardownLeaseTTL,
-		Renew:  teardownLeaseRenew,
-	})
-	if err != nil {
-		slog.ErrorContext(ctx, "eks: teardown lease config invalid", "key", key, "err", err)
-		return teardownLease{}, false
-	}
-	if !lease.TryAcquire(ctx) {
-		return teardownLease{}, false
-	}
-	return teardownLease{release: func() { lease.Release(ctx) }, lost: lease.Lost()}, true
 }
 
 // teardownPassTimeout bounds one converger run, so a step that hangs cannot
@@ -81,11 +46,11 @@ func (s *EKSServiceImpl) acquireTeardownLease(ctx context.Context, accountID, cl
 var teardownPassTimeout = 15 * time.Minute
 
 // holdTeardown claims the right to tear a cluster down on this node: the
-// in-process gate, which the lease cannot provide because it re-adopts a key
+// in-process gate, which also bounds this node's teardowns, then the cross-node
-// its own node holds, then the cross-node lease. It refuses, without waiting,
+// lease. It refuses, without waiting, when either is held, or, if capped, when
-// when either is held, or, if capped, when the node is at teardownConcurrency.
+// the node is at teardownConcurrency.
-func (s *EKSServiceImpl) holdTeardown(accountID, name string, capped bool) (release func(), lost <-chan struct{}, ok bool) {
+func (s *EKSServiceImpl) holdTeardown(accountID, name string, capped bool) (release func(retire bool), lost <-chan struct{}, ok bool) {
-	key := teardownLeaderKey(accountID, name)
+	key := teardownLeaseName(accountID, name)
 	s.teardownMu.Lock()
 	if s.teardownRunning == nil {
 		s.teardownRunning = map[string]struct{}{}
@@ -110,7 +75,15 @@ func (s *EKSServiceImpl) holdTeardown(accountID, name string, capped bool) (rele
 		leave()
 		return nil, nil, false
 	}
-	return func() { lease.release(); leave() }, lease.lost, true
+	// Retiring drops the teardown lease before the gate opens: every Get of it
+	// is under the gate, so no later teardown of a recreated name can hold it.
+	return func(retire bool) {
+		lease.release()
+		if retire {
+			s.leases.Forget(key)
+		}
+		leave()
+	}, lease.lost, true
 }
 
 // startTeardown hands one DELETING cluster to a background converger and
@@ -122,7 +95,8 @@ func (s *EKSServiceImpl) startTeardown(accountID, name string) bool {
 		return false
 	}
 	s.teardownWG.Go(func() {
-		defer release()
+		deleted := false
+		defer func() { release(deleted) }()
 		ctx, cancel := context.WithTimeout(s.bgCtx, teardownPassTimeout)
 		defer cancel()
 		started := time.Now()
@@ -131,6 +105,7 @@ func (s *EKSServiceImpl) startTeardown(accountID, name string) bool {
 				"cluster", name, "account", accountID, "err", err)
 			return
 		}
+		deleted = true
 		slog.Info("eks: cluster teardown complete", "cluster", name, "account", accountID,
 			"duration_ms", telemetry.Millis(time.Since(started)))
 	})
@@ -211,6 +186,9 @@ func (s *EKSServiceImpl) convergeClusterDeletion(ctx context.Context, accountID,
 			return terr
 		}
 
+		// Forgotten while the record still blocks a same-name create, and after
+		// teardownClusterInfra stopped this node's reconciler.
+		s.leases.Forget(clusterLeaseName(accountID, name))
 		done, err := s.removeClusterRecord(ctx, acctKV, name, meta)
 		if err != nil {
 			if !errors.Is(err, errNodegroupsAttached) && !errors.Is(err, errTeardownNotRecorded) {

spinifex/domains/eks/cluster/delete_cluster_async_test.go

@@ -12,6 +12,8 @@ import (
 	"github.com/aws/aws-sdk-go/service/eks"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -132,22 +134,18 @@ func TestS8_EKSSecondDeleteAnswersFromTheRecordWithoutASecondTeardown(t *testing
 	assert.Len(t, f.inst.terminated(), 1, "the repeat request must start no teardown of its own")
 }
 
-// TestS8_EKSTeardownLeaseOutlivesTheRequest: the teardown lease renews on the
+// TestS8_EKSTeardownLeaseOutlivesTheRequest: the teardown lease renews on its
-// service's context, so cancelling the request context mid-teardown does not
+// own, so cancelling the request context mid-teardown does not let the lease
-// let the lease expire and a second node start a duplicate teardown.
+// lapse and a second node start a duplicate teardown.
 func TestS8_EKSTeardownLeaseOutlivesTheRequest(t *testing.T) {
 	f := newDeleteClusterFixture(t, "alpha")
-	_, nc, _ := testutil.StartTestJetStream(t)
+	// A short TTL keeps the wait brief while leaving several renewal chances
-	js := testutil.NewJetStream(t, nc)
+	// inside each TTL window.
-	// A short TTL with frequent renewals keeps the wait brief while leaving
-	// several renewal chances inside each TTL window.
 	const ttl = 400 * time.Millisecond
-	short, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: "eks-teardown-short-ttl", TTL: ttl})
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
-	require.NoError(t, err)
+	f.svc.leases = kvsync.NewLeaseGroup(kvstore.OpenSyncBucket(js), KeyPrefixCluster, "node-1", ttl)
-	f.svc.leaderKV = short
+	other := kvsync.NewLeaseGroup(kvstore.OpenSyncBucket(js), KeyPrefixCluster, "node-2", ttl).
-	origTTL, origRenew := teardownLeaseTTL, teardownLeaseRenew
+		Get(teardownLeaseName(testAccountID, "alpha"))
-	teardownLeaseTTL, teardownLeaseRenew = ttl, 50*time.Millisecond
-	t.Cleanup(func() { teardownLeaseTTL, teardownLeaseRenew = origTTL, origRenew })
 
 	entered, release := blockTerminate(t, f.inst)
 	ctx, cancel := context.WithCancel(context.Background())
@@ -155,19 +153,22 @@ func TestS8_EKSTeardownLeaseOutlivesTheRequest(t *testing.T) {
 	waitEntered(t, entered)
 	cancel()
 
+	// The other node first sees the key, then sees it again past a full TTL: it
+	// takes over only if no renewal moved the key in between.
+	_, err := other.TryLock(t.Context())
+	require.ErrorAs(t, err, new(*kvsync.HeldError))
 	time.Sleep(ttl * 5 / 2)
-	_, err = short.Create(t.Context(), teardownLeaderKey(testAccountID, "alpha"), []byte("node-2"))
+	_, err = other.TryLock(t.Context())
-	require.ErrorIs(t, err, jetstream.ErrKeyExists,
+	require.ErrorAs(t, err, new(*kvsync.HeldError),
 		"the running teardown must still hold its lease after the request context is gone")
 
 	release()
 	f.svc.WaitTeardowns()
 }
 
-// TestS8_EKSKickAndSameNodeSweepRunOneTeardown: the teardown lease re-adopts a
+// TestS8_EKSKickAndSameNodeSweepRunOneTeardown: a same-node reaper sweep must
-// key its own node holds, so only the in-process gate stops a same-node reaper
+// not start a second teardown beside the request's. Each member must be
-// sweep from starting a second teardown beside the request's. Each member must
+// terminated exactly once.
-// be terminated exactly once.
 func TestS8_EKSKickAndSameNodeSweepRunOneTeardown(t *testing.T) {
 	t.Parallel()
 	f := newDeleteClusterFixture(t, "alpha")

spinifex/domains/eks/cluster/delete_cluster_review2_test.go

@@ -180,7 +180,7 @@ func TestEKSBootstrapRunRemovesItsOwnEntryOnExit(t *testing.T) {
 	eventually(t, func() bool {
 		f.svc.bootstrapMu.Lock()
 		defer f.svc.bootstrapMu.Unlock()
-		_, ok := f.svc.bootstraps[reconcilerLeaderKey(testAccountID, "alpha")]
+		_, ok := f.svc.bootstraps[clusterLeaseName(testAccountID, "alpha")]
 		return !ok
 	}, "an exited bootstrap must remove its own entry")
 }

spinifex/domains/eks/cluster/delete_cluster_review_test.go

@@ -252,8 +252,7 @@ func TestS2_EKSFailedClusterReclaimRunsUnderTheTeardownLease(t *testing.T) {
 	require.NoError(t, err)
 	meta.Status = ClusterStatusFailed
 	require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta))
-	_, err = f.svc.leaderKV.Create(t.Context(), teardownLeaderKey(testAccountID, "alpha"), []byte("node-2"))
+	holdElsewhere(t, f.svc, teardownLeaseName(testAccountID, "alpha"))
-	require.NoError(t, err)
 
 	_, err = f.svc.CreateCluster(context.Background(), createInput("alpha"), testAccountID, "")
 	code, _, _ := awserrors.ResolveErrorDetail(err)

spinifex/domains/eks/cluster/delete_cluster_test.go

@@ -15,6 +15,8 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/systeminstance"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
@@ -83,6 +85,7 @@ func newEKSServiceFixture(t *testing.T) *eksServiceFixture {
 		GatewayBaseURL:      "https://gw.local:9999",
 		Region:              "us-east-1",
 		HolderID:            "node-1",
+		Leases:              NewClusterLeases(kvstore.OpenSyncBucket(js), "node-1"),
 		SystemGatewayURL:    "https://gw.local:9999",
 		SystemAccessKey:     "AKIAEXAMPLE",
 		SystemSecretKey:     "s3cr3t-key",
@@ -371,6 +374,33 @@ func TestDeleteCluster_EgressEIPReleaseRealErrorLeavesMeta(t *testing.T) {
 	assert.NotEmpty(t, meta.EgressEIPAllocationID, "alloc ID must remain for retry")
 }
 
+// otherNodeLeases is the cluster lease group as another node holding holder
+// sees it, on its own sync bucket over the same JetStream as svc.
+func otherNodeLeases(t *testing.T, svc *EKSServiceImpl, holder string) *kvsync.LeaseGroup {
+	t.Helper()
+	return NewClusterLeases(kvstore.OpenSyncBucket(testutil.NewJetStream(t, svc.deps.NATSConn)), holder)
+}
+
+// holdElsewhere has another node take the named lease until the test ends.
+func holdElsewhere(t *testing.T, svc *EKSServiceImpl, name string) {
+	t.Helper()
+	lease := otherNodeLeases(t, svc, "node-2").Get(name)
+	tok, err := lease.TryLock(t.Context())
+	require.NoError(t, err)
+	t.Cleanup(func() { lease.Unlock(context.Background(), tok) })
+}
+
+// leaseKeyHeld reports whether any node holds the named lease's key.
+func leaseKeyHeld(t *testing.T, svc *EKSServiceImpl, name string) bool {
+	t.Helper()
+	_, _, err := kvstore.OpenSyncBucket(testutil.NewJetStream(t, svc.deps.NATSConn)).Load(t.Context(), KeyPrefixCluster+name)
+	if errors.Is(err, kvsync.ErrNotFound) {
+		return false
+	}
+	require.NoError(t, err)
+	return true
+}
+
 // TestDeleteClusterLeaseHeldElsewhereLeavesTeardownToTheHolder: when another
 // node holds the teardown lease, DeleteCluster still records DELETING and
 // returns it, but runs no teardown of its own and leaves the record for the
@@ -379,8 +409,7 @@ func TestDeleteClusterLeaseHeldElsewhereLeavesTeardownToTheHolder(t *testing.T)
 	t.Parallel()
 	f := newDeleteClusterFixture(t, "alpha")
 
-	_, err := f.svc.leaderKV.Create(t.Context(), teardownLeaderKey(testAccountID, "alpha"), []byte("node-2"))
+	holdElsewhere(t, f.svc, teardownLeaseName(testAccountID, "alpha"))
-	require.NoError(t, err)
 
 	deleteAndSettle(t, f.svc, "alpha")
 
@@ -399,8 +428,73 @@ func TestDeleteClusterTeardownReleasesItsLease(t *testing.T) {
 
 	deleteAndSettle(t, f.svc, "alpha")
 
-	_, getErr := f.svc.leaderKV.Get(t.Context(), teardownLeaderKey(testAccountID, "alpha"))
+	assert.False(t, leaseKeyHeld(t, f.svc, teardownLeaseName(testAccountID, "alpha")),
-	assert.ErrorIs(t, getErr, jetstream.ErrKeyNotFound, "the teardown lease must be released after the teardown")
+		"the teardown lease must be released after the teardown")
+}
+
+// TestDeleteClusterForgetsTheClusterLeases: a deleted cluster's reconciler and
+// teardown leases leave the group, so a cluster recreated under the same name
+// starts from fresh ones.
+func TestDeleteClusterForgetsTheClusterLeases(t *testing.T) {
+	t.Parallel()
+	f := newDeleteClusterFixture(t, "alpha")
+	names := []string{clusterLeaseName(testAccountID, "alpha"), teardownLeaseName(testAccountID, "alpha")}
+	before := make([]*kvsync.Lease, len(names))
+	for i, name := range names {
+		before[i] = f.svc.leases.Get(name)
+	}
+
+	deleteAndSettle(t, f.svc, "alpha")
+
+	_, err := GetClusterMeta(t.Context(), f.kv, "alpha")
+	require.ErrorIs(t, err, ErrClusterNotFound, "the teardown must have completed")
+	for i, name := range names {
+		assert.NotSame(t, before[i], f.svc.leases.Get(name), "%s must be forgotten", name)
+	}
+}
+
+// TestDeleteClusterCleanupLeavesARecreatedClustersLease: once the record is
+// gone a same-name cluster may lead, and the old teardown's cleanup, still
+// running, must not release the lease it took.
+func TestDeleteClusterCleanupLeavesARecreatedClustersLease(t *testing.T) {
+	t.Parallel()
+	f := newDeleteClusterFixture(t, "alpha")
+	entered, release := blockTerminate(t, f.inst)
+	_, err := f.svc.DeleteCluster(context.Background(), deleteInput("alpha"), testAccountID)
+	require.NoError(t, err)
+	waitEntered(t, entered)
+
+	// Holding the gate's mutex parks the old teardown before its gate opens.
+	f.svc.teardownMu.Lock()
+	release()
+	require.Eventually(t, func() bool {
+		_, err := GetClusterMeta(t.Context(), f.kv, "alpha")
+		return errors.Is(err, ErrClusterNotFound)
+	}, 10*time.Second, 10*time.Millisecond, "the old teardown must remove the record")
+
+	replacement := f.svc.leases.Get(clusterLeaseName(testAccountID, "alpha"))
+	tok, err := replacement.TryLock(t.Context())
+	require.NoError(t, err, "the recreated cluster's reconciler takes the free lease")
+	f.svc.teardownMu.Unlock()
+	f.svc.WaitTeardowns()
+
+	assert.True(t, replacement.Held(), "the old teardown's cleanup released the recreated cluster's lease")
+	replacement.Unlock(t.Context(), tok)
+}
+
+// TestDeleteClusterKeepsTheLeasesOfAnIncompleteTeardown: a teardown that
+// fails leaves the record DELETING for retry, so its leases stay in the group.
+func TestDeleteClusterKeepsTheLeasesOfAnIncompleteTeardown(t *testing.T) {
+	t.Parallel()
+	f := newDeleteClusterFixture(t, "alpha")
+	f.eip.releaseErr = errors.New("AddressLimitExceeded")
+	name := teardownLeaseName(testAccountID, "alpha")
+	before := f.svc.leases.Get(name)
+
+	deleteAndSettle(t, f.svc, "alpha")
+
+	requireTeardownRetained(t, f.kv, "alpha")
+	assert.Same(t, before, f.svc.leases.Get(name))
 }
 
 func TestTeardownLeaseReleaseSurvivesCallerCancellation(t *testing.T) {
@@ -413,8 +507,7 @@ func TestTeardownLeaseReleaseSurvivesCallerCancellation(t *testing.T) {
 	cancel()
 	lease.release()
 
-	_, err := f.svc.leaderKV.Get(t.Context(), teardownLeaderKey(testAccountID, "alpha"))
+	assert.False(t, leaseKeyHeld(t, f.svc, teardownLeaseName(testAccountID, "alpha")))
-	require.ErrorIs(t, err, jetstream.ErrKeyNotFound)
 }
 
 // TestDeleteCluster_ManagedCPVPCReclaimsCustomerVPCSGs guards the contract: in

spinifex/domains/eks/cluster/eks_deleting_reaper_test.go

@@ -62,8 +62,7 @@ func TestDeletingReaperLeavesAHeldTeardownAlone(t *testing.T) {
 	t.Parallel()
 	f := newDeleteClusterFixture(t, "beta")
 	markDeleting(t, f, "beta", time.Second)
-	_, err := f.svc.leaderKV.Create(t.Context(), teardownLeaderKey(testAccountID, "beta"), []byte("node-2"))
+	holdElsewhere(t, f.svc, teardownLeaseName(testAccountID, "beta"))
-	require.NoError(t, err)
 
 	assert.Equal(t, 0, sweepAndSettle(t, f))
 

spinifex/domains/eks/cluster/eks_nodegroup_deleting_reaper.go

@@ -106,18 +106,22 @@ func (r *EKSNodegroupDeletingReaper) due(rec *NodegroupRecord) bool {
 // startNodegroupDelete runs one converger pass in the background and reports
 // whether it started; a pass already running here or on another node refuses.
 func (s *EKSServiceImpl) startNodegroupDelete(accountID string, kv jetstream.KeyValue, cluster, ng string) bool {
-	release, _, ok := s.holdTeardown(accountID, cluster+"/nodegroup/"+ng, false)
+	gate := cluster + "/nodegroup/" + ng
+	release, _, ok := s.holdTeardown(accountID, gate, false)
 	if !ok {
 		return false
 	}
 	s.teardownWG.Go(func() {
-		defer release()
+		deleted := false
+		defer func() { release(deleted) }()
 		ctx, cancel := context.WithTimeout(s.bgCtx, teardownPassTimeout)
 		defer cancel()
 		if err := s.convergeNodegroupDeletion(ctx, accountID, kv, cluster, ng); err != nil {
 			slog.Warn("eks: nodegroup delete incomplete, the record stays DELETING for retry",
 				"cluster", cluster, "nodegroup", ng, "account", accountID, "err", err)
+			return
 		}
+		deleted = true
 	})
 	return true
 }

spinifex/domains/eks/cluster/leases.go

@@ -0,0 +1,48 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+)
+
+// KeyPrefixCluster is the prefix of every per-cluster lease in the sync bucket.
+const KeyPrefixCluster = "eks.cluster."
+
+// clusterLeaseTTL is every per-cluster lease's time to live.
+const clusterLeaseTTL = 60 * time.Second
+
+// NewClusterLeases returns the per-cluster lease group for holder. The EKS
+// owner builds it once per process: a cluster's reconciler and teardown leases
+// are members of it, and a deleted cluster's are forgotten.
+func NewClusterLeases(store kvsync.KVStore, holder string) *kvsync.LeaseGroup {
+	return kvsync.NewLeaseGroup(store, KeyPrefixCluster, holder, clusterLeaseTTL)
+}
+
+// tryLease makes one attempt at lease, logging any failure but a held lease,
+// and returns its release, which survives a cancelled ctx, and its Lost.
+func tryLease(ctx context.Context, lease *kvsync.Lease) (func(), <-chan struct{}, bool) {
+	tok, err := lease.TryLock(ctx)
+	if err != nil {
+		if _, held := errors.AsType[*kvsync.HeldError](err); !held {
+			slog.WarnContext(ctx, "eks: lease attempt failed", "key", lease.Key(), "err", err)
+		}
+		return nil, nil, false
+	}
+	return func() { lease.Unlock(ctx, tok) }, lease.Lost(), true
+}
+
+// clusterLeaseName names a cluster's reconciler lease in the group.
+func clusterLeaseName(accountID, clusterName string) string {
+	return accountID + "/" + clusterName
+}
+
+// teardownLeaseName names the per-cluster cross-node gate for a DELETING
+// teardown. It is distinct from clusterLeaseName so a delete never contends
+// with the still-live reconciler, which holds that lease until it reads DELETING.
+func teardownLeaseName(accountID, clusterName string) string {
+	return clusterLeaseName(accountID, clusterName) + "/teardown"
+}

spinifex/domains/eks/cluster/nodegroup_delete_test.go

@@ -662,3 +662,23 @@ func TestDeleteNodegroup_WorkerStuckPastDeadlineEndsDeleteFailedKeepingProfile(t
 	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
 	assert.NotContains(t, fixtureIAM(t, f).Profiles, rec.InstanceProfileName)
 }
+
+// A finished nodegroup delete drops its teardown lease from the group, so the
+// group does not grow with every nodegroup ever deleted and a recreated
+// nodegroup of the same name starts from a fresh lease.
+func TestNodegroupDelete_RetiresItsTeardownLease(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	name := teardownLeaseName(testAccountID, "c1/nodegroup/ng1")
+	before := f.svc.leases.Get(name)
+
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	sweepNodegroups(t, f)
+
+	_, err = GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
+	require.ErrorIs(t, err, ErrNodegroupNotFound, "the delete must have completed")
+	assert.NotSame(t, before, f.svc.leases.Get(name), "the finished delete's teardown lease must leave the group")
+}

spinifex/domains/eks/cluster/reconciler_registry.go

@@ -5,14 +5,12 @@ import (
 	"errors"
 	"log/slog"
 	"sync"
+	"time"
 
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
-// ErrLeaseHeld signals the leader lease is held by another node.
-// Spawn treats it as benign and retries on the next tick.
-var ErrLeaseHeld = errors.New("eks: reconciler lease held by another node")
-
 // ReconcilerRegistry tracks active per-cluster reconciler goroutines; one entry per (accountID, clusterName).
 type ReconcilerRegistry struct {
 	mu      sync.Mutex
@@ -64,9 +62,6 @@ func (r *ReconcilerRegistry) Spawn(parent context.Context, accountID, clusterNam
 		r.mu.Unlock()
 		cancel()
 		close(done)
-		if errors.Is(err, ErrLeaseHeld) {
-			return nil
-		}
 		return err
 	}
 
@@ -151,33 +146,67 @@ func registryKey(accountID, clusterName string) string {
 	return accountID + "/" + clusterName
 }
 
-// RunClusterReconciler is the production SpawnReconcilerFn: constructs a ClusterReconciler,
+// RunClusterReconciler contends for the cluster's lease and drives Run while it
-// acquires the lease, and drives Run in a goroutine. Returns ErrLeaseHeld if another node wins.
+// holds it, in a goroutine, until ctx ends or the cluster leaves CREATING and
+// ACTIVE. A held lease is asked for again every ttl/3, so a dead holder's key is
+// taken over once it has sat a TTL, and a lost one is contended for again.
+// release stops the goroutine and returns once the lease is unlocked.
 func RunClusterReconciler(
 	ctx context.Context,
-	leaderKV, acctKV jetstream.KeyValue,
+	lease *kvsync.Lease, acctKV jetstream.KeyValue,
-	accountID, clusterName, holderID, healthURL string,
+	accountID, clusterName, healthURL string,
 	opts ...ReconcilerOption,
 ) (func(), <-chan struct{}, error) {
-	r, err := NewClusterReconciler(leaderKV, acctKV, accountID, clusterName, holderID, healthURL, opts...)
+	newRun := func() (*ClusterReconciler, error) {
+		return NewClusterReconciler(lease, acctKV, accountID, clusterName, healthURL, opts...)
+	}
+	r, err := newRun()
 	if err != nil {
 		return nil, nil, err
 	}
-	release, ok := r.AcquireLease(ctx)
+	runCtx, cancel := context.WithCancel(ctx)
-	if !ok {
-		slog.Info("RunClusterReconciler: lease held elsewhere, skipping spawn",
-			"accountID", accountID, "cluster", clusterName)
-		return nil, nil, ErrLeaseHeld
-	}
-	// Closed when Run returns so the registry can drop this entry on a self-exit
-	// (terminal state / lost lease), letting a later same-name create re-spawn.
 	finished := make(chan struct{})
 	go func() {
 		defer close(finished)
-		if runErr := r.Run(ctx); runErr != nil && !errors.Is(runErr, context.Canceled) {
+		for {
-			slog.Info("RunClusterReconciler: Run exited",
+			if release, ok := r.AcquireLease(runCtx); ok {
-				"accountID", accountID, "cluster", clusterName, "err", runErr)
+				err := r.Run(runCtx)
+				release()
+				if !errors.Is(err, ErrReconcilerLeaseLost) {
+					if err != nil && !errors.Is(err, context.Canceled) {
+						slog.Info("RunClusterReconciler: Run exited",
+							"accountID", accountID, "cluster", clusterName, "err", err)
+					}
+					return
+				}
+				slog.Info("RunClusterReconciler: lease lost, contending again",
+					"accountID", accountID, "cluster", clusterName)
+			}
+			if !clusterContended(runCtx, acctKV, clusterName) {
+				return
+			}
+			select {
+			case <-runCtx.Done():
+				return
+			case <-time.After(lease.TTL() / 3):
+			}
+			// A fresh reconciler per holding: a finished Run keeps its exit reason.
+			r, _ = newRun()
 		}
 	}()
-	return release, finished, nil
+	return func() { cancel(); <-finished }, finished, nil
+}
+
+// clusterContended reports whether the cluster is still one a reconciler is
+// wanted for. A read that fails is not evidence it is gone, so it keeps trying.
+func clusterContended(ctx context.Context, acctKV jetstream.KeyValue, clusterName string) bool {
+	meta, err := GetClusterMeta(ctx, acctKV, clusterName)
+	switch {
+	case errors.Is(err, ErrClusterNotFound):
+		return false
+	case err != nil:
+		return ctx.Err() == nil
+	default:
+		return meta.Status == ClusterStatusCreating || meta.Status == ClusterStatusActive
+	}
 }

spinifex/domains/eks/cluster/reconciler_registry_test.go

@@ -63,34 +63,6 @@ func TestReconcilerRegistry_SpawnFnErrorRemovesEntry(t *testing.T) {
 	assert.False(t, reg.Has("111122223333", "alpha"), "entry must not linger after spawn failure")
 }
 
-func TestReconcilerRegistry_LeaseHeldDropsEntryAndRetries(t *testing.T) {
-	t.Parallel()
-	reg := NewReconcilerRegistry()
-	var spawnCalls atomic.Int32
-	leaseHeld := atomic.Bool{}
-	leaseHeld.Store(true)
-	fn := func(ctx context.Context, _, _ string) (func(), <-chan struct{}, error) {
-		spawnCalls.Add(1)
-		if leaseHeld.Load() {
-			return nil, nil, ErrLeaseHeld
-		}
-		go func() { <-ctx.Done() }()
-		return func() {}, nil, nil
-	}
-
-	// Lease held elsewhere: Spawn is benign (nil), records no phantom holder.
-	require.NoError(t, reg.Spawn(t.Context(), "111122223333", "alpha", fn))
-	assert.False(t, reg.Has("111122223333", "alpha"), "lease-held must not leave a holder")
-
-	// Holder's TTL lapses; a later Spawn re-invokes fn and now takes over —
-	// no daemon restart needed.
-	leaseHeld.Store(false)
-	require.NoError(t, reg.Spawn(t.Context(), "111122223333", "alpha", fn))
-	assert.True(t, reg.Has("111122223333", "alpha"))
-	assert.EqualValues(t, 2, spawnCalls.Load(), "second Spawn must re-attempt after lease-held")
-	reg.StopAll()
-}
-
 // TestReconcilerRegistry_SelfExitDropsEntryAndRespawns covers a reconciler that
 // exits on its own — a terminal cluster state or a lost lease — with no Stop()
 // to cancel its ctx. The registry must drop the entry (and release the lease) so

spinifex/domains/eks/cluster/service_impl.go

@@ -34,6 +34,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/paging"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/idempotency"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -55,6 +56,9 @@ type EKSServiceDeps struct {
 	Region         string
 	HolderID       string
 
+	// Leases is the per-cluster lease group, from NewClusterLeases.
+	Leases *kvsync.LeaseGroup
+
 	// InternalSuffix is the AWS-parity internal DNS suffix (e.g. spinifex.internal)
 	// used to compose the worker's ECR registry host.
 	InternalSuffix string
@@ -262,7 +266,7 @@ type csiVolumeReclaimer interface {
 // EKSServiceImpl is the daemon-side EKSService implementation.
 type EKSServiceImpl struct {
 	deps     EKSServiceDeps
-	leaderKV jetstream.KeyValue
+	leases   *kvsync.LeaseGroup
 	registry *ReconcilerRegistry
 	owner    *access.Owner
 
@@ -340,7 +344,7 @@ const (
 	defaultLaunchStaleAfter     = 90 * time.Second
 )
 
-// NewEKSServiceImpl initialises EKSServiceImpl, wiring the leader KV and reconciler registry.
+// NewEKSServiceImpl initialises EKSServiceImpl, wiring the lease group and reconciler registry.
 func NewEKSServiceImpl(deps EKSServiceDeps) (*EKSServiceImpl, error) {
 	if err := addon.ValidateCatalog(); err != nil {
 		return nil, fmt.Errorf("eks: validate add-on catalog: %w", err)
@@ -348,21 +352,13 @@ func NewEKSServiceImpl(deps EKSServiceDeps) (*EKSServiceImpl, error) {
 	if deps.NATSConn == nil {
 		return nil, errors.New("eks: NewEKSServiceImpl nil NATSConn")
 	}
-	js, err := jetstream.New(deps.NATSConn)
+	if deps.Leases == nil {
-	if err != nil {
+		return nil, errors.New("eks: NewEKSServiceImpl nil Leases")
-		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
-	// The leader bucket outlives every request, so its open is bounded by the
-	// service's own background context rather than a caller's.
 	ctx, cancel := context.WithCancel(context.Background())
-	leaderKV, err := InitLeaderBucket(ctx, js)
-	if err != nil {
-		cancel()
-		return nil, err
-	}
 	return &EKSServiceImpl{
 		deps:                     deps,
-		leaderKV:                 leaderKV,
+		leases:                   deps.Leases,
 		registry:                 NewReconcilerRegistry(),
 		owner:                    access.New(deps.Region),
 		bgCtx:                    ctx,
@@ -1384,7 +1380,7 @@ func (s *EKSServiceImpl) claimClusterName(ctx context.Context, accountID string,
 		if !held {
 			return errors.New(awserrors.ErrorEKSResourceInUse)
 		}
-		defer release()
+		defer release(false)
 	}
 	var oldRefs *ClusterMeta
 	reclaimed := false
@@ -2368,7 +2364,7 @@ func (s *EKSServiceImpl) spawnBootstrap(accountID, clusterName string, kv jetstr
 		return
 	}
 	ctx, cancel := context.WithCancel(s.bgCtx)
-	key := reconcilerLeaderKey(accountID, clusterName)
+	key := clusterLeaseName(accountID, clusterName)
 	entry := &bootstrapRun{cancel: cancel}
 	s.bootstrapMu.Lock()
 	if s.bootstraps == nil {
@@ -2413,7 +2409,7 @@ type bootstrapRun struct {
 // stopBootstrap cancels this process's bootstrap subscriber for a cluster, so
 // a cluster being torn down stops collecting artifacts into its prefix.
 func (s *EKSServiceImpl) stopBootstrap(accountID, clusterName string) {
-	key := reconcilerLeaderKey(accountID, clusterName)
+	key := clusterLeaseName(accountID, clusterName)
 	s.bootstrapMu.Lock()
 	run := s.bootstraps[key]
 	delete(s.bootstraps, key)
@@ -2454,7 +2450,7 @@ func (s *EKSServiceImpl) spawnReconciler(accountID, clusterName string, _ *Clust
 		opts = append(opts, WithEtcdResetRecovery(0, 0, 0))
 	}
 	spawn := func(ctx context.Context, _, _ string) (func(), <-chan struct{}, error) {
-		return RunClusterReconciler(ctx, s.leaderKV, acctKV, accountID, clusterName, s.deps.HolderID, "", opts...)
+		return RunClusterReconciler(ctx, s.leases.Get(clusterLeaseName(accountID, clusterName)), acctKV, accountID, clusterName, "", opts...)
 	}
 	if err := s.registry.Spawn(s.bgCtx, accountID, clusterName, spawn); err != nil {
 		slog.Error("spawnReconciler: registry spawn", "cluster", clusterName, "err", err)

spinifex/domains/eks/cluster/service_impl_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/domains/eks/access"
 	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
@@ -20,12 +21,20 @@ import (
 
 func setupTestService(t *testing.T) *EKSServiceImpl {
 	t.Helper()
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, nc, js := testutil.StartTestJetStream(t)
-	svc, err := NewEKSServiceImpl(EKSServiceDeps{NATSConn: nc})
+	svc, err := NewEKSServiceImpl(EKSServiceDeps{NATSConn: nc, Leases: NewClusterLeases(kvstore.OpenSyncBucket(js), "test")})
 	require.NoError(t, err)
 	return svc
 }
 
+// A service without the daemon's lease group would contend on keys no other
+// owner sees, so construction refuses it.
+func TestNewEKSServiceImpl_RequiresLeases(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	_, err := NewEKSServiceImpl(EKSServiceDeps{NATSConn: nc})
+	require.ErrorContains(t, err, "nil Leases")
+}
+
 // TestEKSServiceImpl_ClusterLifecycleShimMode covers the four lifecycle
 // methods when the service is constructed with only NATS wiring (the path the
 // daemon-handler routing test uses): orchestration deps

spinifex/domains/eks/cluster/store.go

@@ -5,9 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"strings"
-	"time"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
@@ -19,10 +17,6 @@ import (
 // Per-account bucket "eks-account-{accountID}" holds all customer-visible
 // cluster state. It is created lazily on first cluster touch (no daemon-boot
 // pre-creation) so accounts without any EKS clusters never grow a bucket.
-//
-// Shared leader bucket "spinifex-eks-leader" holds 60s-TTL CAS locks
-// keyed by "{accountID}/{clusterName}" for the per-cluster reconciler
-// leader election. Created once at daemon boot.
 const (
 	KVBucketEKSAccountPrefix  = "eks-account-"
 	KVBucketEKSAccountVersion = 1
@@ -33,10 +27,6 @@ const (
 	// survives a DeleteCluster. Keep this decoupled from the schema version so a
 	// future version bump can't silently widen history and resurrect ciphertext.
 	KVBucketEKSAccountHistory = 1
-
-	KVBucketEKSLeader        = "spinifex-eks-leader"
-	KVBucketEKSLeaderVersion = 1
-	KVBucketEKSLeaderTTL     = 60 * time.Second
 )
 
 // Key-path helpers for the per-account bucket. The layout matches the
@@ -123,15 +113,15 @@ func RecoveryDirectiveKey(cluster, instanceID string) string {
 	return RecoveryPrefix(cluster) + instanceID
 }
 
-// Store is the per-daemon EKS KV handle. Per-account and leader buckets are
+// Store is the per-daemon EKS KV handle. Per-account buckets are accessed via
-// accessed via the package-level factories below.
+// the package-level factories below.
 type Store struct {
 	nc *nats.Conn
 }
 
 // NewStore constructs a Store bound to the supplied NATS connection. It does
 // not touch JetStream — per-account buckets are created lazily by
-// GetOrCreateAccountBucket and the leader bucket by InitLeaderBucket.
+// GetOrCreateAccountBucket.
 func NewStore(nc *nats.Conn) (*Store, error) {
 	if nc == nil {
 		return nil, errors.New("eks store: nats connection is nil")
@@ -212,13 +202,3 @@ func GetOrCreateAccountBucket(ctx context.Context, js jetstream.JetStream, accou
 func OpenAccountBucket(ctx context.Context, js jetstream.JetStream, accountID string) (jetstream.KeyValue, error) {
 	return js.KeyValue(ctx, AccountBucketName(accountID))
 }
-
-// InitLeaderBucket creates (or attaches to) the shared spinifex-eks-leader
-// bucket used for per-cluster reconciler leader-lease CAS locks.
-func InitLeaderBucket(ctx context.Context, js jetstream.JetStream) (jetstream.KeyValue, error) {
-	return kvlease.OpenBucket(ctx, js, kvlease.BucketConfig{
-		Name:    KVBucketEKSLeader,
-		TTL:     KVBucketEKSLeaderTTL,
-		Version: KVBucketEKSLeaderVersion,
-	})
-}

spinifex/domains/eks/cluster/store_test.go

@@ -49,40 +49,6 @@ func TestGetOrCreateAccountBucket_ReplicasClamped(t *testing.T) {
 	assert.Equal(t, 1, si.Config.Replicas)
 }
 
-func TestInitLeaderBucket_Idempotent(t *testing.T) {
-	t.Parallel()
-	_, nc, _ := testutil.StartTestJetStream(t)
-	js := testutil.NewJetStream(t, nc)
-
-	kv1, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
-	require.NotNil(t, kv1)
-	assert.Equal(t, KVBucketEKSLeader, kv1.Bucket())
-
-	kv2, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
-	require.NotNil(t, kv2)
-	assert.Equal(t, KVBucketEKSLeader, kv2.Bucket())
-}
-
-// TestInitLeaderBucket_ReplicasClamped confirms the leader bucket is created
-// with the requested replica count, clamped to a minimum of 1.
-func TestInitLeaderBucket_ReplicasClamped(t *testing.T) {
-	t.Parallel()
-	_, nc, _ := testutil.StartTestJetStream(t)
-	js := testutil.NewJetStream(t, nc)
-
-	kv, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
-	require.NotNil(t, kv)
-
-	stream, err := js.Stream(t.Context(), "KV_"+KVBucketEKSLeader)
-	require.NoError(t, err)
-	si, err := stream.Info(t.Context())
-	require.NoError(t, err)
-	assert.Equal(t, 1, si.Config.Replicas)
-}
-
 func TestKeyPaths_MatchQ2Spec(t *testing.T) {
 	cases := []struct {
 		name string

spinifex/domains/elbv2/lb_repair.go

@@ -10,20 +10,16 @@ import (
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/systeminstance"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
 	"github.com/nats-io/nats.go"
-	"github.com/nats-io/nats.go/jetstream"
 )
 
-const (
+// KeyDataPath is the data-path reconciler's leader lease in the sync bucket.
-	// KVBucketELBv2Leader holds the lease key of the data-path reconciler.
+const KeyDataPath = "elbv2.datapath"
-	KVBucketELBv2Leader        = "spinifex-elbv2-leader"
-	KVBucketELBv2LeaderVersion = 1
-	KVBucketELBv2LeaderTTL     = 60 * time.Second
 
-	lbRepairLeaderKey     = "datapath"
+const (
-	lbRepairLeaseRenew    = 20 * time.Second
+	lbRepairLeaseTTL      = 60 * time.Second
 	lbRepairPassInterval  = 30 * time.Second
 	lbRepairMinFailing    = 60 * time.Second
 	lbRepairMinVMAge      = 5 * time.Minute
@@ -87,34 +83,29 @@ func newLBRepairReconciler(svc *ELBv2ServiceImpl, rebooter SystemInstanceReboote
 	}
 }
 
-// StartDataPathReconciler elects a leader among the daemons and runs the repair
+// NewDataPathLease returns the data-path reconciler's leader lease for holder.
-// pass on it every lbRepairPassInterval until ctx or the service is cancelled.
+func NewDataPathLease(store kvsync.KVStore, holder string) *kvsync.Lease {
-func (s *ELBv2ServiceImpl) StartDataPathReconciler(ctx context.Context, holder string) {
+	return kvsync.NewLease(store, KeyDataPath, holder, lbRepairLeaseTTL)
-	if s.InstanceLauncher == nil || s.store == nil || s.store.js == nil {
+}
+
+// StartDataPathReconciler contends for lease and, while holding it, runs the
+// repair pass every lbRepairPassInterval until ctx or the service is cancelled.
+func (s *ELBv2ServiceImpl) StartDataPathReconciler(ctx context.Context, lease *kvsync.Lease) {
+	if s.InstanceLauncher == nil || s.store == nil || lease == nil {
 		return
 	}
-	lease, err := kvlease.New(kvlease.Config{
+	ctx, cancel := context.WithCancel(ctx)
-		Name:   "elbv2/datapath",
+	if s.ctx != nil {
-		Bucket: s.leaderBucket,
+		context.AfterFunc(s.ctx, cancel)
-		Key:    lbRepairLeaderKey,
-		Holder: holder,
-		TTL:    KVBucketELBv2LeaderTTL,
-		Renew:  lbRepairLeaseRenew,
-		Retry:  lbRepairLeaseRenew,
-	})
-	if err != nil {
-		slog.ErrorContext(ctx, "ELBv2 data-path reconciler: lease config invalid", "err", err)
-		return
 	}
 	rec := newLBRepairReconciler(s, s.InstanceLauncher, lease.Held)
-	go lease.Run(ctx)
+	go kvsync.RunElection(ctx, lease, nil, nil)
 	go func() {
+		defer cancel()
 		ticker := time.NewTicker(lbRepairPassInterval)
 		defer ticker.Stop()
 		for {
 			select {
-			case <-s.ctx.Done():
-				return
 			case <-ctx.Done():
 				return
 			case <-ticker.C:
@@ -124,14 +115,6 @@ func (s *ELBv2ServiceImpl) StartDataPathReconciler(ctx context.Context, holder s
 	}()
 }
 
-func (s *ELBv2ServiceImpl) leaderBucket(ctx context.Context) (jetstream.KeyValue, error) {
-	return kvlease.OpenBucket(ctx, s.store.js, kvlease.BucketConfig{
-		Name:    KVBucketELBv2Leader,
-		TTL:     KVBucketELBv2LeaderTTL,
-		Version: KVBucketELBv2LeaderVersion,
-	})
-}
-
 // candidate is a load balancer that is failing and may be repaired.
 type lbRepairCandidate struct {
 	lb    *LoadBalancerRecord

spinifex/domains/elbv2/lb_repair_test.go

@@ -11,6 +11,8 @@ import (
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/systeminstance"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -179,6 +181,45 @@ func TestLBRepair_NonHolderNeverRepairs(t *testing.T) {
 	assert.Len(t, e.reboots(), 1)
 }
 
+// Leadership comes from the data-path lease: a node repairs nothing while
+// another holds it, repairs once elected, and stops the moment it is taken over.
+func TestLBRepair_ActsOnlyWhileHoldingTheDataPathLease(t *testing.T) {
+	t.Parallel()
+	e := newRepairEnv(t)
+	e.seed("lbr4a", true)
+	e.seed("lbr4b", true)
+	e.fail("lbr4b", 3*time.Minute)
+
+	store := kvstore.OpenSyncBucket(e.svc.store.js)
+	lease := kvsync.NewLease(store, KeyDataPath, "node-a", 300*time.Millisecond)
+	e.rec.isLeader = lease.Held
+	other := NewDataPathLease(kvstore.OpenSyncBucket(e.svc.store.js), "node-b")
+	tok, err := other.TryLock(t.Context())
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(t.Context())
+	done := make(chan struct{})
+	go func() { defer close(done); kvsync.RunElection(ctx, lease, nil, nil) }()
+	t.Cleanup(func() { cancel(); <-done })
+
+	e.pass()
+	assert.False(t, lease.Held())
+	assert.Empty(t, e.reboots(), "a node that lost the election must not repair")
+
+	other.Unlock(t.Context(), tok)
+	require.Eventually(t, lease.Held, 5*time.Second, 5*time.Millisecond)
+	e.pass()
+	assert.Equal(t, []string{"i-lbr4b"}, e.reboots())
+
+	syncKV, err := store.KV(t.Context())
+	require.NoError(t, err)
+	_, err = syncKV.Put(t.Context(), KeyDataPath, []byte(`{"holder":"node-b","generation":0}`))
+	require.NoError(t, err)
+	require.Eventually(t, func() bool { return !lease.Held() }, 5*time.Second, 5*time.Millisecond)
+	e.pass()
+	assert.Equal(t, []string{"i-lbr4b"}, e.reboots(), "a node whose lease was taken over must not repair")
+}
+
 func TestLBRepair_OneRepairPerPass(t *testing.T) {
 	t.Parallel()
 	e := newRepairEnv(t)

spinifex/domains/network/reconcile/drift.go

@@ -14,6 +14,7 @@ import (
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
 	loop "github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/reconciler"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -82,14 +83,14 @@ func intentSource(js jetstream.JetStream) loop.Source {
 }
 
 // DriftLoop reconciles on every intent change and every DriftInterval, gated on
-// AcquireLeader so only one vpcd scans at a time. An incomplete pass asks to be
+// AcquireLeader on lease so only one vpcd scans at a time. An incomplete pass asks to be
 // revisited on a short backoff instead. Returns when ctx is cancelled.
 //
 // startup is the outcome of the bootstrap ReconcileApplyOnly pass, or nil when
 // this node did not run one. It seeds the first deadline rather than triggering
 // a pass: that bootstrap pass is the startup scan, and running a second one here
 // would duplicate it and pull a scan forward on every clean boot.
-func DriftLoop(ctx context.Context, rec Reconciler, nc *nats.Conn, localAZ, holder string, startup error) {
+func DriftLoop(ctx context.Context, rec Reconciler, nc *nats.Conn, lease *kvsync.Lease, localAZ string, startup error) {
 	js, err := jetstream.New(nc)
 	if err != nil {
 		slog.Error("reconcile/drift: JetStream context unavailable, drift loop disabled", "err", err)
@@ -99,7 +100,7 @@ func DriftLoop(ctx context.Context, rec Reconciler, nc *nats.Conn, localAZ, hold
 	loop.Run(ctx, loop.Config{
 		Name:      "vpcd/drift",
 		Sources:   []loop.Source{intentSource(js)},
-		Reconcile: driftPass(rec, nc, js, localAZ, holder, startup),
+		Reconcile: driftPass(rec, lease, js, localAZ, startup),
 		Resync:    DriftInterval,
 		Debounce:  driftDebounce,
 	})
@@ -108,7 +109,7 @@ func DriftLoop(ctx context.Context, rec Reconciler, nc *nats.Conn, localAZ, hold
 // driftPass returns the reconcile pass, closing over the backoff ladder and the
 // last pass time. Run never calls it concurrently with itself, which is why
 // neither needs a lock.
-func driftPass(rec Reconciler, nc *nats.Conn, js jetstream.JetStream, localAZ, holder string, startup error) func(context.Context) (time.Duration, error) {
+func driftPass(rec Reconciler, lease *kvsync.Lease, js jetstream.JetStream, localAZ string, startup error) func(context.Context) (time.Duration, error) {
 	var backoff time.Duration
 	var lastPass time.Time
 	seeded := false
@@ -127,7 +128,8 @@ func driftPass(rec Reconciler, nc *nats.Conn, js jetstream.JetStream, localAZ, h
 			return wait, nil
 		}
 		previous := backoff
-		backoff = nextDriftBackoff(backoff, runDriftCycle(ctx, rec, nc, js, localAZ, holder))
+		retry, err := runDriftCycle(ctx, rec, lease, js, localAZ)
+		backoff = nextDriftBackoff(backoff, err)
 		// Stamped after the scan, not before: the floor is a gap between passes,
 		// and a pass slower than the floor would otherwise leave none at all.
 		lastPass = time.Now()
@@ -136,7 +138,7 @@ func driftPass(rec Reconciler, nc *nats.Conn, js jetstream.JetStream, localAZ, h
 		}
 		// The outcome is logged by the cycle, and an unconverged pass is a change
 		// of schedule rather than a failure of the loop.
-		return backoff, nil
+		return loop.Earliest(backoff, retry), nil
 	}
 }
 
@@ -157,22 +159,22 @@ func nextDriftBackoff(current time.Duration, err error) time.Duration {
 
 // runDriftCycle is one pass body, split out so tests can drive it directly.
 // Returns the reconcile outcome so the caller can size the next wait; a cycle
-// this node did not win the election for returns nil.
+// this node did not win the election for returns nil and the election's revisit.
-func runDriftCycle(ctx context.Context, rec Reconciler, nc *nats.Conn, js jetstream.JetStream, localAZ, holder string) error {
+func runDriftCycle(ctx context.Context, rec Reconciler, lease *kvsync.Lease, js jetstream.JetStream, localAZ string) (time.Duration, error) {
-	release, elected := AcquireLeader(ctx, nc, KVBucketVPCDReconcile, holder)
+	release, retry, elected := ElectPass(ctx, lease)
 	if !elected {
-		return nil
+		return retry, nil
 	}
 	defer release()
 
 	intent, err := LoadIntentFromKV(ctx, js, localAZ)
 	if err != nil {
 		slog.Warn("reconcile/drift: load intent failed", "err", err)
-		return err
+		return 0, err
 	}
 	if err := rec.Reconcile(ctx, intent); err != nil {
 		slog.Warn("reconcile/drift: reconcile failed", "err", err)
-		return err
+		return 0, err
 	}
-	return nil
+	return 0, nil
 }

spinifex/domains/network/reconcile/drift_test.go

@@ -72,7 +72,7 @@ func startDriftLoop(t *testing.T, rec Reconciler, nc *nats.Conn, startup error)
 	done := make(chan struct{})
 	go func() {
 		defer close(done)
-		DriftLoop(ctx, rec, nc, "us-east-1a", "node-1", startup)
+		DriftLoop(ctx, rec, nc, vpcdLease(testutil.NewJetStream(t, nc), "node-1"), "us-east-1a", startup)
 	}()
 	t.Cleanup(func() {
 		cancel()
@@ -171,34 +171,38 @@ func TestDriftLoop_ConvergedPassReturnsToDriftInterval(t *testing.T) {
 // win must not be reported as converged-or-broken; it reports nil and the caller
 // leaves the cadence alone.
 func TestRunDriftCycle_ReportsReconcileOutcome(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 
 	rec := &stubReconciler{outcomes: []error{incompleteErr()}}
-	err := runDriftCycle(t.Context(), rec, nc, js, "us-east-1a", "node-1")
+	_, err := runDriftCycle(t.Context(), rec, vpcdLease(js, "node-1"), js, "us-east-1a")
 	if !errors.Is(err, ErrPassIncomplete) {
 		t.Fatalf("runDriftCycle = %v, want ErrPassIncomplete propagated from the pass", err)
 	}
 
 	converged := &stubReconciler{outcomes: []error{nil}}
-	if err := runDriftCycle(t.Context(), converged, nc, js, "us-east-1a", "node-1"); err != nil {
+	if _, err := runDriftCycle(t.Context(), converged, vpcdLease(js, "node-1"), js, "us-east-1a"); err != nil {
 		t.Errorf("runDriftCycle = %v, want nil for a converged pass", err)
 	}
 }
 
 // A cycle this node loses the election for must not run a pass at all.
 func TestRunDriftCycle_SkipsWhenNotElected(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 
-	release, ok := AcquireLeader(t.Context(), nc, KVBucketVPCDReconcile, "other-node")
+	release, ok := AcquireLeader(t.Context(), vpcdLease(js, "other-node"))
 	if !ok {
 		t.Fatal("failed to take the lock the test needs held")
 	}
 	defer release()
 
 	rec := &stubReconciler{outcomes: []error{incompleteErr()}}
-	if err := runDriftCycle(t.Context(), rec, nc, js, "us-east-1a", "node-1"); err != nil {
+	retry, err := runDriftCycle(t.Context(), rec, vpcdLease(js, "node-1"), js, "us-east-1a")
+	if err != nil {
 		t.Errorf("runDriftCycle = %v, want nil when not elected", err)
 	}
+	if retry <= 0 || retry > LeaderTTL/3 {
+		t.Errorf("revisit = %v, want within %v so a dead holder is taken over promptly", retry, LeaderTTL/3)
+	}
 	if got := rec.callCount(); got != 0 {
 		t.Errorf("reconcile ran %d times without winning the election, want 0", got)
 	}

spinifex/domains/network/reconcile/drift_watch_test.go

@@ -213,11 +213,11 @@ func TestIntentSource_CoversEveryBucketTheIntentLoadReads(t *testing.T) {
 // The floor is what stops a busy cluster scanning continuously, so it is pinned
 // on the pass rather than inferred from the loop's timing.
 func TestDriftPass_FloorsTheChangeDrivenRate(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	shrinkDriftTiming(t, time.Minute, time.Hour)
 
 	rec := &stubReconciler{outcomes: []error{nil}}
-	pass := driftPass(rec, nc, js, "us-east-1a", "node-1", nil)
+	pass := driftPass(rec, vpcdLease(js, "node-1"), js, "us-east-1a", nil)
 
 	// The seed pass does no work: the bootstrap ReconcileApplyOnly was the
 	// startup scan and this one must not repeat it.

spinifex/domains/network/reconcile/host_ingress_test.go

@@ -358,7 +358,7 @@ func TestHostIngress_IntentReadFailureNeverPrunes(t *testing.T) {
 // while another node holds the drift lease and no event arrives, well inside
 // its resync.
 func TestRunHostIngress_IntentWriteConvergesEveryNodeWithoutLeaseOrEvent(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	ctx := t.Context()
 	vpcs, err := kvstore.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 1)
 	if err != nil {
@@ -369,7 +369,7 @@ func TestRunHostIngress_IntentWriteConvergesEveryNodeWithoutLeaseOrEvent(t *test
 		t.Fatalf("create IGW bucket: %v", err)
 	}
 	put(t, vpcs, "acct/vpc-1", ec2vpc.VPCRecord{VpcId: "vpc-1", CidrBlock: "10.1.0.0/16"})
-	release, elected := reconcile.AcquireLeader(ctx, nc, reconcile.KVBucketVPCDReconcile, "node-elsewhere")
+	release, elected := reconcile.AcquireLeader(ctx, reconcile.NewLeaderLease(kvstore.OpenSyncBucket(js), reconcile.KeyVPCDReconcile, "node-elsewhere"))
 	if !elected {
 		t.Fatal("could not take the drift lease for another node")
 	}

spinifex/domains/network/reconcile/lock.go

@@ -2,52 +2,100 @@ package reconcile
 
 import (
 	"context"
+	"errors"
 	"log/slog"
 	"time"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
-	"github.com/nats-io/nats.go"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	otelsetup "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
 )
 
-// Single CAS-elected leader key; TTL bounds crash-recovery.
+// KeyVPCDReconcile is vpcd's network reconcile lease in the sync bucket.
-const (
+const KeyVPCDReconcile = "vpcd.reconcile"
-	KVBucketVPCDReconcile = "spinifex-vpcd-reconcile"
-	reconcileLeaderKey    = "leader"
-)
 
-// Leader-key lifetime. Vars (not consts) so tests can shrink them.
+// LeaderTTL is the per-pass leader leases' time to live. A pass can outlive it
-//
+// on its own (one stalled gw-lrp DORA runs ~64s), so the lease renews itself
-// reconcileLeaderRenew is the gap between refreshes: a pass can outlive the TTL
+// while the pass runs rather than lapsing underneath it.
-// on its own — one stalled gw-lrp DORA runs ~64s — so the key is renewed while
+const LeaderTTL = 60 * time.Second
-// the pass runs rather than left to expire underneath it.
-var (
-	reconcileLeaderTTL   = 60 * time.Second
-	reconcileLeaderRenew = 20 * time.Second
-)
 
-// AcquireLeader elects one leader on the named lock bucket. Independent
+// NewLeaderLease returns a per-pass leader lease on key. Each is built once per
-// reconcile loops pass distinct buckets so they never share a single mutex: the
+// process and shared by every site that reconciles under it, so those sites
-// gateway quota reconcile must not block vpcd's network reconcile, and vice
+// exclude each other on one node as well as across nodes.
-// versa.
+func NewLeaderLease(store kvsync.KVStore, key, holder string) *kvsync.Lease {
-func AcquireLeader(ctx context.Context, nc *nats.Conn, bucket, holder string) (func(), bool) {
+	return kvsync.NewLease(store, key, holder, LeaderTTL)
-	lease, err := kvlease.New(kvlease.Config{
+}
-		Name:   "reconcile/lock",
+
-		Bucket: kvlease.NATSBucket(nc, bucket, reconcileLeaderTTL),
+// takeoverRetry is when a pass that found the lease held asks to run again. A
-		Key:    reconcileLeaderKey,
+// dead holder's key is taken over once a follower has seen its revision sit for
-		Holder: holder,
+// a TTL, so retrying at the renewal interval takes over within about 4*TTL/3.
-		Attrs:  []any{"bucket", bucket},
+const takeoverRetry = LeaderTTL / 3
-		TTL:    reconcileLeaderTTL,
+
-		Renew:  reconcileLeaderRenew,
+// Bounded wait for JetStream quorum on a cold multi-node start, for a one-shot
-	})
+// election that has no later pass to retry it. Vars so tests can shrink them.
-	if err != nil {
+var (
-		slog.ErrorContext(ctx, "reconcile/lock: lease config invalid",
+	startRetryFor  = 60 * time.Second
-			"node", holder, "bucket", bucket, "err", err)
+	startRetryStep = time.Second
-		return nil, false
+)
-	}
+
-	if !lease.TryAcquire(ctx) {
+// ElectPass is AcquireLeader for a pass in a loop, adding a revisit deadline:
-		slog.InfoContext(ctx, "reconcile/lock: not leader, skipping reconcile",
+// a pass that found the lease held asks to run again once a dead holder could
-			"node", holder, "bucket", bucket)
+// have been waited out, rather than waiting for the loop's next resync. The
-		return nil, false
+// release survives a cancelled ctx.
-	}
+func ElectPass(ctx context.Context, lease *kvsync.Lease) (release func(), revisit time.Duration, ok bool) {
-	return func() { lease.Release(ctx) }, true
+	tok, err := lease.TryLock(ctx)
+	if err != nil {
+		logFailure(ctx, lease, err)
+		if _, held := errors.AsType[*kvsync.HeldError](err); held {
+			return nil, takeoverRetry, false
+		}
+		return nil, 0, false
+	}
+	return func() { lease.Unlock(ctx, tok) }, 0, true
+}
+
+// AcquireLeaderAtStart is AcquireLeader for a one-shot election at start,
+// retrying until JetStream has formed quorum or the wait runs out, since a cold
+// cluster fails in many shapes. A held lease or a config fault answers at once.
+func AcquireLeaderAtStart(ctx context.Context, lease *kvsync.Lease) (func(), bool) {
+	deadline := time.Now().Add(startRetryFor)
+	for {
+		tok, err := lease.TryLock(ctx)
+		if err == nil {
+			return func() { lease.Unlock(ctx, tok) }, true
+		}
+		if _, held := errors.AsType[*kvsync.HeldError](err); held || clustersize.Permanent(err) {
+			logFailure(ctx, lease, err)
+			return nil, false
+		}
+		if !time.Now().Before(deadline) {
+			slog.WarnContext(ctx, "reconcile/lock: election failed until the start wait ran out, skipping reconcile",
+				"key", lease.Key(), "wait_ms", otelsetup.Millis(startRetryFor), "err", err)
+			return nil, false
+		}
+		slog.DebugContext(ctx, "reconcile/lock: election failed at start, retrying", "key", lease.Key(), "err", err)
+		select {
+		case <-ctx.Done():
+			return nil, false
+		case <-time.After(startRetryStep):
+		}
+	}
+}
+
+// AcquireLeader makes one attempt to take lease for a pass. Independent
+// reconcile loops hold distinct leases, so the gateway quota reconcile never
+// blocks vpcd's network reconcile. The release survives a cancelled ctx.
+func AcquireLeader(ctx context.Context, lease *kvsync.Lease) (func(), bool) {
+	release, _, ok := ElectPass(ctx, lease)
+	return release, ok
+}
+
+func logFailure(ctx context.Context, lease *kvsync.Lease, err error) {
+	if held, ok := errors.AsType[*kvsync.HeldError](err); ok {
+		slog.InfoContext(ctx, "reconcile/lock: not leader, skipping reconcile",
+			"key", lease.Key(), "holder", held.Current.Holder)
+		return
+	}
+	slog.WarnContext(ctx, "reconcile/lock: election failed, skipping reconcile",
+		"key", lease.Key(), "err", err)
 }

spinifex/domains/network/reconcile/lock_test.go

@@ -5,114 +5,83 @@ import (
 	"context"
 	"encoding/json"
 	"log/slog"
+	"sync/atomic"
 	"testing"
 	"time"
 
-	"github.com/mulgadc/spinifex/internal/testkit"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-// TestAcquireLeader_GetOrCreateAcrossCalls pins the get-or-create fix: first
+// vpcdLease returns holder's vpcd lease on a store of its own, standing in for
-// acquire creates the bucket and wins; second must attach (not hang on "stream
+// a separate process.
-// name already in use") and lose; after release a new acquire wins.
+func vpcdLease(js jetstream.JetStream, holder string) *kvsync.Lease {
-func TestAcquireLeader_GetOrCreateAcrossCalls(t *testing.T) {
+	return NewLeaderLease(kvstore.OpenSyncBucket(js), KeyVPCDReconcile, holder)
-	_, nc, _ := testutil.StartTestJetStream(t)
+}
 
-	release, ok := AcquireLeader(t.Context(), nc, KVBucketVPCDReconcile, "node-1")
+func TestAcquireLeader_OneHolderAtATimeAcrossProcesses(t *testing.T) {
-	require.True(t, ok, "first acquire must win and create the bucket")
+	_, _, js := testutil.StartTestJetStream(t)
-	require.NotNil(t, release)
+	node1, node2 := vpcdLease(js, "node-1"), vpcdLease(js, "node-2")
 
-	// Bucket now exists; the second acquire must reach the Create-key contention
+	release, ok := AcquireLeader(t.Context(), node1)
-	// path (and lose) rather than dead-ending on CreateKeyValue.
+	require.True(t, ok, "the first acquire on a free key must win")
-	loserRelease, ok := AcquireLeader(t.Context(), nc, KVBucketVPCDReconcile, "node-2")
+
-	assert.False(t, ok, "second acquire must lose while the lock is held")
+	loserRelease, ok := AcquireLeader(t.Context(), node2)
+	assert.False(t, ok, "a second process must lose while the lease is held")
 	assert.Nil(t, loserRelease)
 
-	// Releasing frees the key so a subsequent acquire can take over.
 	release()
-	release2, ok := AcquireLeader(t.Context(), nc, KVBucketVPCDReconcile, "node-3")
+	release2, ok := AcquireLeader(t.Context(), node2)
-	require.True(t, ok, "acquire after release must win")
+	require.True(t, ok, "acquire after release must win at once")
-	require.NotNil(t, release2)
 	release2()
 }
 
-// TestAcquireLeader_ReleaseSurvivesCancelledContext pins that the release runs
+// The two vpcd sites share one Lease, so they exclude each other on one node
-// on a context of its own. Shutdown cancels the acquiring context first and then
+// as well as across nodes.
-// releases, so a release bound to that context would silently no-op and park the
+func TestAcquireLeader_SitesSharingALeaseExcludeEachOther(t *testing.T) {
-// lock for the full TTL, blocking every other node's reconcile.
+	_, _, js := testutil.StartTestJetStream(t)
-func TestAcquireLeader_ReleaseSurvivesCancelledContext(t *testing.T) {
+	shared := vpcdLease(js, "node-1")
-	_, nc, _ := testutil.StartTestJetStream(t)
 
-	ctx, cancel := context.WithCancel(t.Context())
+	startup, ok := AcquireLeader(t.Context(), shared)
-	release, ok := AcquireLeader(ctx, nc, KVBucketVPCDReconcile, "node-1")
 	require.True(t, ok)
 
+	_, ok = AcquireLeader(t.Context(), shared)
+	assert.False(t, ok, "the drift pass ran while the startup pass held the lease")
+
+	startup()
+	drift, ok := AcquireLeader(t.Context(), shared)
+	require.True(t, ok, "the drift pass must run once the startup pass has released")
+	drift()
+}
+
+// Shutdown cancels the acquiring context first and then releases, so a release
+// bound to that context would no-op and leave the next leader waiting a TTL.
+func TestAcquireLeader_ReleaseSurvivesCancelledContext(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	node1, node2 := vpcdLease(js, "node-1"), vpcdLease(js, "node-2")
+
+	ctx, cancel := context.WithCancel(t.Context())
+	release, ok := AcquireLeader(ctx, node1)
+	require.True(t, ok)
 	cancel()
 	release()
 
-	release2, ok := AcquireLeader(t.Context(), nc, KVBucketVPCDReconcile, "node-2")
+	release2, ok := AcquireLeader(t.Context(), node2)
-	require.True(t, ok, "lock must be free immediately after release, not held until the TTL reaps it")
+	require.True(t, ok, "the lease must be free immediately after release")
 	release2()
 }
 
-// shrinkLeaderTiming compresses the leader key's lifetime so a test can outlast
+// A losing acquire logs who holds the key, not the local node as holder.
-// the TTL in under a second rather than a minute.
+func TestAcquireLeader_NotLeaderLogNamesTheHolder(t *testing.T) {
-func shrinkLeaderTiming(t *testing.T, ttl, renew time.Duration) {
+	_, _, js := testutil.StartTestJetStream(t)
-	t.Helper()
+	node1, node2 := vpcdLease(js, "node-1"), vpcdLease(js, "node-2")
-	oldTTL, oldRenew := reconcileLeaderTTL, reconcileLeaderRenew
-	reconcileLeaderTTL, reconcileLeaderRenew = ttl, renew
-	t.Cleanup(func() { reconcileLeaderTTL, reconcileLeaderRenew = oldTTL, oldRenew })
-}
 
-// A pass can outlive the TTL on its own — one stalled gw-lrp DORA runs ~64s
+	release, ok := AcquireLeader(t.Context(), node1)
-// against a 60s TTL — so the key must be renewed while the pass runs. Without
-// this a peer elects itself mid-pass and two reconcilers race the same OVN NB.
-func TestAcquireLeader_RenewsKeyBeyondTTL(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	shrinkLeaderTiming(t, 300*time.Millisecond, 100*time.Millisecond)
-
-	release, ok := AcquireLeader(t.Context(), nc, KVBucketVPCDReconcile, "node-1")
-	require.True(t, ok)
-	defer release()
-
-	// Well past the TTL: an unrenewed key would have expired by now.
-	time.Sleep(750 * time.Millisecond)
-
-	loserRelease, ok := AcquireLeader(t.Context(), nc, KVBucketVPCDReconcile, "node-2")
-	assert.False(t, ok, "leader key expired mid-pass: a peer can now reconcile concurrently")
-	assert.Nil(t, loserRelease)
-}
-
-// If the key was lost anyway, the stale holder's release must not delete the
-// successor's key — an unguarded delete hands the lock to a third node while
-// the successor believes it still holds it.
-func TestAcquireLeader_ReleaseDoesNotDeleteSuccessorKey(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
-
-	release, ok := AcquireLeader(t.Context(), nc, KVBucketVPCDReconcile, "node-1")
-	require.True(t, ok)
-
-	// Stand in for a successor that claimed the key after it expired: the write
-	// bumps the revision, so node-1's release is now CAS-stale.
-	kv, err := js.KeyValue(t.Context(), KVBucketVPCDReconcile)
-	require.NoError(t, err)
-	_, err = kv.Put(t.Context(), reconcileLeaderKey, []byte("node-2"))
-	require.NoError(t, err)
-
-	release()
-
-	entry, err := kv.Get(t.Context(), reconcileLeaderKey)
-	require.NoError(t, err, "stale release deleted the successor's leader key")
-	assert.Equal(t, "node-2", string(entry.Value()), "successor's key must survive a stale release")
-}
-
-// A node that loses the election logs itself as node: "holder=node-2" on
-// node-2's log reads as though node-2 holds the lock.
-func TestAcquireLeader_NotLeaderLogNamesLocalNode(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-
-	release, ok := AcquireLeader(t.Context(), nc, KVBucketVPCDReconcile, "node-1")
 	require.True(t, ok)
 	defer release()
 
@@ -121,7 +90,7 @@ func TestAcquireLeader_NotLeaderLogNamesLocalNode(t *testing.T) {
 	slog.SetDefault(slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo})))
 	t.Cleanup(func() { slog.SetDefault(prev) })
 
-	_, ok = AcquireLeader(t.Context(), nc, KVBucketVPCDReconcile, "node-2")
+	_, ok = AcquireLeader(t.Context(), node2)
 	require.False(t, ok)
 
 	var line map[string]any
@@ -134,6 +103,148 @@ func TestAcquireLeader_NotLeaderLogNamesLocalNode(t *testing.T) {
 		}
 	}
 	require.NotNil(t, line, "a losing acquire must log that it is skipping")
-	assert.Equal(t, "node-2", line["node"])
+	assert.Equal(t, "node-1", line["holder"])
-	assert.NotEqual(t, "node-2", line["holder"], "the local node is not the holder")
+	assert.Equal(t, KeyVPCDReconcile, line["key"])
+}
+
+// coldStore answers every Load with ErrUnavailable until down reaches zero,
+// standing in for a JetStream still forming quorum.
+type coldStore struct {
+	kvsync.KVStore
+
+	down atomic.Int32
+}
+
+func (c *coldStore) Load(ctx context.Context, key string) ([]byte, uint64, error) {
+	if c.down.Add(-1) >= 0 {
+		return nil, 0, kvsync.ErrUnavailable
+	}
+	return c.KVStore.Load(ctx, key)
+}
+
+func shrinkStartRetry(t *testing.T, retryFor, step time.Duration) {
+	t.Helper()
+	prevFor, prevStep := startRetryFor, startRetryStep
+	startRetryFor, startRetryStep = retryFor, step
+	t.Cleanup(func() { startRetryFor, startRetryStep = prevFor, prevStep })
+}
+
+// A pass that loses to a peer asks to run again within a renewal interval, so a
+// follower that first saw a dead holder's revision sees it a TTL later and takes
+// over about 4*TTL/3 after the last renewal rather than up to 2*TTL.
+func TestElectPass_AHeldLeaseAsksForATakeoverRevisit(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	release, ok := AcquireLeader(t.Context(), vpcdLease(js, "node-1"))
+	require.True(t, ok)
+	defer release()
+
+	_, revisit, ok := ElectPass(t.Context(), vpcdLease(js, "node-2"))
+	assert.False(t, ok)
+	assert.Positive(t, revisit, "a held pass must ask to run again, not wait for a resync")
+	assert.LessOrEqual(t, revisit, LeaderTTL/3, "the retry is a renewal interval out, so takeover is not delayed a further TTL")
+}
+
+func TestElectPass_AWonPassAsksForNoRevisit(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	release, revisit, ok := ElectPass(t.Context(), vpcdLease(js, "node-1"))
+	require.True(t, ok)
+	release()
+	assert.Zero(t, revisit)
+}
+
+// A cold multi-node start has no quorum yet, and the one startup election
+// must wait for it rather than skip the startup reconcile.
+func TestAcquireLeaderAtStart_WaitsOutAnUnavailableStore(t *testing.T) {
+	shrinkStartRetry(t, 10*time.Second, 5*time.Millisecond)
+	_, _, js := testutil.StartTestJetStream(t)
+	cold := &coldStore{KVStore: kvstore.OpenSyncBucket(js)}
+	cold.down.Store(3)
+
+	release, ok := AcquireLeaderAtStart(t.Context(), NewLeaderLease(cold, KeyVPCDReconcile, "node-1"))
+	require.True(t, ok, "the election must be won once the store answers")
+	release()
+}
+
+// formingJS fails its first down bucket lookups and creates with err, as a
+// cluster still electing its meta leader or placing replicas does.
+type formingJS struct {
+	jetstream.JetStream
+
+	err  error
+	down atomic.Int32
+}
+
+func (f *formingJS) KeyValue(ctx context.Context, name string) (jetstream.KeyValue, error) {
+	if f.down.Add(-1) >= 0 {
+		return nil, f.err
+	}
+	return f.JetStream.KeyValue(ctx, name)
+}
+
+func (f *formingJS) CreateKeyValue(ctx context.Context, cfg jetstream.KeyValueConfig) (jetstream.KeyValue, error) {
+	if f.down.Add(-1) >= 0 {
+		return nil, f.err
+	}
+	return f.JetStream.CreateKeyValue(ctx, cfg)
+}
+
+// The startup election waits out each shape a forming cluster fails in, as the
+// real bucket reports it, not only the errors already classed as unavailable.
+func TestAcquireLeaderAtStart_WaitsOutAFormingCluster(t *testing.T) {
+	cases := map[string]error{
+		"no meta leader":       &jetstream.APIError{Code: 503, ErrorCode: 10008, Description: "JetStream system temporarily unavailable"},
+		"no suitable peers":    &jetstream.APIError{Code: 400, ErrorCode: 10005, Description: "no suitable peers for placement"},
+		"insufficient storage": &jetstream.APIError{Code: 503, ErrorCode: 10023, Description: "insufficient resources"},
+	}
+	for name, failure := range cases {
+		t.Run(name, func(t *testing.T) {
+			shrinkStartRetry(t, 10*time.Second, 5*time.Millisecond)
+			_, _, js := testutil.StartTestJetStream(t)
+			forming := &formingJS{JetStream: js, err: failure}
+			forming.down.Store(4)
+
+			release, ok := AcquireLeaderAtStart(t.Context(), NewLeaderLease(kvstore.OpenSyncBucket(forming), KeyVPCDReconcile, "node-1"))
+			require.True(t, ok, "the election must be won once the cluster has formed")
+			release()
+		})
+	}
+}
+
+func TestAcquireLeaderAtStart_GivesUpWhenTheWaitRunsOut(t *testing.T) {
+	shrinkStartRetry(t, 50*time.Millisecond, 5*time.Millisecond)
+	_, _, js := testutil.StartTestJetStream(t)
+	cold := &coldStore{KVStore: kvstore.OpenSyncBucket(js)}
+	cold.down.Store(1 << 30)
+
+	start := time.Now()
+	_, ok := AcquireLeaderAtStart(t.Context(), NewLeaderLease(cold, KeyVPCDReconcile, "node-1"))
+	assert.False(t, ok)
+	assert.Less(t, time.Since(start), 5*time.Second)
+}
+
+// A held lease is an answer, not an outage, so it is not waited on.
+func TestAcquireLeaderAtStart_AHeldLeaseAnswersAtOnce(t *testing.T) {
+	shrinkStartRetry(t, time.Minute, time.Second)
+	_, _, js := testutil.StartTestJetStream(t)
+	release, ok := AcquireLeader(t.Context(), vpcdLease(js, "node-1"))
+	require.True(t, ok)
+	defer release()
+
+	start := time.Now()
+	_, ok = AcquireLeaderAtStart(t.Context(), vpcdLease(js, "node-2"))
+	assert.False(t, ok)
+	assert.Less(t, time.Since(start), 5*time.Second, "a held lease must not consume the retry window")
+}
+
+// An undeclared cluster size never becomes declared by waiting, so it ends the
+// wait at once instead of holding the startup reconcile for the whole window.
+func TestAcquireLeaderAtStart_StopsOnAConfigFault(t *testing.T) {
+	shrinkStartRetry(t, time.Minute, time.Second)
+	_, _, js := testutil.StartTestJetStream(t)
+	clustersize.RedeclareForTest(t, 0)
+
+	start := time.Now()
+	_, ok := AcquireLeaderAtStart(t.Context(), vpcdLease(js, "node-1"))
+	assert.False(t, ok)
+	assert.Less(t, time.Since(start), 10*time.Second, "a permanent fault must not consume the retry window")
 }

spinifex/domains/ochre/reaper.go

@@ -8,16 +8,18 @@ import (
 	"log/slog"
 	"time"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
 )
 
-// Sweep and lease timing. The holder refreshes well inside the bucket's TTL,
+// KeyReaper is the reaper's leader lease in the sync bucket.
-// so a leader that dies is replaced within one TTL rather than one refresh.
+const KeyReaper = "ochre.reaper"
+
+// Sweep and lease timing. A leader that dies is replaced within about one TTL.
 const (
+	reaperLeaseTTL        = 60 * time.Second
 	defaultReaperInterval = 30 * time.Second
-	defaultLeaseRefresh   = 20 * time.Second
 	defaultIdleTTL        = 15 * time.Minute // How long an endpoint must have been idle before its GPU is taken back.
 	maxScrapeFailures     = 5                // How many consecutive failed scrapes mean the serving process is wedged rather than briefly busy.
 	defaultScrapeTimeout  = 5 * time.Second  // Per-scrape budget.
@@ -28,7 +30,6 @@ const (
 type ReaperDeps struct {
 	Interval      time.Duration
 	IdleTTL       time.Duration
-	LeaseRefresh  time.Duration
 	ScrapeTimeout time.Duration
 }
 
@@ -36,26 +37,19 @@ type ReaperDeps struct {
 // does the scraping and reaping; every node keeps serving the API, so a
 // leaderless gap delays a reclaim rather than failing a request.
 type Reaper struct {
-	svc      *Service
+	svc   *Service
-	holder   string
+	deps  ReaperDeps
-	deps     ReaperDeps
+	lease *kvsync.Lease
-	lease    *kvlease.Lease
-	leaseErr error
 }
 
-// NewReaper builds the reaper for svc; holder identifies this daemon in the lease.
+// NewReaperLease returns the reaper's leader lease for holder.
-func NewReaper(svc *Service, holder string, deps ReaperDeps) *Reaper {
+func NewReaperLease(store kvsync.KVStore, holder string) *kvsync.Lease {
-	r := &Reaper{svc: svc, holder: holder, deps: deps}
+	return kvsync.NewLease(store, KeyReaper, holder, reaperLeaseTTL)
-	r.lease, r.leaseErr = kvlease.New(kvlease.Config{
+}
-		Name:   "bedrock/reaper",
+
-		Bucket: svc.leader.KV,
+// NewReaper builds the reaper for svc, which leads while it holds lease.
-		Key:    leaderKey,
+func NewReaper(svc *Service, lease *kvsync.Lease, deps ReaperDeps) *Reaper {
-		Holder: holder,
+	return &Reaper{svc: svc, deps: deps, lease: lease}
-		TTL:    KVBucketLeaderTTL,
-		Renew:  r.leaseRefresh(),
-		Retry:  r.leaseRefresh(),
-	})
-	return r
 }
 
 func (r *Reaper) interval() time.Duration {
@@ -72,13 +66,6 @@ func (r *Reaper) idleTTL() time.Duration {
 	return defaultIdleTTL
 }
 
-func (r *Reaper) leaseRefresh() time.Duration {
-	if r.deps.LeaseRefresh > 0 {
-		return r.deps.LeaseRefresh
-	}
-	return defaultLeaseRefresh
-}
-
 func (r *Reaper) scrapeTimeout() time.Duration {
 	if r.deps.ScrapeTimeout > 0 {
 		return r.deps.ScrapeTimeout
@@ -89,17 +76,13 @@ func (r *Reaper) scrapeTimeout() time.Duration {
 // Run drives the leadership and sweep loop until ctx is cancelled. Intended as
 // a daemon-boot goroutine; panics are the caller's recover concern.
 func (r *Reaper) Run(ctx context.Context) {
-	if r.leaseErr != nil {
-		slog.ErrorContext(ctx, "bedrock reaper: lease config invalid", "holder", r.holder, "err", r.leaseErr)
-		return
-	}
 	sweepTicker := time.NewTicker(r.interval())
 	defer sweepTicker.Stop()
 
 	leaseDone := make(chan struct{})
 	go func() {
 		defer close(leaseDone)
-		r.lease.Run(ctx)
+		kvsync.RunElection(ctx, r.lease, nil, nil)
 	}()
 
 	for {
@@ -112,7 +95,7 @@ func (r *Reaper) Run(ctx context.Context) {
 				continue
 			}
 			if err := r.sweepOnce(ctx); err != nil {
-				slog.ErrorContext(ctx, "bedrock reaper: sweep failed", "holder", r.holder, "err", err)
+				slog.ErrorContext(ctx, "bedrock reaper: sweep failed", "key", r.lease.Key(), "err", err)
 			}
 		}
 	}

spinifex/domains/ochre/reaper_test.go

@@ -14,6 +14,9 @@ import (
 
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -95,13 +98,31 @@ func newReaperFixture(t *testing.T, deps ReaperDeps) *reaperFixture {
 	})
 	return &reaperFixture{
 		svc:       svc,
-		reaper:    NewReaper(svc, "node-a", deps),
+		reaper:    testReaper(t, svc, "node-a", deps),
 		harness:   h,
 		transport: transport,
 		store:     svc.store,
 	}
 }
 
+// testReaper is holder's reaper on a sync bucket of its own, as a separate
+// daemon's would be.
+func testReaper(t *testing.T, svc *Service, holder string, deps ReaperDeps) *Reaper {
+	t.Helper()
+	js, err := jetstream.New(svc.nc)
+	require.NoError(t, err)
+	return NewReaper(svc, NewReaperLease(kvstore.OpenSyncBucket(js), holder), deps)
+}
+
+// lead takes r's lease, and unlocks it when the test ends.
+func lead(t *testing.T, r *Reaper) kvsync.Token {
+	t.Helper()
+	tok, err := r.lease.TryLock(t.Context())
+	require.NoError(t, err)
+	t.Cleanup(func() { r.lease.Unlock(context.Background(), tok) })
+	return tok
+}
+
 // ready launches testModelID and waits for its record to reach READY.
 func (f *reaperFixture) ready(t *testing.T) EndpointRecord {
 	t.Helper()
@@ -393,31 +414,28 @@ func TestReaper_SkipsNonReadyRecords(t *testing.T) {
 
 func TestReaper_LeaseAdmitsOneLeader(t *testing.T) {
 	f := newReaperFixture(t, ReaperDeps{})
-	other := NewReaper(f.svc, "node-b", ReaperDeps{})
+	other := testReaper(t, f.svc, "node-b", ReaperDeps{})
 
-	require.True(t, f.reaper.lease.TryAcquire(t.Context()))
+	lead(t, f.reaper)
-	require.False(t, other.lease.TryAcquire(t.Context()),
+	_, err := other.lease.TryLock(t.Context())
+	require.ErrorAs(t, err, new(*kvsync.HeldError),
 		"two nodes must never sweep the same endpoints at once")
 
 	assert.True(t, f.reaper.IsLeader())
 	assert.False(t, other.IsLeader())
-
-	// A refresh by the holder keeps it, and does not hand it away.
-	require.True(t, f.reaper.lease.TryAcquire(t.Context()))
-	assert.True(t, f.reaper.IsLeader())
 }
 
 // Releasing on shutdown is what makes a rolling restart cost seconds rather
 // than a full lease TTL of no reclaim.
 func TestReaper_ReleaseFreesTheLeaseImmediately(t *testing.T) {
 	f := newReaperFixture(t, ReaperDeps{})
-	other := NewReaper(f.svc, "node-b", ReaperDeps{})
+	other := testReaper(t, f.svc, "node-b", ReaperDeps{})
 
-	f.reaper.lease.TryAcquire(t.Context())
+	tok := lead(t, f.reaper)
 	require.True(t, f.reaper.IsLeader())
 
-	f.reaper.lease.Release(t.Context())
+	f.reaper.lease.Unlock(t.Context(), tok)
-	other.lease.TryAcquire(t.Context())
+	lead(t, other)
 	assert.True(t, other.IsLeader())
 }
 
@@ -428,8 +446,8 @@ func TestReaper_NonLeaderSweepsNothing(t *testing.T) {
 	f.ready(t)
 	f.age(t, time.Hour, time.Hour)
 
-	holder := NewReaper(f.svc, "node-b", ReaperDeps{})
+	holder := testReaper(t, f.svc, "node-b", ReaperDeps{})
-	holder.lease.TryAcquire(t.Context())
+	lead(t, holder)
 	require.True(t, holder.IsLeader())
 
 	ctx, cancel := context.WithTimeout(t.Context(), 60*time.Millisecond)
@@ -457,7 +475,6 @@ func TestEndpointRecord_LastActive(t *testing.T) {
 // full TTL rather than freeing it immediately.
 func TestReaper_RunReleasesLeaseOnShutdown(t *testing.T) {
 	f := newReaperFixture(t, ReaperDeps{Interval: time.Hour})
-	require.NoError(t, f.reaper.leaseErr)
 
 	ctx, cancel := context.WithCancel(t.Context())
 	done := make(chan struct{})
@@ -467,7 +484,45 @@ func TestReaper_RunReleasesLeaseOnShutdown(t *testing.T) {
 	cancel()
 	<-done
 
-	other := NewReaper(f.svc, "node-b", ReaperDeps{})
+	other := testReaper(t, f.svc, "node-b", ReaperDeps{})
-	assert.True(t, other.lease.TryAcquire(t.Context()),
+	tok, err := other.lease.TryLock(t.Context())
-		"Run returned with the lease key still present")
+	require.NoError(t, err, "Run returned with the lease key still present")
+	other.lease.Unlock(t.Context(), tok)
+}
+
+// A reaper whose lease is taken over must stop reclaiming at once, or two
+// nodes would tear the same endpoint down; re-elected, it reclaims again.
+func TestReaper_LosingTheLeaseStopsReclaimUntilReElection(t *testing.T) {
+	deps := ReaperDeps{Interval: 5 * time.Millisecond, IdleTTL: time.Minute}
+	f := newReaperFixture(t, deps)
+	js, err := jetstream.New(f.svc.nc)
+	require.NoError(t, err)
+	store := kvstore.OpenSyncBucket(js)
+	f.reaper = NewReaper(f.svc, kvsync.NewLease(store, KeyReaper, "node-a", 300*time.Millisecond), deps)
+	f.ready(t)
+
+	ctx, cancel := context.WithCancel(t.Context())
+	done := make(chan struct{})
+	go func() { defer close(done); f.reaper.Run(ctx) }()
+	t.Cleanup(func() { cancel(); <-done })
+	require.Eventually(t, f.reaper.IsLeader, 5*time.Second, 5*time.Millisecond)
+
+	syncKV, err := store.KV(t.Context())
+	require.NoError(t, err)
+	_, err = syncKV.Put(t.Context(), KeyReaper, []byte(`{"holder":"node-b","generation":0}`))
+	require.NoError(t, err)
+	require.Eventually(t, func() bool { return !f.reaper.IsLeader() }, 5*time.Second, 5*time.Millisecond)
+
+	// Idle from here on, so only the lost lease keeps it from being reaped.
+	f.age(t, 10*time.Minute, 10*time.Minute)
+	time.Sleep(20 * time.Millisecond) // a sweep already running when the lease went
+	scrapes := f.transport.scrapeCount()
+	time.Sleep(100 * time.Millisecond)
+	_, found := f.current(t)
+	assert.True(t, found, "a reaper that lost the lease must not reclaim")
+	assert.Equal(t, scrapes, f.transport.scrapeCount(), "a reaper that lost the lease must not even scrape")
+
+	require.NoError(t, syncKV.Delete(t.Context(), KeyReaper))
+	require.Eventually(t, func() bool { _, found := f.current(t); return !found }, 5*time.Second, 5*time.Millisecond,
+		"re-elected, the reaper reclaims the idle endpoint")
 }

spinifex/domains/ochre/service.go

@@ -63,9 +63,6 @@ type Service struct {
 	nc    *nats.Conn
 	deps  ServiceDeps
 	store *endpointStore
-	// leader is the reaper's TTL-backed lease bucket, built alongside the
-	// endpoints store so both share this Service's one JetStream client.
-	leader *kvstore.Bucket
 
 	// ensureMu collapses concurrent Ensure calls FOR THE SAME KEY on THIS
 	// replica into one KV round trip: without it, N concurrent local callers
@@ -82,15 +79,14 @@ type Service struct {
 var _ EndpointService = (*Service)(nil)
 
 // NewService constructs a Service bound to nc, using deps for launch/capacity.
-// Both buckets are resolved lazily but memoised, so the get-or-create round
+// The endpoints bucket is resolved lazily but memoised, so the get-or-create round
 // trip is paid once for the daemon rather than once per KV call.
 func NewService(nc *nats.Conn, deps ServiceDeps) *Service {
 	js := serviceJetStream(nc)
 	return &Service{
-		nc:     nc,
+		nc:    nc,
-		deps:   deps,
+		deps:  deps,
-		store:  newEndpointStore(js),
+		store: newEndpointStore(js),
-		leader: newLeaderBucket(js),
 	}
 }
 

spinifex/domains/ochre/store.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/base64"
 	"errors"
-	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
@@ -21,30 +20,10 @@ const KVBucketEndpoints = "bedrock-endpoints"
 // not meaningful to retain, only the current one CAS-guards against races.
 const KVBucketEndpointsHistory = 1
 
-// KVBucketLeader holds the reaper's leader lease. Its TTL is what makes the
-// lease self-healing: a leader that dies without releasing it is replaced
-// within one TTL rather than never.
-const (
-	KVBucketLeader    = "bedrock-leader"
-	KVBucketLeaderTTL = 60 * time.Second
-	leaderKey         = "reaper"
-)
-
 // missingJetStream is the error every accessor reports when the Service was
 // built without a NATS connection to derive a JetStream client from.
 const missingJetStream = "bedrock service: nil nats connection"
 
-// newLeaderBucket returns the reaper's TTL-backed leader bucket. The lease
-// carries no value of its own, so it is a Bucket rather than a Store[T].
-func newLeaderBucket(js jetstream.JetStream) *kvstore.Bucket {
-	return kvstore.NewBucket(js, kvstore.Config{
-		Name:    KVBucketLeader,
-		History: 1,
-		TTL:     KVBucketLeaderTTL,
-		Missing: missingJetStream,
-	})
-}
-
 // EndpointKey returns the KV key for accountID's modelID endpoint record.
 // Model IDs contain ':' (e.g. "meta.llama3-2-1b-instruct-v1:0"), which NATS
 // rejects in a KV key, so the segment is base64url-encoded, mirroring

spinifex/domains/rds/instance/backup_test.go

@@ -823,7 +823,7 @@ func TestReconciler_RunsTheBackupWindowOnItsAccountPass(t *testing.T) {
 	rec.Agent = AgentState{InstanceID: testInstance, EngineHealth: EngineHealthHealthy, LastSeen: &now}
 	seedInstance(t, h.svc, rec)
 
-	require.NoError(t, onePass(t, NewReconciler(h.svc, "node-a")))
+	require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-a")))
 
 	assert.Len(t, h.snaps.created, 1)
 	assert.Len(t, h.automatedStamps(t, testDBID), 1)
@@ -840,7 +840,8 @@ func TestReconciler_ClusterLeaseFollowsLeadership(t *testing.T) {
 	require.NotNil(t, release)
 	assert.False(t, ok, "a node that does not hold the lease does not sweep")
 
-	require.True(t, h.rec.lease.TryAcquire(t.Context()))
+	_, err := lead(t, h.rec)
+	require.NoError(t, err)
 	release, ok = h.rec.AcquireClusterLease()
 	require.True(t, ok)
 

spinifex/domains/rds/instance/delete_test.go

@@ -202,7 +202,7 @@ func TestReconciler_AdoptsAnInterruptedFinalSnapshot(t *testing.T) {
 		},
 	}
 
-	require.NoError(t, onePass(t, NewReconciler(h.svc, "node-a")))
+	require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-a")))
 
 	assert.Empty(t, h.snaps.created, "the resumed teardown must adopt the cut snapshot, not create another")
 	stored, found := h.snapshotRecord(t, rec.FinalSnapshotIdentifier)
@@ -328,7 +328,7 @@ func TestReconciler_IsIdempotentAcrossARetriedTeardown(t *testing.T) {
 	rec := abandonedDelete(availableRecord())
 	rec.FinalSnapshotIdentifier = "orders-db-final"
 	seedInstance(t, h.svc, rec)
-	reconciler := NewReconciler(h.svc, "node-a")
+	reconciler := testReconciler(t, h.svc, "node-a")
 	require.NoError(t, onePass(t, reconciler))
 
 	// The record left behind by the interrupted attempt, replayed.
@@ -346,7 +346,7 @@ func TestReconciler_ResumesAnInterruptedDelete(t *testing.T) {
 	h := newLifecycleHarness(t, false)
 	seedInstance(t, h.svc, abandonedDelete(availableRecord()))
 
-	reconciler := NewReconciler(h.svc, "node-a")
+	reconciler := testReconciler(t, h.svc, "node-a")
 	require.NoError(t, onePass(t, reconciler))
 
 	assert.Equal(t, []string{testInstance}, h.launcher.terminated)
@@ -364,7 +364,7 @@ func TestReconciler_LeavesADeleteItsHandlerMayStillBeRunning(t *testing.T) {
 	rec.TransitionStartedAt = &started
 	seedInstance(t, h.svc, rec)
 
-	require.NoError(t, onePass(t, NewReconciler(h.svc, "node-a")))
+	require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-a")))
 
 	assert.Empty(t, h.launcher.terminated)
 	assert.True(t, h.recordExists(t, testDBID))
@@ -443,7 +443,7 @@ func TestReconciler_ResumesAnInterruptedStop(t *testing.T) {
 	rec.Status = StatusStopping
 	seedInstance(t, h.svc, rec)
 
-	reconciler := NewReconciler(h.svc, "node-a")
+	reconciler := testReconciler(t, h.svc, "node-a")
 	require.NoError(t, onePass(t, reconciler))
 
 	assert.Equal(t, []string{"stop:" + testInstance}, h.cmdr.calls)

spinifex/domains/rds/instance/lifecycle_test.go

@@ -578,7 +578,7 @@ func TestReconciler_DoesNotCallAStillRunningVMStopped(t *testing.T) {
 	rec.TransitionStartedAt = &started
 	seedInstance(t, h.svc, rec)
 
-	require.NoError(t, onePass(t, NewReconciler(h.svc, "node-a")))
+	require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-a")))
 
 	// Left stopping so the next pass retries; the bound is what ends it.
 	assert.Equal(t, StatusStopping, h.record(t).Status)
@@ -598,7 +598,7 @@ func TestReconciler_ResumesAStopTheLiveCallerIsAlreadyDriving(t *testing.T) {
 	rec.TransitionStartedAt = &started
 	seedInstance(t, h.svc, rec)
 
-	require.NoError(t, onePass(t, NewReconciler(h.svc, "node-a")))
+	require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-a")))
 
 	assert.Equal(t, StatusStopped, h.record(t).Status)
 }
@@ -770,7 +770,7 @@ func TestReconciler_CompletesARestartOnAHeartbeatFromTheRestartedEngine(t *testi
 			now := time.Now().UTC()
 			seedInstance(t, h.svc, restartingRecord(status, now.Add(-time.Minute), now))
 
-			require.NoError(t, onePass(t, NewReconciler(h.svc, "node-a")))
+			require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-a")))
 
 			rec := h.record(t)
 			assert.Equal(t, StatusAvailable, rec.Status)
@@ -789,7 +789,7 @@ func TestReconciler_CompletesARestartOnAPersistedHeartbeatInsideTheFloor(t *test
 	persisted := time.Now().UTC().Add(-HeartbeatStaleAfter - time.Minute)
 	seedInstance(t, h.svc, restartingRecord(StatusRebooting, persisted.Add(-time.Second), persisted))
 
-	require.NoError(t, onePass(t, NewReconciler(h.svc, "node-a")))
+	require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-a")))
 
 	assert.Equal(t, StatusAvailable, h.record(t).Status)
 }
@@ -803,7 +803,7 @@ func TestReconciler_IgnoresAHeartbeatPredatingTheRestart(t *testing.T) {
 	now := time.Now().UTC()
 	seedInstance(t, h.svc, restartingRecord(StatusRebooting, now, now.Add(-time.Minute)))
 
-	require.NoError(t, onePass(t, NewReconciler(h.svc, "node-a")))
+	require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-a")))
 
 	assert.Equal(t, StatusRebooting, h.record(t).Status)
 }
@@ -821,7 +821,7 @@ func TestReconciler_IgnoresAHeartbeatFromTheAgentTheRestartReplaces(t *testing.T
 	rec.Agent.StartedAt = &predecessor
 	seedInstance(t, h.svc, rec)
 
-	require.NoError(t, onePass(t, NewReconciler(h.svc, "node-a")))
+	require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-a")))
 	assert.Equal(t, StatusRebooting, h.record(t).Status)
 
 	restarted := started.Add(time.Second)
@@ -829,7 +829,7 @@ func TestReconciler_IgnoresAHeartbeatFromTheAgentTheRestartReplaces(t *testing.T
 	rec.Agent.StartedAt = &restarted
 	seedInstance(t, h.svc, rec)
 
-	require.NoError(t, onePass(t, NewReconciler(h.svc, "node-a")))
+	require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-a")))
 	assert.Equal(t, StatusAvailable, h.record(t).Status)
 }
 
@@ -842,7 +842,7 @@ func TestReconciler_MarksFailedWhenARestartOverrunsItsBound(t *testing.T) {
 	started := now.Add(-2 * transitionTimeout)
 	seedInstance(t, h.svc, restartingRecord(StatusStarting, started, started.Add(-time.Minute)))
 
-	require.NoError(t, onePass(t, NewReconciler(h.svc, "node-a")))
+	require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-a")))
 
 	rec := h.record(t)
 	assert.Equal(t, StatusFailed, rec.Status)
@@ -861,7 +861,7 @@ func TestReconciler_CarriesTheAgentsReasonIntoAFailedRestart(t *testing.T) {
 		`FATAL: invalid value for parameter "max_connections"`
 	seedInstance(t, h.svc, rec)
 
-	require.NoError(t, onePass(t, NewReconciler(h.svc, "node-a")))
+	require.NoError(t, onePass(t, testReconciler(t, h.svc, "node-a")))
 
 	stored := h.record(t)
 	assert.Equal(t, StatusFailed, stored.Status)

spinifex/domains/rds/instance/modify_test.go

@@ -72,7 +72,7 @@ func newModifyHarnessWithAgent(t *testing.T, agentFails bool) *modifyHarness {
 		ServingCertKeyBits: testServingCertKeyBits,
 		ApplyParamsTimeout: testApplyParamsTimeout,
 	})
-	h.rec = NewReconciler(h.svc, "node-a")
+	h.rec = testReconciler(t, h.svc, "node-a")
 	return h
 }
 

spinifex/domains/rds/instance/reconciler.go

@@ -13,17 +13,19 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/reconciler"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/nats-io/nats.go/jetstream"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 )
 
-// Lease and sweep timing. The holder refreshes well inside the bucket's 60s TTL,
+// KeyReconciler is the RDS reconciler's leader lease in the sync bucket.
-// so a leader that dies is replaced within one TTL rather than one refresh.
+const KeyReconciler = "rds.reconciler"
+
+// Lease and sweep timing. A follower asks again at the lease's own renewal
+// rate, so a leader that dies is replaced within about one TTL.
 const (
-	reconcilerLeaderKey = "reconciler"
+	reconcilerLeaseTTL = 60 * time.Second
-	leaseRefresh        = 20 * time.Second
+	leaseRefresh       = reconcilerLeaseTTL / 3
-	reconcileInterval   = 15 * time.Second
+	reconcileInterval  = 15 * time.Second
 
 	// The drift backstop, and so also the cadence on which a backup or
 	// maintenance window is noticed. A window is a half-hour slot, so noticing
@@ -81,11 +83,8 @@ type InstanceStateResolver interface {
 // the failure classifier that gives a settled instance an honest health state.
 // The backup sweep extends the same loop.
 type Reconciler struct {
-	svc    *Service
+	svc   *Service
-	holder string
+	lease *kvsync.Lease
-
-	lease    *kvlease.Lease
-	leaseErr error
 
 	// The payloads already reported as stuck pending. The condition persists
 	// until an operator acts, so without this the event would be re-recorded
@@ -100,34 +99,24 @@ type Reconciler struct {
 	systemSGID string
 }
 
-// NewReconciler returns a Reconciler for svc; holder identifies this daemon in the lease. A bad lease
+// NewReconcilerLease returns the RDS reconciler's leader lease for holder.
-// config is kept, and Run logs it and returns.
+func NewReconcilerLease(store kvsync.KVStore, holder string) *kvsync.Lease {
-func NewReconciler(svc *Service, holder string) *Reconciler {
+	return kvsync.NewLease(store, KeyReconciler, holder, reconcilerLeaseTTL)
-	r := &Reconciler{svc: svc, holder: holder, reportedPending: make(map[string]string)}
+}
-	r.lease, r.leaseErr = kvlease.New(kvlease.Config{
+
-		Name:   "rds/reconciler",
+// NewReconciler returns a Reconciler for svc that leads while it holds lease.
-		Bucket: r.leaderBucket,
+func NewReconciler(svc *Service, lease *kvsync.Lease) *Reconciler {
-		Key:    reconcilerLeaderKey,
+	return &Reconciler{svc: svc, lease: lease, reportedPending: make(map[string]string)}
-		Holder: holder,
-		TTL:    KVBucketRDSLeaderTTL,
-		Renew:  leaseRefresh,
-		Retry:  leaseRefresh,
-	})
-	return r
 }
 
 // Run drives the leadership and reconcile loop until ctx is canceled. Intended as a daemon-boot
 // goroutine; panics are the caller's recover concern.
 func (r *Reconciler) Run(ctx context.Context) {
-	if r.leaseErr != nil {
-		slog.ErrorContext(ctx, "rds reconciler: lease config invalid", "holder", r.holder, "err", r.leaseErr)
-		return
-	}
 	leadershipCtx, cancelLeadership := context.WithCancel(ctx)
 	leadershipDone := make(chan struct{})
 	go func() {
 		defer close(leadershipDone)
-		r.lease.Run(leadershipCtx)
+		kvsync.RunElection(leadershipCtx, r.lease, nil, nil)
 	}()
 	defer func() {
 		cancelLeadership()
@@ -162,7 +151,7 @@ func (r *Reconciler) reconcilePass(ctx context.Context) (time.Duration, error) {
 	}
 	revisit, err := r.reconcileOnce(ctx)
 	if err != nil {
-		slog.ErrorContext(ctx, "rds reconciler: pass failed", "holder", r.holder, "err", err)
+		slog.ErrorContext(ctx, "rds reconciler: pass failed", "key", r.lease.Key(), "err", err)
 	}
 	// Already logged, so it is not returned again: the loop would log it a
 	// second time under its own message.
@@ -176,14 +165,6 @@ func (r *Reconciler) AcquireClusterLease() (func(), bool) {
 	return func() {}, r.lease.Held()
 }
 
-func (r *Reconciler) leaderBucket(ctx context.Context) (jetstream.KeyValue, error) {
-	js, err := r.svc.js()
-	if err != nil {
-		return nil, err
-	}
-	return InitLeaderBucket(ctx, js)
-}
-
 // One pass across every tenant. A bucket that cannot be read stops the pass
 // with an error rather than being skipped silently, so a partial view shows up
 // in the log instead of looking like a fleet with nothing to do.

spinifex/domains/rds/instance/reconciler_test.go

@@ -10,11 +10,32 @@ import (
 	"github.com/aws/aws-sdk-go/service/rds"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+// testReconciler is holder's reconciler on a sync bucket of its own, as a
+// separate daemon's would be.
+func testReconciler(t *testing.T, svc *Service, holder string) *Reconciler {
+	t.Helper()
+	js, err := svc.js()
+	require.NoError(t, err)
+	return NewReconciler(svc, NewReconcilerLease(kvstore.OpenSyncBucket(js), holder))
+}
+
+// lead takes r's lease, and unlocks it when the test ends.
+func lead(t *testing.T, r *Reconciler) (kvsync.Token, error) {
+	t.Helper()
+	tok, err := r.lease.TryLock(t.Context())
+	if err == nil {
+		t.Cleanup(func() { r.lease.Unlock(context.Background(), tok) })
+	}
+	return tok, err
+}
+
 // onePass runs one reconcile and keeps only its error, for the tests that assert
 // on what converged rather than on when the loop should next run. The tests that
 // do care about the deadline call reconcileOnce directly.
@@ -70,7 +91,7 @@ func newReconcileHarness(t *testing.T, opts ...func(*Deps)) *reconcileHarness {
 		opt(&deps)
 	}
 	svc := NewService(nc, testRegion).WithDeps(deps)
-	return &reconcileHarness{svc: svc, rec: NewReconciler(svc, "node-a"), state: state, nc: nc}
+	return &reconcileHarness{svc: svc, rec: testReconciler(t, svc, "node-a"), state: state, nc: nc}
 }
 
 // healthyRecord is a creating instance whose agent has just reported a healthy
@@ -377,18 +398,21 @@ func TestReconciler_RetriesTheSystemNICMoveAfterAFailure(t *testing.T) {
 func TestReconciler_ElectsASingleLeader(t *testing.T) {
 	t.Parallel()
 	h := newReconcileHarness(t)
-	other := NewReconciler(h.svc, "node-b")
+	other := testReconciler(t, h.svc, "node-b")
 
-	assert.True(t, h.rec.lease.TryAcquire(t.Context()))
+	tok, err := lead(t, h.rec)
-	assert.False(t, other.lease.TryAcquire(t.Context()))
+	require.NoError(t, err)
-
+	_, err = lead(t, other)
-	// The holder reports the leasership it already holds, and does not hand it away.
+	assert.ErrorAs(t, err, new(*kvsync.HeldError))
-	assert.True(t, h.rec.lease.TryAcquire(t.Context()))
+	assert.True(t, h.rec.lease.Held())
+	assert.False(t, other.lease.Held())
 
 	// Releasing on shutdown hands over immediately instead of after the TTL.
-	h.rec.lease.Release(t.Context())
+	h.rec.lease.Unlock(t.Context(), tok)
-	assert.True(t, other.lease.TryAcquire(t.Context()))
+	_, err = lead(t, other)
-	assert.False(t, h.rec.lease.TryAcquire(t.Context()))
+	require.NoError(t, err)
+	_, err = lead(t, h.rec)
+	assert.ErrorAs(t, err, new(*kvsync.HeldError))
 }
 
 func TestReconciler_RunHoldsLeadershipIndependentlyOfTheReconcileLoop(t *testing.T) {
@@ -404,21 +428,23 @@ func TestReconciler_RunHoldsLeadershipIndependentlyOfTheReconcileLoop(t *testing
 	cancel()
 	<-done
 
-	other := NewReconciler(h.svc, "node-b")
+	other := testReconciler(t, h.svc, "node-b")
-	assert.True(t, other.lease.TryAcquire(t.Context()),
+	_, err := lead(t, other)
-		"Run returned with the lease key still present")
+	assert.NoError(t, err, "Run returned with the lease key still present")
 }
 
 // A node that never won the lease must not delete the holder's key on shutdown.
 func TestReconciler_RelinquishOnlyReleasesItsOwnLease(t *testing.T) {
 	t.Parallel()
 	h := newReconcileHarness(t)
-	other := NewReconciler(h.svc, "node-b")
+	other := testReconciler(t, h.svc, "node-b")
 
-	require.True(t, h.rec.lease.TryAcquire(t.Context()))
+	tok, err := lead(t, h.rec)
-	other.lease.Release(t.Context())
+	require.NoError(t, err)
+	other.lease.Unlock(t.Context(), tok)
 
-	assert.False(t, other.lease.TryAcquire(t.Context()), "the original holder still owns the lease")
+	_, err = lead(t, other)
+	assert.ErrorAs(t, err, new(*kvsync.HeldError), "the original holder still owns the lease")
 }
 
 // An overrun modify failed on a fresh read must not fail a delete accepted since
@@ -440,3 +466,46 @@ func TestTransitionFresh_LeavesAnAcceptedDelete(t *testing.T) {
 	require.True(t, found)
 	assert.Equal(t, StatusDeleting, stored.Status)
 }
+
+// The retention sweep deletes customer snapshots under AcquireClusterLease, so
+// a node whose lease was taken over must stop sweeping and reconciling at once,
+// and only start again once it is re-elected.
+func TestReconciler_ATakenOverLeaseStopsTheSweepUntilReElection(t *testing.T) {
+	t.Parallel()
+	h := newReconcileHarness(t)
+	js, err := h.svc.js()
+	require.NoError(t, err)
+	store := kvstore.OpenSyncBucket(js)
+	h.rec.lease = kvsync.NewLease(store, KeyReconciler, "node-a", 300*time.Millisecond)
+	seedInstance(t, h.svc, healthyRecord())
+
+	_, err = lead(t, h.rec)
+	require.NoError(t, err)
+	_, ok := h.rec.AcquireClusterLease()
+	require.True(t, ok)
+
+	syncKV, err := store.KV(t.Context())
+	require.NoError(t, err)
+	_, err = syncKV.Put(t.Context(), KeyReconciler, []byte(`{"holder":"node-b","generation":0}`))
+	require.NoError(t, err)
+	require.Eventually(t, func() bool { return !h.rec.lease.Held() }, 5*time.Second, 10*time.Millisecond,
+		"the takeover must cost this node the lease")
+
+	_, ok = h.rec.AcquireClusterLease()
+	assert.False(t, ok, "a node that lost the lease must not sweep snapshots")
+	revisit, err := h.rec.reconcilePass(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, leaseRefresh, revisit)
+	status, _ := h.statusOf(t, testDBID)
+	assert.Equal(t, StatusCreating, status, "a node that lost the lease must not reconcile")
+
+	require.NoError(t, syncKV.Delete(t.Context(), KeyReconciler))
+	_, err = lead(t, h.rec)
+	require.NoError(t, err, "re-election")
+	_, ok = h.rec.AcquireClusterLease()
+	assert.True(t, ok)
+	_, err = h.rec.reconcilePass(t.Context())
+	require.NoError(t, err)
+	status, _ = h.statusOf(t, testDBID)
+	assert.NotEqual(t, StatusCreating, status, "the re-elected node reconciles again")
+}

spinifex/domains/rds/instance/recovery_test.go

@@ -211,7 +211,7 @@ func TestReconciler_FailsOnAClockStampedByAnEarlierLeader(t *testing.T) {
 	seedInstance(t, h.svc, rec)
 
 	// A different node holding the lease, with no beats of its own in memory.
-	other := NewReconciler(h.svc, "node-b")
+	other := testReconciler(t, h.svc, "node-b")
 	require.NoError(t, onePass(t, other))
 
 	stored := h.recordOf(t, testDBID)

spinifex/domains/rds/instance/snapshot_test.go

@@ -568,7 +568,7 @@ func TestProjectDBSnapshot_ReportsProgressOnlyWhenTheDataExists(t *testing.T) {
 func TestReconciler_AdoptsTheEC2SnapshotOfAnUnfinishedDBSnapshot(t *testing.T) {
 	t.Parallel()
 	h := newSnapshotHarness(t, false)
-	rec := NewReconciler(h.svc, "node-a")
+	rec := testReconciler(t, h.svc, "node-a")
 
 	kv, err := h.svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
@@ -601,7 +601,7 @@ func TestReconciler_AdoptsTheEC2SnapshotOfAnUnfinishedDBSnapshot(t *testing.T) {
 func TestReconciler_DoesNotAdoptAnotherAccountsSnapshot(t *testing.T) {
 	t.Parallel()
 	h := newSnapshotHarness(t, false)
-	reconciler := NewReconciler(h.svc, "node-a")
+	reconciler := testReconciler(t, h.svc, "node-a")
 
 	kv, err := h.svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
@@ -636,7 +636,7 @@ func TestReconciler_KeepsCreatingSnapshotWhenEC2LookupIsIncomplete(t *testing.T)
 	t.Parallel()
 	h := newSnapshotHarness(t, false)
 	h.snaps.describeErr = errors.New("snapshot metadata temporarily unavailable")
-	reconciler := NewReconciler(h.svc, "node-a")
+	reconciler := testReconciler(t, h.svc, "node-a")
 
 	kv, err := h.svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
@@ -660,7 +660,7 @@ func TestReconciler_KeepsCreatingSnapshotWhenEC2LookupIsIncomplete(t *testing.T)
 func TestReconciler_DoesNotDeleteAConcurrentlyCompletedSnapshot(t *testing.T) {
 	t.Parallel()
 	h := newSnapshotHarness(t, false)
-	reconciler := NewReconciler(h.svc, "node-a")
+	reconciler := testReconciler(t, h.svc, "node-a")
 
 	kv, err := h.svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
@@ -693,7 +693,7 @@ func TestReconciler_DoesNotDeleteAConcurrentlyCompletedSnapshot(t *testing.T) {
 func TestReconciler_RejectsAmbiguousEC2Snapshots(t *testing.T) {
 	t.Parallel()
 	h := newSnapshotHarness(t, false)
-	reconciler := NewReconciler(h.svc, "node-a")
+	reconciler := testReconciler(t, h.svc, "node-a")
 
 	kv, err := h.svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
@@ -728,7 +728,7 @@ func TestReconciler_RejectsAmbiguousEC2Snapshots(t *testing.T) {
 func TestReconciler_WithdrawsADBSnapshotWhoseDataWasNeverCut(t *testing.T) {
 	t.Parallel()
 	h := newSnapshotHarness(t, false)
-	rec := NewReconciler(h.svc, "node-a")
+	rec := testReconciler(t, h.svc, "node-a")
 
 	kv, err := h.svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
@@ -753,7 +753,7 @@ func TestReconciler_WithdrawsADBSnapshotWhoseDataWasNeverCut(t *testing.T) {
 func TestReconciler_LeavesAFreshCreatingSnapshotAlone(t *testing.T) {
 	t.Parallel()
 	h := newSnapshotHarness(t, false)
-	rec := NewReconciler(h.svc, "node-a")
+	rec := testReconciler(t, h.svc, "node-a")
 
 	kv, err := h.svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
@@ -780,7 +780,7 @@ func TestReconciler_ReturnsAnInstanceStuckInBackingUp(t *testing.T) {
 		t.Run(string(resume), func(t *testing.T) {
 			t.Parallel()
 			h := newSnapshotHarness(t, false)
-			reconciler := NewReconciler(h.svc, "node-a")
+			reconciler := testReconciler(t, h.svc, "node-a")
 
 			started := time.Now().UTC().Add(-2 * transitionTimeout)
 			rec := availableRecord()
@@ -807,7 +807,7 @@ func TestReconciler_ReturnsAnInstanceStuckInBackingUp(t *testing.T) {
 func TestReconciler_LeavesAnInFlightSnapshotAlone(t *testing.T) {
 	t.Parallel()
 	h := newSnapshotHarness(t, false)
-	reconciler := NewReconciler(h.svc, "node-a")
+	reconciler := testReconciler(t, h.svc, "node-a")
 
 	started := time.Now().UTC()
 	rec := availableRecord()

spinifex/domains/rds/instance/store.go

@@ -9,9 +9,7 @@ import (
 	"errors"
 	"fmt"
 	"strings"
-	"time"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
@@ -29,13 +27,6 @@ const (
 	KVBucketRDSSystem        = "rds-system"
 	KVBucketRDSSystemVersion = 1
 	KVBucketRDSSystemHistory = 1
-
-	// "spinifex-rds-leader" holds the single reconciler lease. Its TTL expires a
-	// lease whose holder died mid-cycle, so control work resumes without an
-	// operator.
-	KVBucketRDSLeader        = "spinifex-rds-leader"
-	KVBucketRDSLeaderVersion = 1
-	KVBucketRDSLeaderTTL     = 60 * time.Second
 )
 
 // Key-path helpers for the per-account bucket:
@@ -230,16 +221,6 @@ func GetOrCreateSystemBucket(ctx context.Context, js jetstream.JetStream) (jetst
 	return kv, nil
 }
 
-// InitLeaderBucket creates (or attaches to) the shared spinifex-rds-leader
-// bucket used for the reconciler's leader-lease CAS lock.
-func InitLeaderBucket(ctx context.Context, js jetstream.JetStream) (jetstream.KeyValue, error) {
-	return kvlease.OpenBucket(ctx, js, kvlease.BucketConfig{
-		Name:    KVBucketRDSLeader,
-		TTL:     KVBucketRDSLeaderTTL,
-		Version: KVBucketRDSLeaderVersion,
-	})
-}
-
 // AccountBucketNames lists every RDS per-account bucket in the cluster. The reconciler and the DNS
 // desired-set both need a cross-tenant view, which only this provides.
 func AccountBucketNames(ctx context.Context, js jetstream.JetStream) ([]string, error) {

spinifex/domains/route53/realizer.go

@@ -39,8 +39,9 @@ type Realizer struct {
 	store *Store
 	pub   Publisher
 
-	// Elect wins this pass for the calling node; nil means always.
+	// Elect wins this pass for the calling node; nil means always. A lost
-	Elect func(ctx context.Context) (release func(), ok bool)
+	// election's revisit is the pass's own.
+	Elect func(ctx context.Context) (release func(), revisit time.Duration, ok bool)
 	// Reserved reports platform zone names, which are never written or deleted.
 	Reserved func(name string) bool
 	// Now is the clock; tests replace it.
@@ -75,9 +76,9 @@ func (r *Realizer) Run(ctx context.Context) {
 // a prompt revisit, since nothing in KV changes when it does.
 func (r *Realizer) Reconcile(ctx context.Context) (time.Duration, error) {
 	if r.Elect != nil {
-		release, ok := r.Elect(ctx)
+		release, revisit, ok := r.Elect(ctx)
 		if !ok {
-			return 0, nil
+			return revisit, nil
 		}
 		defer release()
 	}

spinifex/domains/route53/realizer_test.go

@@ -279,13 +279,14 @@ func TestRealizerDoesNothingWhenNotElected(t *testing.T) {
 	svc := newService(t)
 	ns := newFakeNorthstar(1)
 	r := newRealizer(svc, ns)
-	r.Elect = func(context.Context) (func(), bool) { return nil, false }
+	r.Elect = func(context.Context) (func(), time.Duration, bool) { return nil, 61 * time.Second, false }
 	id := createZone(t, svc, acctA, "example.com")
 	setState(t, svc, acctA, id, route53.StateActive)
 
-	_, err := r.Reconcile(ctx)
+	revisit, err := r.Reconcile(ctx)
 	require.NoError(t, err)
 	assert.False(t, ns.has("example.com"))
+	assert.Equal(t, 61*time.Second, revisit, "a lost election's revisit is the pass's, so a dead leader is replaced")
 }
 
 func TestRealizerRendersEveryRecordType(t *testing.T) {

spinifex/domains/route53/store.go

@@ -14,10 +14,10 @@ import (
 const (
 	KVBucketRoute53        = "spinifex-route53"
 	KVBucketRoute53Version = 1
-	// KVBucketRoute53Realize elects the one node that publishes zones.
+	// KeyRealize is the sync-bucket lease electing the node that publishes zones.
-	KVBucketRoute53Realize = "spinifex-route53-realize"
+	KeyRealize = "route53.realize"
-	// KVBucketRoute53Verify elects the one node that checks delegations.
+	// KeyVerify is the sync-bucket lease electing the node that checks delegations.
-	KVBucketRoute53Verify = "spinifex-route53-verify"
+	KeyVerify = "route53.verify"
 
 	keyPrefixZone     = "zone."
 	keyPrefixRRSets   = "rrsets."

spinifex/domains/route53/verifier.go

@@ -45,8 +45,9 @@ type Verifier struct {
 	Platform func(ctx context.Context, name string) (d Delegation, ok bool, err error)
 	// Public asks the parent's authoritative servers, without recursion.
 	Public func(ctx context.Context, name string) (Delegation, error)
-	// Elect wins this pass for the calling node; nil means always.
+	// Elect wins this pass for the calling node; nil means always. A lost
-	Elect func(ctx context.Context) (release func(), ok bool)
+	// election's revisit is the pass's own.
+	Elect func(ctx context.Context) (release func(), revisit time.Duration, ok bool)
 	Now   func() time.Time
 	// Interval is the pass period; zero means one minute.
 	Interval time.Duration
@@ -72,9 +73,9 @@ func (v *Verifier) Run(ctx context.Context) {
 // Reconcile checks every zone that is due.
 func (v *Verifier) Reconcile(ctx context.Context) (time.Duration, error) {
 	if v.Elect != nil {
-		release, ok := v.Elect(ctx)
+		release, revisit, ok := v.Elect(ctx)
 		if !ok {
-			return 0, nil
+			return revisit, nil
 		}
 		defer release()
 	}

spinifex/domains/route53/verifier_test.go

@@ -12,6 +12,8 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/mulgadc/spinifex/internal/testkit"
+	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
 	"github.com/mulgadc/spinifex/spinifex/domains/route53"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 )
@@ -330,3 +332,35 @@ func TestDuplicateActiveZoneReturnsToPendingAndItsChangesSettle(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, "INSYNC", changeStatus(t, svc, acctB, aws.StringValue(out.ChangeInfo.Id)))
 }
+
+// Delegation checks are elected: while a peer holds the verify lease this node
+// looks nothing up and changes no zone, and it asks to come back once a dead
+// holder's lease could be taken over.
+func TestVerifierChecksNothingWhileAPeerHoldsTheLease(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	svc := route53.NewService(route53.NewStore(js))
+	svc.Nameservers = ours
+	dns := &fakeDNS{parents: map[string][]string{"example.com": ours()}}
+	v := route53.NewVerifier(svc.Store())
+	v.Public = dns.lookup
+	store := kvstore.OpenSyncBucket(js)
+	lease := reconcilelock.NewLeaderLease(store, route53.KeyVerify, "node-a")
+	v.Elect = func(ctx context.Context) (func(), time.Duration, bool) { return reconcilelock.ElectPass(ctx, lease) }
+	id := createZone(t, svc, acctA, "example.com")
+
+	syncKV, err := store.KV(ctx)
+	require.NoError(t, err)
+	_, err = syncKV.Put(ctx, route53.KeyVerify, []byte(`{"holder":"node-b","generation":1}`))
+	require.NoError(t, err)
+
+	revisit, err := v.Reconcile(ctx)
+	require.NoError(t, err)
+	assert.Zero(t, dns.lookups, "a node that does not hold the lease must not check delegations")
+	assert.Equal(t, route53.StatePendingNameservers, zone(t, svc, acctA, id).State)
+	assert.Positive(t, revisit, "a held pass comes back so a dead holder is replaced")
+	assert.LessOrEqual(t, revisit, reconcilelock.LeaderTTL/3, "the retry comes within a renewal interval")
+
+	require.NoError(t, syncKV.Delete(ctx, route53.KeyVerify))
+	pass(t, v)
+	assert.Equal(t, route53.StateActive, zone(t, svc, acctA, id).State, "elected, the node checks again")
+}

spinifex/foundation/state/kvlease/bucket.go

@@ -1,119 +0,0 @@
-package kvlease
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"log/slog"
-	"time"
-
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
-	telemetry "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
-	"github.com/nats-io/nats.go"
-	"github.com/nats-io/nats.go/jetstream"
-)
-
-// Bounded wait for JetStream quorum on cold multi-node start. Vars (not consts) so tests can shrink them.
-var (
-	bucketRetryFor  = 60 * time.Second
-	bucketRetryStep = 1 * time.Second
-)
-
-// BucketConfig describes a lease bucket. History is not a field: a lease key has
-// no history worth keeping, so every lease bucket is created with one.
-type BucketConfig struct {
-	Name string
-
-	// TTL expires a lease its holder died without releasing, so a crashed leader
-	// does not park the key forever. It must match the lease's own Config.TTL.
-	TTL time.Duration
-
-	// Version runs the registered schema migrations for the bucket after it is
-	// opened. Zero skips them, for a bucket holding nothing but lease keys.
-	Version int
-}
-
-// OpenBucket attaches to a lease bucket, creating it only when genuinely absent,
-// and runs any schema migration registered for it.
-//
-// Attach before create, because callers re-open this on every reconcile tick:
-// CreateKeyValue against a bucket that already exists is a STREAM.CREATE the
-// meta leader answers with an error, so creating first bills one per tick.
-//
-// A leader lease is the worst thing here to lose to a single node. Nobody can
-// acquire a lease whose bucket has no quorum, so every reconciler that shares
-// it stops at once, cluster-wide, at exactly the moment a node has failed.
-func OpenBucket(ctx context.Context, js jetstream.JetStream, cfg BucketConfig) (jetstream.KeyValue, error) {
-	replicas, err := clustersize.Replicas()
-	if err != nil {
-		return nil, fmt.Errorf("kvlease: open or create lease bucket %s: %w", cfg.Name, err)
-	}
-
-	kv, err := js.KeyValue(ctx, cfg.Name)
-	switch {
-	case errors.Is(err, jetstream.ErrBucketNotFound):
-		kv, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{
-			Bucket:   cfg.Name,
-			History:  1,
-			TTL:      cfg.TTL,
-			Replicas: replicas,
-		})
-	case err == nil:
-		// A failed raise must not fail the open. This runs on every reconcile
-		// tick, so refusing here would stop every reconciler sharing the bucket
-		// — the outage this replica count exists to prevent — over a bucket that
-		// is present and quorate at the count it already has.
-		kvstore.TryRaiseBucketReplicas(ctx, js, cfg.Name, replicas)
-	}
-	if err != nil {
-		return nil, fmt.Errorf("kvlease: open or create lease bucket %s: %w", cfg.Name, err)
-	}
-	if cfg.Version > 0 {
-		if err := migrate.DefaultRegistry.RunKV(ctx, cfg.Name, kv, cfg.Version); err != nil {
-			return nil, fmt.Errorf("migrate %s: %w", cfg.Name, err)
-		}
-	}
-	slog.DebugContext(ctx, "kvlease: lease bucket ready", "bucket", cfg.Name, "ttl_ms", telemetry.Millis(cfg.TTL))
-	return kv, nil
-}
-
-// NATSBucket is OpenBucket as a BucketFunc, retrying while JetStream is still
-// forming quorum on a cold multi-node start. Callers with their own bucket
-// initialiser should pass that instead.
-func NATSBucket(nc *nats.Conn, bucket string, ttl time.Duration) BucketFunc {
-	return func(ctx context.Context) (jetstream.KeyValue, error) {
-		js, err := jetstream.New(nc)
-		if err != nil {
-			return nil, fmt.Errorf("kvlease: JetStream unavailable: %w", err)
-		}
-		deadline := time.Now().Add(bucketRetryFor)
-		for {
-			kv, err := OpenBucket(ctx, js, BucketConfig{Name: bucket, TTL: ttl})
-			if err == nil {
-				return kv, nil
-			}
-			// An undeclared cluster size does not become declared by waiting, so
-			// it surfaces now rather than after the whole retry window.
-			if clustersize.Permanent(err) {
-				return nil, err
-			}
-			if time.Now().After(deadline) {
-				return nil, fmt.Errorf("kvlease: KV bucket %q unreachable after %s: %w", bucket, bucketRetryFor, err)
-			}
-			// A shutdown mid-wait must not sit out the remaining retry window.
-			select {
-			case <-ctx.Done():
-				return nil, ctx.Err()
-			case <-time.After(bucketRetryStep):
-			}
-		}
-	}
-}
-
-// StaticBucket adapts an already-open bucket to a BucketFunc, for callers handed
-// a KeyValue rather than opening one themselves.
-func StaticBucket(kv jetstream.KeyValue) BucketFunc {
-	return func(context.Context) (jetstream.KeyValue, error) { return kv, nil }
-}

spinifex/foundation/state/kvlease/bucket_test.go

@@ -1,76 +0,0 @@
-package kvlease_test
-
-import (
-	"testing"
-	"time"
-
-	"github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// TestOpenBucket_OpensALeaseBucketTheClusterCannotYetRaise is the assertion that
-// keeps this fix from becoming the outage it prevents.
-//
-// OpenBucket runs on every reconcile tick. If a raise it cannot place failed the
-// open, no reconciler could acquire its lease, and every one of them would stop
-// cluster-wide — over a lease bucket that is present and quorate at the replica
-// count it already has. The config naming a node that is not serving yet is the
-// documented way a cluster grows, so this is a state to tolerate, not to refuse.
-func TestOpenBucket_OpensALeaseBucketTheClusterCannotYetRaise(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
-	defer nc.Close()
-
-	cfg := kvlease.BucketConfig{Name: "lease-growing-cluster", TTL: time.Minute}
-	kv, err := kvlease.OpenBucket(t.Context(), js, cfg)
-	require.NoError(t, err)
-	_, err = kv.Put(t.Context(), "leader", []byte("node1"))
-	require.NoError(t, err)
-
-	// One embedded server cannot hold three replicas, so the raise must fail.
-	clustersize.RedeclareForTest(t, 3)
-
-	reopened, err := kvlease.OpenBucket(t.Context(), js, cfg)
-	require.NoError(t, err, "a reconciler must still be able to reach its lease bucket")
-	entry, err := reopened.Get(t.Context(), "leader")
-	require.NoError(t, err)
-	assert.Equal(t, "node1", string(entry.Value()))
-}
-
-// TestOpenBucket_RefusesWhenClusterSizeIsUndeclared pins the fail-closed half:
-// creating a lease bucket without knowing the cluster size must not guess at one
-// replica, which is the state that makes a lease unacquirable when a node fails.
-func TestOpenBucket_RefusesWhenClusterSizeIsUndeclared(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
-	defer nc.Close()
-
-	clustersize.RedeclareForTest(t, 0)
-
-	_, err := kvlease.OpenBucket(t.Context(), js, kvlease.BucketConfig{Name: "lease-undeclared", TTL: time.Minute})
-	require.ErrorIs(t, err, clustersize.ErrUndeclared)
-}
-
-// TestNATSBucket_StopsOnAConfigFault covers the retry loop rather than the open.
-//
-// The loop exists because a cold multi-node start has no JetStream quorum yet, and
-// waiting for one is right. An undeclared cluster size never becomes declared by
-// waiting, so a loop that cannot tell them apart holds a reconciler for its whole
-// retry window and then reports the bucket as unreachable — which was never the
-// problem.
-func TestNATSBucket_StopsOnAConfigFault(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	defer nc.Close()
-
-	clustersize.RedeclareForTest(t, 0)
-
-	start := time.Now()
-	_, err := kvlease.NATSBucket(nc, "lease-config-fault", time.Minute)(t.Context())
-	elapsed := time.Since(start)
-
-	require.ErrorIs(t, err, clustersize.ErrUndeclared,
-		"the error has to name the config fault, not the bucket it never reached")
-	require.Lessf(t, elapsed, 10*time.Second,
-		"returned after %s; a permanent fault must not consume the retry window", elapsed)
-}

spinifex/foundation/state/kvlease/lease.go

@@ -1,257 +0,0 @@
-// Package kvlease elects a single holder of a JetStream KV key. The key's TTL
-// bounds crash recovery; background renewal keeps it alive while the holder
-// works, so a pass that outlives the TTL does not lose the lease underneath it.
-package kvlease
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"log/slog"
-	"sync"
-	"time"
-
-	"github.com/nats-io/nats.go/jetstream"
-)
-
-// releaseTimeout bounds the delete on the way out, which runs on a detached
-// context and so cannot inherit the caller's deadline.
-const releaseTimeout = 5 * time.Second
-
-// heldError reports the node that owns the key when a claim loses to it.
-type heldError struct{ holder string }
-
-func (e *heldError) Error() string { return "kvlease: lease held by " + e.holder }
-
-// BucketFunc returns the KV bucket holding the lease key. It is injected because
-// each subsystem initialises its own bucket, some with migrations attached.
-type BucketFunc func(context.Context) (jetstream.KeyValue, error)
-
-// Config describes a single lease. Name appears in logs; Retry applies only to Run.
-type Config struct {
-	Name   string
-	Bucket BucketFunc
-	Key    string
-	Holder string
-	Attrs  []any
-
-	TTL   time.Duration // must match the bucket's TTL
-	Renew time.Duration // gap between refreshes
-	Retry time.Duration // session mode only: non-leader re-attempt interval
-
-	// Edges fire per transition, never under the lock
-	OnGained func(context.Context) error
-	OnLost   func()
-}
-
-// Lease is a CAS-guarded claim on a single key. The zero value is unusable; call New.
-type Lease struct {
-	cfg  Config
-	mu   sync.Mutex
-	kv   jetstream.KeyValue
-	rev  uint64
-	held bool
-	stop context.CancelFunc
-	lost chan struct{}
-}
-
-// New validates cfg and returns an unclaimed lease. Renew must leave room for a failed refresh,
-// so it may be at most half the TTL.
-func New(cfg Config) (*Lease, error) {
-	switch {
-	case cfg.Bucket == nil:
-		return nil, errors.New("kvlease: Bucket is required")
-	case cfg.Key == "":
-		return nil, errors.New("kvlease: Key is required")
-	case cfg.Holder == "":
-		return nil, errors.New("kvlease: Holder is required")
-	case cfg.TTL <= 0:
-		return nil, errors.New("kvlease: TTL must be positive")
-	case cfg.Renew <= 0 || cfg.Renew*2 > cfg.TTL:
-		return nil, fmt.Errorf("kvlease: Renew must be in (0, TTL/2]; got renew=%s ttl=%s", cfg.Renew, cfg.TTL)
-	}
-	if cfg.Name == "" {
-		cfg.Name = cfg.Key
-	}
-	if cfg.Retry <= 0 {
-		cfg.Retry = cfg.Renew
-	}
-	return &Lease{cfg: cfg, lost: make(chan struct{})}, nil
-}
-
-// Held reports whether this node currently holds the lease.
-func (l *Lease) Held() bool {
-	l.mu.Lock()
-	defer l.mu.Unlock()
-	return l.held
-}
-
-// logArgs prefixes the lease identity onto a log line's own fields. The local
-// node logs as "node"; "holder" is reserved for whoever owns the key.
-func (l *Lease) logArgs(extra ...any) []any {
-	args := make([]any, 0, 4+len(l.cfg.Attrs)+len(extra))
-	args = append(args, "lease", l.cfg.Name, "node", l.cfg.Holder)
-	args = append(args, l.cfg.Attrs...)
-	return append(args, extra...)
-}
-
-// TryAcquire makes one attempt to claim the key, starting background renewal and firing OnGained on
-// success. A node that already holds the lease is reported as still holding it without touching the key.
-func (l *Lease) TryAcquire(ctx context.Context) bool {
-	if l.Held() {
-		return true
-	}
-	kv, err := l.cfg.Bucket(ctx)
-	if err != nil {
-		slog.WarnContext(ctx, "kvlease: bucket unavailable", l.logArgs("err", err)...)
-		return false
-	}
-	rev, err := l.claim(ctx, kv)
-	if err != nil {
-		if held, ok := errors.AsType[*heldError](err); ok {
-			slog.DebugContext(ctx, "kvlease: held by another node", l.logArgs("holder", held.holder)...)
-		}
-		return false
-	}
-
-	renewCtx, stop := context.WithCancel(ctx)
-	l.mu.Lock()
-	if l.stop != nil {
-		l.stop()
-	}
-	l.lost = make(chan struct{})
-	l.kv, l.rev, l.held, l.stop = kv, rev, true, stop
-	l.mu.Unlock()
-	go l.renew(renewCtx)
-
-	slog.InfoContext(ctx, "kvlease: elected", l.logArgs()...)
-	if l.cfg.OnGained != nil {
-		if err := l.cfg.OnGained(ctx); err != nil {
-			// A leader that cannot set up its work is worse than no leader. Stand down so a healthy node wins.
-			slog.ErrorContext(ctx, "kvlease: OnGained failed, standing down", l.logArgs("err", err)...)
-			l.Release(ctx)
-			return false
-		}
-	}
-	return true
-}
-
-// claim creates the key or adopts one this holder left behind. A restarted
-// process finds its own key still present and must refresh it rather than
-// sit out the TTL with no leader elected.
-func (l *Lease) claim(ctx context.Context, kv jetstream.KeyValue) (uint64, error) {
-	rev, err := kv.Create(ctx, l.cfg.Key, []byte(l.cfg.Holder))
-	if err == nil {
-		return rev, nil
-	}
-	entry, gerr := kv.Get(ctx, l.cfg.Key)
-	if gerr != nil {
-		return 0, gerr
-	}
-	if string(entry.Value()) != l.cfg.Holder {
-		return 0, &heldError{holder: string(entry.Value())}
-	}
-	return kv.Update(ctx, l.cfg.Key, []byte(l.cfg.Holder), entry.Revision())
-}
-
-// renew refreshes the key at the revision last observed, until ctx is cancelled or a refresh
-// loses the CAS - which means the key expired and another node claimed it.
-func (l *Lease) renew(ctx context.Context) {
-	ticker := time.NewTicker(l.cfg.Renew)
-	defer ticker.Stop()
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-ticker.C:
-			l.mu.Lock()
-			kv, rev, held := l.kv, l.rev, l.held
-			l.mu.Unlock()
-			if !held {
-				return
-			}
-			next, err := kv.Update(ctx, l.cfg.Key, []byte(l.cfg.Holder), rev)
-			if err != nil {
-				slog.WarnContext(ctx, "kvlease: renewal lost the key, another node may take over mid-pass", l.logArgs("err", err)...)
-				l.markLost()
-				return
-			}
-			l.mu.Lock()
-			l.rev = next
-			l.mu.Unlock()
-		}
-	}
-}
-
-// markLost clears held state and fires OnLost. It reports whether this call was the transition,
-// so renewal and release never double-fire the edge.
-func (l *Lease) markLost() bool {
-	l.mu.Lock()
-	was := l.held
-	l.held = false
-	if was {
-		close(l.lost)
-	}
-	l.mu.Unlock()
-	if !was {
-		return false
-	}
-	if l.cfg.OnLost != nil {
-		l.cfg.OnLost()
-	}
-	return true
-}
-
-// Release deletes the key if this node still owns it at the revision it last observed.
-// It runs on a detached context: shutdown cancels ctx first, and skipping the delete would park
-// the lease for the full TTL.
-func (l *Lease) Release(ctx context.Context) {
-	l.mu.Lock()
-	kv, rev, stop := l.kv, l.rev, l.stop
-	l.kv, l.stop = nil, nil
-	l.mu.Unlock()
-	if stop != nil {
-		stop()
-	}
-	if kv == nil {
-		// Never acquired, or already released.
-		return
-	}
-	if !l.markLost() {
-		// Renewal already lost the key, so another node may hold it now. An unguarded delete here
-		// would drop that node's lease, not ours.
-		slog.WarnContext(ctx, "kvlease: already lost before release, not deleting", l.logArgs()...)
-		return
-	}
-	releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), releaseTimeout)
-	defer cancel()
-	if err := kv.Delete(releaseCtx, l.cfg.Key, jetstream.LastRevision(rev)); err != nil {
-		slog.WarnContext(releaseCtx, "kvlease: release failed, TTL will reap", l.logArgs("err", err)...)
-	}
-}
-
-// Run holds the lease across many passes, re-attempting on a ticker while this node is not the
-// holder. A holder does not re-attempt: renewal keeps the key.
-func (l *Lease) Run(ctx context.Context) {
-	ticker := time.NewTicker(l.cfg.Retry)
-	defer ticker.Stop()
-	l.TryAcquire(ctx)
-	for {
-		select {
-		case <-ctx.Done():
-			l.Release(ctx)
-			return
-		case <-ticker.C:
-			l.TryAcquire(ctx)
-		}
-	}
-}
-
-// Lost returns a channel closed when this lease stops being held, so work that
-// must not outlive the lease can select on it rather than poll Held. The channel
-// is per-acquisition: re-acquiring installs a fresh one.
-func (l *Lease) Lost() <-chan struct{} {
-	l.mu.Lock()
-	defer l.mu.Unlock()
-	return l.lost
-}

spinifex/foundation/state/kvlease/lease_test.go

@@ -1,437 +0,0 @@
-package kvlease_test
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"errors"
-	"log/slog"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
-	"github.com/nats-io/nats.go"
-	"github.com/nats-io/nats.go/jetstream"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-const (
-	testBucket = "kvlease-test"
-	testKey    = "leader"
-)
-
-// newTestLease builds a lease over a short-lived bucket so a test can outlast
-// the TTL in a second rather than a minute. Bucket is built after mut so a test
-// that lengthens the TTL gets a bucket that matches it.
-func newTestLease(t *testing.T, nc *nats.Conn, holder string, mut func(*kvlease.Config)) *kvlease.Lease {
-	t.Helper()
-	cfg := kvlease.Config{
-		Name:   "test",
-		Key:    testKey,
-		Holder: holder,
-		TTL:    time.Second,
-		Renew:  250 * time.Millisecond,
-		Retry:  100 * time.Millisecond,
-	}
-	if mut != nil {
-		mut(&cfg)
-	}
-	cfg.Bucket = kvlease.NATSBucket(nc, testBucket, cfg.TTL)
-	l, err := kvlease.New(cfg)
-	require.NoError(t, err)
-	return l
-}
-
-func testKV(t *testing.T, nc *nats.Conn, ttl time.Duration) jetstream.KeyValue {
-	t.Helper()
-	kv, err := kvlease.NATSBucket(nc, testBucket, ttl)(t.Context())
-	require.NoError(t, err)
-	return kv
-}
-
-// isOpen reports whether a Lost channel has not yet fired, without blocking.
-func isOpen(ch <-chan struct{}) bool {
-	select {
-	case <-ch:
-		return false
-	default:
-		return true
-	}
-}
-
-func TestTryAcquire_SecondHolderLoses(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	a := newTestLease(t, nc, "node-a", nil)
-	b := newTestLease(t, nc, "node-b", nil)
-
-	require.True(t, a.TryAcquire(t.Context()), "first acquire must win and create the bucket")
-	assert.False(t, b.TryAcquire(t.Context()), "second acquire must lose while the lease is held")
-
-	a.Release(t.Context())
-	assert.True(t, b.TryAcquire(t.Context()), "acquire after release must win")
-	b.Release(t.Context())
-}
-
-// A restarted process finds its own key still present. Refusing it would leave the subsystem leaderless for the rest of
-// the TTL for no reason.
-func TestTryAcquire_AdoptsOwnKeyAfterRestart(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	kv := testKV(t, nc, time.Second)
-
-	rev, err := kv.Create(t.Context(), testKey, []byte("node-a"))
-	require.NoError(t, err)
-
-	restarted := newTestLease(t, nc, "node-a", nil)
-	require.True(t, restarted.TryAcquire(t.Context()), "a holder must adopt the key it left behind")
-
-	entry, err := kv.Get(t.Context(), testKey)
-	require.NoError(t, err)
-	assert.Greater(t, entry.Revision(), rev, "adopting must refresh the key, resetting its TTL")
-	restarted.Release(t.Context())
-}
-
-// A pass can outlive the TTL on its own, so the key must be renewed while the pass runs. Without this a peer elects
-// itself mid-pass and both act at once.
-func TestRenew_KeySurvivesBeyondTTL(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	// Three renewals per TTL, and above the 100ms floor NATS puts on a bucket TTL.
-	short := func(c *kvlease.Config) { c.TTL, c.Renew = 300*time.Millisecond, 100*time.Millisecond }
-	a := newTestLease(t, nc, "node-a", short)
-	require.True(t, a.TryAcquire(t.Context()))
-	defer a.Release(t.Context())
-
-	// Well past the TTL: an unrenewed key would have expired by now.
-	time.Sleep(750 * time.Millisecond)
-
-	b := newTestLease(t, nc, "node-b", short)
-	assert.False(t, b.TryAcquire(t.Context()), "lease expired mid-pass: a peer can now act concurrently")
-	assert.True(t, a.Held())
-}
-
-// If the key was lost anyway, the stale holder's release must not delete the successor's key -
-// that hand the lease to a third node silently.
-func TestRelease_DoesNotDeleteSuccessorKey(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	a := newTestLease(t, nc, "node-a", func(c *kvlease.Config) {
-		c.TTL, c.Renew = time.Minute, 20*time.Second // no renewal during the test
-	})
-	require.True(t, a.TryAcquire(t.Context()))
-
-	kv := testKV(t, nc, time.Minute)
-	_, err := kv.Put(t.Context(), testKey, []byte("node-b"))
-	require.NoError(t, err)
-
-	a.Release(t.Context())
-
-	entry, err := kv.Get(t.Context(), testKey)
-	require.NoError(t, err, "stale release deleted the successor's key")
-	assert.Equal(t, "node-b", string(entry.Value()))
-}
-
-// Shutdown cancels the acquiring context first. A release bound to it would no-op and park the lease for the full TTL.
-func TestRelease_SurvivesCancelledContext(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	ctx, cancel := context.WithCancel(t.Context())
-	a := newTestLease(t, nc, "node-a", nil)
-	require.True(t, a.TryAcquire(ctx))
-
-	cancel()
-	a.Release(ctx)
-
-	b := newTestLease(t, nc, "node-b", nil)
-	assert.True(t, b.TryAcquire(t.Context()), "lease must be free immediately, not at the TTL")
-	b.Release(t.Context())
-}
-
-func TestEdges_FireOncePerTransition(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	var gained, lost atomic.Int32
-	a := newTestLease(t, nc, "node-a", func(c *kvlease.Config) {
-		c.OnGained = func(context.Context) error { gained.Add(1); return nil }
-		c.OnLost = func() { lost.Add(1) }
-	})
-
-	require.True(t, a.TryAcquire(t.Context()))
-	require.True(t, a.TryAcquire(t.Context()), "a holder re-attempting must stay held")
-	assert.EqualValues(t, 1, gained.Load(), "OnGained fired twice for one election")
-	assert.EqualValues(t, 0, lost.Load())
-
-	a.Release(t.Context())
-	assert.EqualValues(t, 1, lost.Load())
-	a.Release(t.Context())
-	assert.EqualValues(t, 1, lost.Load(), "a second release must not re-fire OnLost")
-}
-
-func TestRenew_LossFiresOnLost(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	lost := make(chan struct{}, 4)
-	a := newTestLease(t, nc, "node-a", func(c *kvlease.Config) {
-		c.OnLost = func() { lost <- struct{}{} }
-	})
-	require.True(t, a.TryAcquire(t.Context()))
-
-	// Stand in for the key expiring and a peer claiming it.
-	kv := testKV(t, nc, time.Second)
-	require.NoError(t, kv.Delete(t.Context(), testKey))
-	_, err := kv.Create(t.Context(), testKey, []byte("node-b"))
-	require.NoError(t, err)
-
-	select {
-	case <-lost:
-	case <-time.After(3 * time.Second):
-		t.Fatal("renewal lost the CAS but OnLost never fired")
-	}
-	assert.False(t, a.Held())
-}
-
-// A leader that cannot set up its own work is worse than no leader.
-func TestTryAcquire_OnGainedFailureStandsDown(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	var lost atomic.Int32
-	a := newTestLease(t, nc, "node-a", func(c *kvlease.Config) {
-		c.OnGained = func(context.Context) error { return errors.New("subscribe failed") }
-		c.OnLost = func() { lost.Add(1) }
-	})
-
-	assert.False(t, a.TryAcquire(t.Context()), "a holder that cannot set up its work must not report leadership")
-	assert.False(t, a.Held())
-	assert.EqualValues(t, 1, lost.Load(), "OnLost must fire so the caller tears down partial statea")
-
-	b := newTestLease(t, nc, "node-b", nil)
-	assert.True(t, b.TryAcquire(t.Context()), "standing down must free the key immediately")
-	b.Release(t.Context())
-}
-
-func TestNew_RejectUnworkableConfig(t *testing.T) {
-	base := func() kvlease.Config {
-		return kvlease.Config{
-			Bucket: func(context.Context) (jetstream.KeyValue, error) { return nil, nil },
-			Key:    testKey,
-			Holder: "node-a",
-			TTL:    time.Minute,
-			Renew:  20 * time.Second,
-		}
-	}
-	_, err := kvlease.New(base())
-	require.NoError(t, err)
-
-	cases := map[string]func(*kvlease.Config){
-		"no bucket":           func(c *kvlease.Config) { c.Bucket = nil },
-		"no key":              func(c *kvlease.Config) { c.Key = "" },
-		"no holder":           func(c *kvlease.Config) { c.Holder = "" },
-		"no ttl":              func(c *kvlease.Config) { c.TTL = 0 },
-		"renew equals ttl":    func(c *kvlease.Config) { c.Renew = c.TTL },
-		"renew over half ttl": func(c *kvlease.Config) { c.Renew = 31 * time.Second },
-	}
-	for name, mut := range cases {
-		t.Run(name, func(t *testing.T) {
-			cfg := base()
-			mut(&cfg)
-			_, err := kvlease.New(cfg)
-			assert.Error(t, err)
-		})
-	}
-}
-
-// Session mode: a loser must take over on its retry tick rather than wait out the TTL, which is what makes a rolling restart a sub-second gap.
-func TestRun_LoserTakesOverAfterLeaderStops(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	aCtx, stopA := context.WithCancel(t.Context())
-	a := newTestLease(t, nc, "node-a", nil)
-	go a.Run(aCtx)
-	require.Eventually(t, a.Held, 2*time.Second, 20*time.Millisecond)
-
-	b := newTestLease(t, nc, "node-b", nil)
-	go b.Run(t.Context())
-	time.Sleep(300 * time.Millisecond)
-	require.False(t, b.Held(), "two holders at once")
-
-	stopA()
-	require.Eventually(t, b.Held, 2*time.Second, 20*time.Millisecond, "loser must take over on its retry tick, not at the TTL")
-}
-
-// Work gated on Lose must run while the lease is held. A channel closed at construction or on acquire would abort every pass before it started.
-func TestLost_OpenWhileHeld(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	a := newTestLease(t, nc, "node-a", nil)
-
-	assert.True(t, isOpen(a.Lost()), "an unclaimed lease has lost nothing")
-
-	require.True(t, a.TryAcquire(t.Context()))
-	assert.True(t, isOpen(a.Lost()))
-
-	// Well past two renewal intervals: a renewing holder must stay held.
-	time.Sleep(600 * time.Millisecond)
-	assert.True(t, isOpen(a.Lost()), "renewal succeeded but lost fired anyway")
-
-	a.Release(t.Context())
-}
-
-// The signal callers select on to abandon work. Polling Held cannot express "stop what you are doing now", which is what a lost lease requires.
-func TestLost_ClosesOnRenewalLoss(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	a := newTestLease(t, nc, "node-a", nil)
-	require.True(t, a.TryAcquire(t.Context()))
-	lost := a.Lost()
-
-	// Stand in for the key expiring and a peer claiming it.
-	kv := testKV(t, nc, time.Second)
-	require.NoError(t, kv.Delete(t.Context(), testKey))
-	_, err := kv.Create(t.Context(), testKey, []byte("node-b"))
-	require.NoError(t, err)
-
-	select {
-	case <-lost:
-	case <-time.After(3 * time.Second):
-		t.Fatal("renewal lost the CAS but Lost never closed")
-	}
-	assert.False(t, a.Held())
-}
-
-func TestLost_ClosesOnRelease(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	a := newTestLease(t, nc, "node-a", nil)
-	require.True(t, a.TryAcquire(t.Context()))
-	lost := a.Lost()
-
-	a.Release(t.Context())
-	assert.False(t, isOpen(lost), "release must close Lost so gated work stops")
-
-	// A second release must not close an already-closed channel.
-	assert.NotPanics(t, func() { a.Release(t.Context()) })
-}
-
-// The channel is per-acquisition. Handing a re-elected holder the closed channel from its previous term would abort its new pass immediately.
-func TestLost_FreshChannelAfterReacquire(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	a := newTestLease(t, nc, "node-a", nil)
-
-	require.True(t, a.TryAcquire(t.Context()))
-	first := a.Lost()
-	a.Release(t.Context())
-	require.False(t, isOpen(first))
-
-	require.True(t, a.TryAcquire(t.Context()))
-	second := a.Lost()
-	assert.True(t, isOpen(second), "a re-elected holder got its old term's closed channel")
-	assert.NotEqual(t, first, second, "Lost must be replaced on acquire, not reused")
-
-	a.Release(t.Context())
-}
-
-func TestAttrs_AppearOnLeaseOwnedLogLines(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-
-	var buf bytes.Buffer
-	prev := slog.Default()
-	slog.SetDefault(slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo})))
-	t.Cleanup(func() { slog.SetDefault(prev) })
-
-	lease := newTestLease(t, nc, "node-a", func(cfg *kvlease.Config) {
-		cfg.Attrs = []any{"bucket", "quota-reconcile"}
-	})
-	require.True(t, lease.TryAcquire(t.Context()))
-	lease.Release(t.Context())
-
-	require.Contains(t, buf.String(), `"bucket":"quota-reconcile"`)
-}
-
-// A node that never won the lease still unwinds its Run loop through Release.
-// That is not a lost key, so it must not raise the warning that says one.
-func TestRelease_WithoutAcquireIsSilent(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-
-	holder := newTestLease(t, nc, "node-a", nil)
-	require.True(t, holder.TryAcquire(t.Context()))
-	t.Cleanup(func() { holder.Release(t.Context()) })
-
-	var buf bytes.Buffer
-	prev := slog.Default()
-	slog.SetDefault(slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo})))
-	t.Cleanup(func() { slog.SetDefault(prev) })
-
-	loser := newTestLease(t, nc, "node-b", nil)
-	require.False(t, loser.TryAcquire(t.Context()))
-	loser.Release(t.Context())
-
-	assert.NotContains(t, buf.String(), "already lost before release")
-}
-
-// The warning still has to fire where it means something: renewal lost the key,
-// so a delete here would drop the successor's lease rather than.
-func TestRelease_AfterRenewalLossWarns(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	lost := make(chan struct{}, 4)
-	a := newTestLease(t, nc, "node-a", func(c *kvlease.Config) {
-		c.OnLost = func() { lost <- struct{}{} }
-	})
-	require.True(t, a.TryAcquire(t.Context()))
-
-	kv := testKV(t, nc, time.Second)
-	require.NoError(t, kv.Delete(t.Context(), testKey))
-	_, err := kv.Create(t.Context(), testKey, []byte("node-b"))
-	require.NoError(t, err)
-
-	select {
-	case <-lost:
-	case <-time.After(3 * time.Second):
-		t.Fatal("renewal lost the CAS but OnLost never fired")
-	}
-
-	var buf bytes.Buffer
-	prev := slog.Default()
-	slog.SetDefault(slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo})))
-	t.Cleanup(func() { slog.SetDefault(prev) })
-
-	a.Release(t.Context())
-	assert.Contains(t, buf.String(), "already lost before release")
-}
-
-// logLines captures default-logger output as decoded JSON records.
-func logLines(t *testing.T) func() []map[string]any {
-	t.Helper()
-	var buf bytes.Buffer
-	prev := slog.Default()
-	slog.SetDefault(slog.New(slog.NewJSONHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})))
-	t.Cleanup(func() { slog.SetDefault(prev) })
-	return func() []map[string]any {
-		var out []map[string]any
-		dec := json.NewDecoder(&buf)
-		for dec.More() {
-			var rec map[string]any
-			require.NoError(t, dec.Decode(&rec))
-			out = append(out, rec)
-		}
-		return out
-	}
-}
-
-// "holder" must name whoever owns the key. The local node logs as "node", so a
-// loser's log does not read as though it won.
-func TestLogFields_SeparateLocalNodeFromCurrentHolder(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	read := logLines(t)
-
-	a := newTestLease(t, nc, "node-a", nil)
-	require.True(t, a.TryAcquire(t.Context()))
-	t.Cleanup(func() { a.Release(t.Context()) })
-	b := newTestLease(t, nc, "node-b", nil)
-	require.False(t, b.TryAcquire(t.Context()))
-
-	var elected, refused map[string]any
-	for _, rec := range read() {
-		switch {
-		case rec["msg"] == "kvlease: elected" && rec["node"] == "node-a":
-			elected = rec
-		case rec["node"] == "node-b":
-			refused = rec
-		}
-	}
-	require.NotNil(t, elected, "the elected line must carry the local node as node")
-	assert.NotContains(t, elected, "holder")
-	require.NotNil(t, refused, "a refused acquire must log the local node as node")
-	assert.Equal(t, "node-a", refused["holder"], "a refused acquire must report the node that holds the key")
-}

spinifex/foundation/state/kvstore/bucket.go

@@ -53,8 +53,14 @@ type Bucket struct {
 	js  jetstream.JetStream
 	cfg Config
 
+	// mu guards the fields below and is never held across a network call.
 	mu sync.Mutex
 	kv jetstream.KeyValue
+	// opened is set once a handle has been held, after which a missing bucket
+	// is a lost one, and only RecreateIfMissing may create it again.
+	opened bool
+	// resolving is closed when the one open or reopen in flight finishes.
+	resolving chan struct{}
 }
 
 // NewBucket returns a Bucket over js. A nil js is permitted: every call to KV then
@@ -68,7 +74,7 @@ func NewBucket(js jetstream.JetStream, cfg Config) *Bucket {
 // rather than on first use. js is what recovery reopens against; pass nil to
 // keep the handle fixed for a caller not ready to recover.
 func NewOpenBucket(js jetstream.JetStream, kv jetstream.KeyValue, cfg Config) *Bucket {
-	return &Bucket{js: js, kv: kv, cfg: cfg}
+	return &Bucket{js: js, kv: kv, cfg: cfg, opened: kv != nil}
 }
 
 // Name returns the configured bucket name, for callers holding several buckets
@@ -84,27 +90,9 @@ func (b *Bucket) Configured() bool {
 }
 
 // KV opens the bucket on first use and returns the cached handle thereafter.
+// Concurrent callers share one open, and each stops waiting when its ctx ends.
 func (b *Bucket) KV(ctx context.Context) (jetstream.KeyValue, error) {
-	b.mu.Lock()
+	return b.resolve(ctx, false)
-	defer b.mu.Unlock()
-	if b.kv != nil {
-		return b.kv, nil
-	}
-	if b.js == nil {
-		if b.cfg.Missing == "" {
-			return nil, errors.New("kvstore: no JetStream client configured")
-		}
-		return nil, errors.New(b.cfg.Missing)
-	}
-	kv, err := b.open(ctx)
-	if err != nil {
-		return nil, err
-	}
-	if err := b.onOpen(ctx, kv); err != nil {
-		return nil, err
-	}
-	b.kv = kv
-	return kv, nil
 }
 
 // LastSequence reports the stream sequence of the newest message matching
@@ -140,41 +128,115 @@ func (b *Bucket) LastSequence(ctx context.Context, filter string) (uint64, error
 // caller that saw the stream go out from under it. It reconnects; whether it
 // may also recreate an absent bucket is Config.RecreateIfMissing.
 func (b *Bucket) Reopen(ctx context.Context) (jetstream.KeyValue, error) {
-	b.mu.Lock()
+	return b.resolve(ctx, true)
-	defer b.mu.Unlock()
+}
 
-	// Checked before the handle is discarded. A Bucket over a fixed handle has
+// resolve returns the cached handle, or runs the one open in flight and
-	// nothing to resolve against, so dropping it would turn one transient
+// caches its result. A waiter for a reopen takes whatever handle the flight
-	// outage into a permanent failure for every later call.
+// left, since another goroutine resolving it is the repair this caller wanted.
-	if b.js == nil {
+func (b *Bucket) resolve(ctx context.Context, reopen bool) (jetstream.KeyValue, error) {
-		if b.cfg.Missing == "" {
+	for {
-			return nil, errors.New("kvstore: cannot reopen without a JetStream client")
+		b.mu.Lock()
+		if !reopen && b.kv != nil {
+			kv := b.kv
+			b.mu.Unlock()
+			return kv, nil
 		}
-		return nil, errors.New(b.cfg.Missing)
+		// Checked before the handle is discarded. A Bucket over a fixed handle
+		// has nothing to resolve against, so dropping it would turn one
+		// transient outage into a permanent failure for every later call.
+		if b.js == nil {
+			b.mu.Unlock()
+			return nil, b.missingErr(reopen)
+		}
+		if wait := b.resolving; wait != nil {
+			b.mu.Unlock()
+			select {
+			case <-wait:
+			case <-ctx.Done():
+				return nil, fmt.Errorf("kvstore: open %s: %w", b.cfg.Name, ctx.Err())
+			}
+			if reopen {
+				b.mu.Lock()
+				kv := b.kv
+				b.mu.Unlock()
+				if kv != nil {
+					return kv, nil
+				}
+			}
+			continue
+		}
+		done := make(chan struct{})
+		b.resolving = done
+		if reopen {
+			b.kv = nil
+		}
+		opened := b.opened
+		b.mu.Unlock()
+
+		kv, err := b.connect(ctx, reopen, opened)
+
+		b.mu.Lock()
+		if err == nil {
+			b.kv, b.opened = kv, true
+		}
+		b.resolving = nil
+		close(done)
+		b.mu.Unlock()
+		return kv, err
 	}
-	b.kv = nil
+}
 
-	// Another goroutine may already have repaired it, so reconnecting comes
+func (b *Bucket) missingErr(reopen bool) error {
-	// first and is the only step that is always safe.
+	switch {
-	kv, err := b.js.KeyValue(ctx, b.cfg.Name)
+	case b.cfg.Missing != "":
-	if err != nil {
+		return errors.New(b.cfg.Missing)
-		if !b.cfg.RecreateIfMissing {
+	case reopen:
+		return errors.New("kvstore: cannot reopen without a JetStream client")
+	default:
+		return errors.New("kvstore: no JetStream client configured")
+	}
+}
+
+// connect does the network half of resolve, with no lock held. A bucket that
+// was open before is only attached to unless RecreateIfMissing allows more.
+func (b *Bucket) connect(ctx context.Context, reopen, opened bool) (jetstream.KeyValue, error) {
+	var (
+		kv  jetstream.KeyValue
+		err error
+	)
+	switch {
+	case reopen:
+		// Another goroutine may already have repaired it, so reconnecting comes
+		// first and is the only step that is always safe.
+		kv, err = b.js.KeyValue(ctx, b.cfg.Name)
+		if err != nil {
+			if !b.cfg.RecreateIfMissing {
+				return nil, fmt.Errorf("kvstore: reopen %s: %w", b.cfg.Name, err)
+			}
+			if !errors.Is(err, jetstream.ErrBucketNotFound) && !IsStreamUnavailable(err) {
+				return nil, fmt.Errorf("kvstore: reopen %s: %w", b.cfg.Name, err)
+			}
+			slog.WarnContext(ctx, "kvstore: bucket stream lost, recreating", "bucket", b.cfg.Name)
+			if kv, err = b.open(ctx); err != nil {
+				return nil, fmt.Errorf("kvstore: recreate %s: %w", b.cfg.Name, err)
+			}
+		}
+	case opened && !b.cfg.RecreateIfMissing:
+		if kv, err = b.js.KeyValue(ctx, b.cfg.Name); err != nil {
 			return nil, fmt.Errorf("kvstore: reopen %s: %w", b.cfg.Name, err)
 		}
-		if !errors.Is(err, jetstream.ErrBucketNotFound) && !IsStreamUnavailable(err) {
+	default:
-			return nil, fmt.Errorf("kvstore: reopen %s: %w", b.cfg.Name, err)
-		}
-		slog.WarnContext(ctx, "kvstore: bucket stream lost, recreating", "bucket", b.cfg.Name)
 		if kv, err = b.open(ctx); err != nil {
-			return nil, fmt.Errorf("kvstore: recreate %s: %w", b.cfg.Name, err)
+			return nil, err
 		}
 	}
-
 	if err := b.onOpen(ctx, kv); err != nil {
 		return nil, err
 	}
-	b.kv = kv
+	if reopen {
-	slog.InfoContext(ctx, "kvstore: bucket reopened", "bucket", b.cfg.Name)
+		slog.InfoContext(ctx, "kvstore: bucket reopened", "bucket", b.cfg.Name)
+	}
 	return kv, nil
 }
 
@@ -219,8 +281,8 @@ func (b *Bucket) Delete(ctx context.Context, key string) error {
 	return nil
 }
 
-// CompareAndDelete removes a key only if it is still at rev, returning
+// CompareAndDelete removes a key only if it is still at rev, returning an
-// ErrConflict when it is not.
+// error that is both ErrConflict and kvsync.ErrRevisionMismatch when it is not.
 //
 // Unlike Delete this is not idempotent: a guarded delete carries an expected
 // last sequence, and an absent, tombstoned or purged key fails that check, so
@@ -232,15 +294,12 @@ func (b *Bucket) Delete(ctx context.Context, key string) error {
 func (b *Bucket) CompareAndDelete(ctx context.Context, key string, rev uint64) error {
 	kv, err := b.KV(ctx)
 	if err != nil {
-		return err
+		return syncError("delete", key, err)
 	}
 	// Not through withKV: the revision is the caller's, so a replay against a
 	// reopened bucket would be guarded on a revision from the bucket it lost.
 	if err := kv.Delete(ctx, key, jetstream.LastRevision(rev)); err != nil {
-		if errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
+		return syncError("delete", key, err)
-			return fmt.Errorf("%w: %s", ErrConflict, key)
-		}
-		return fmt.Errorf("kvstore: delete %s: %w", key, err)
 	}
 	return nil
 }

spinifex/foundation/state/kvstore/bucket_creation_invariants_test.go

@@ -15,11 +15,9 @@ import (
 var createFuncs = []string{"CreateKeyValue(", "CreateOrUpdateKeyValue("}
 
 // exempt are the only files and packages allowed to call them: the chokepoint
-// itself, the lease-bucket opener it shares its rule with, and the test helpers
+// itself and the test helpers that build buckets for a single embedded server.
-// that build buckets for a single embedded server.
 var exempt = []string{
 	filepath.Join("spinifex", "foundation", "state", "kvstore", "buckets.go"),
-	filepath.Join("spinifex", "foundation", "state", "kvlease"),
 	filepath.Join("internal", "testkit"),
 }
 

spinifex/foundation/state/kvstore/recovery_test.go

@@ -133,6 +133,10 @@ func TestStore_RecreateIfMissingDefaultsOff(t *testing.T) {
 
 	err := store.Set(t.Context(), "acct-a/one", &record{Name: "two"})
 	require.Error(t, err, "recovery must not silently recreate a bucket the caller did not opt into")
+	// The failed recovery dropped the handle, so this one reaches the first-use
+	// open, which must not create either.
+	err = store.Set(t.Context(), "acct-a/one", &record{Name: "three"})
+	require.Error(t, err, "a later operation must not recreate it through the first-use open")
 
 	_, err = js.KeyValue(t.Context(), recoveryBucket)
 	require.Error(t, err, "the bucket must still be absent")

spinifex/foundation/state/kvstore/sync.go

@@ -0,0 +1,110 @@
+package kvstore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+var _ kvsync.KVStore = (*Bucket)(nil)
+
+// syncBucket holds every kvsync lock key, namespaced by the key itself.
+const (
+	syncBucket        = "spinifex-sync"
+	syncBucketVersion = 1
+)
+
+// OpenSyncBucket returns the Bucket every kvsync lock in a process is built on:
+// file storage, one revision of history and no TTL, since a lease's expiry is
+// observed by its contenders rather than kept by the store. It opens on first use.
+//
+// RecreateIfMissing stays false: a recreated stream restarts its revisions, and
+// a generation must only grow.
+func OpenSyncBucket(js jetstream.JetStream) *Bucket {
+	return NewBucket(js, Config{
+		Name:        syncBucket,
+		History:     1,
+		Description: "kvsync claims and leases",
+		Missing:     "kvstore: sync bucket has no JetStream client",
+		OnOpen: func(ctx context.Context, kv jetstream.KeyValue) error {
+			return WriteVersion(ctx, kv, syncBucketVersion)
+		},
+	})
+}
+
+// Load returns the value and revision of key, with an error that is both
+// ErrNotFound and kvsync.ErrNotFound when the key is absent.
+func (b *Bucket) Load(ctx context.Context, key string) ([]byte, uint64, error) {
+	var (
+		value []byte
+		rev   uint64
+	)
+	err := b.withKV(ctx, func(kv jetstream.KeyValue) error {
+		entry, err := kv.Get(ctx, key)
+		if err != nil {
+			return err
+		}
+		value, rev = entry.Value(), entry.Revision()
+		return nil
+	})
+	if err != nil {
+		return nil, 0, syncError("load", key, err)
+	}
+	return value, rev, nil
+}
+
+// CompareAndSwap writes value only while key is at rev, where rev 0 means the
+// key must be absent or deleted, and returns the new revision. A lost race is
+// both ErrConflict and kvsync.ErrRevisionMismatch.
+func (b *Bucket) CompareAndSwap(ctx context.Context, key string, value []byte, rev uint64) (uint64, error) {
+	kv, err := b.KV(ctx)
+	if err != nil {
+		return 0, syncError("compare-and-swap", key, err)
+	}
+	// Not through withKV: the revision is the caller's, as in CompareAndDelete.
+	var next uint64
+	if rev == 0 {
+		next, err = kv.Create(ctx, key, value)
+	} else {
+		next, err = kv.Update(ctx, key, value, rev)
+	}
+	if err != nil {
+		return 0, syncError("compare-and-swap", key, err)
+	}
+	return next, nil
+}
+
+// syncError classifies a JetStream failure into both packages' sentinels.
+func syncError(op, key string, err error) error {
+	switch {
+	case errors.Is(err, jetstream.ErrKeyNotFound):
+		return fmt.Errorf("%w: %w: %s", ErrNotFound, kvsync.ErrNotFound, key)
+	case IsRevisionConflict(err):
+		return fmt.Errorf("%w: %w: %s", ErrConflict, kvsync.ErrRevisionMismatch, key)
+	case isUnreachable(err):
+		return fmt.Errorf("kvstore: %s %s: %w: %w", op, key, kvsync.ErrUnavailable, err)
+	default:
+		return fmt.Errorf("kvstore: %s %s: %w", op, key, err)
+	}
+}
+
+// isUnreachable is IsStreamUnavailable widened to a connection that is down, a
+// request that went unanswered, or a server answering 503 (10008 while JetStream
+// has no meta leader), since a lock caller retries all alike.
+func isUnreachable(err error) bool {
+	if apiErr, ok := errors.AsType[*jetstream.APIError](err); ok && apiErr.Code == http.StatusServiceUnavailable {
+		return true
+	}
+	return IsStreamUnavailable(err) ||
+		errors.Is(err, context.DeadlineExceeded) ||
+		errors.Is(err, nats.ErrTimeout) ||
+		errors.Is(err, nats.ErrConnectionClosed) ||
+		errors.Is(err, nats.ErrConnectionReconnecting) ||
+		errors.Is(err, nats.ErrNoServers) ||
+		errors.Is(err, nats.ErrDisconnected)
+}

spinifex/foundation/state/kvstore/sync_test.go

@@ -0,0 +1,217 @@
+package kvstore_test
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSyncBucket_OpensWithOneRevisionNoTTLAndASchemaVersion(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	b := kvstore.OpenSyncBucket(js)
+	kv, err := b.KV(t.Context())
+	require.NoError(t, err)
+
+	status, err := kv.Status(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, "spinifex-sync", status.Bucket())
+	assert.Equal(t, int64(1), status.History())
+	assert.Zero(t, status.TTL(), "a lease's expiry is observed by contenders, never kept by the store")
+	info := status.(*jetstream.KeyValueBucketStatus).StreamInfo()
+	assert.Equal(t, jetstream.FileStorage, info.Config.Storage)
+
+	version, err := kvstore.ReadVersion(t.Context(), kv)
+	require.NoError(t, err)
+	assert.Equal(t, 1, version)
+}
+
+func TestSyncBucket_ErrorsCarryBothPackagesSentinels(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	b := kvstore.OpenSyncBucket(js)
+
+	_, _, err := b.Load(t.Context(), "k")
+	assert.ErrorIs(t, err, kvsync.ErrNotFound)
+	assert.ErrorIs(t, err, kvstore.ErrNotFound)
+
+	rev, err := b.CompareAndSwap(t.Context(), "k", []byte("a"), 0)
+	require.NoError(t, err)
+
+	mismatches := map[string]func() error{
+		"create over a live key": func() error {
+			_, err := b.CompareAndSwap(t.Context(), "k", []byte("b"), 0)
+			return err
+		},
+		"update at a stale revision": func() error {
+			_, err := b.CompareAndSwap(t.Context(), "k", []byte("b"), rev+100)
+			return err
+		},
+		"delete at a stale revision": func() error {
+			return b.CompareAndDelete(t.Context(), "k", rev+100)
+		},
+	}
+	for name, op := range mismatches {
+		t.Run(name, func(t *testing.T) {
+			err := op()
+			assert.ErrorIs(t, err, kvsync.ErrRevisionMismatch)
+			assert.ErrorIs(t, err, kvstore.ErrConflict)
+		})
+	}
+
+	value, got, err := b.Load(t.Context(), "k")
+	require.NoError(t, err)
+	assert.Equal(t, rev, got)
+	assert.Equal(t, []byte("a"), value)
+}
+
+func TestSyncBucket_RevisionGrowsAcrossADelete(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	b := kvstore.OpenSyncBucket(js)
+
+	first, err := b.CompareAndSwap(t.Context(), "k", []byte("a"), 0)
+	require.NoError(t, err)
+	require.NoError(t, b.CompareAndDelete(t.Context(), "k", first))
+
+	// Rev 0 creates over the delete marker, at a revision above the first.
+	second, err := b.CompareAndSwap(t.Context(), "k", []byte("b"), 0)
+	require.NoError(t, err)
+	assert.Greater(t, second, first)
+
+	third, err := b.CompareAndSwap(t.Context(), "k", []byte("c"), second)
+	require.NoError(t, err)
+	assert.Greater(t, third, second)
+}
+
+func TestSyncBucket_StoppedJetStreamIsUnavailable(t *testing.T) {
+	stops := map[string]func(t *testing.T, ns *server.Server){
+		"jetstream disabled": func(t *testing.T, ns *server.Server) { require.NoError(t, ns.DisableJetStream()) },
+		"server shut down":   func(_ *testing.T, ns *server.Server) { ns.Shutdown(); ns.WaitForShutdown() },
+	}
+	for name, stop := range stops {
+		t.Run(name, func(t *testing.T) {
+			ns, _, js := testutil.StartTestJetStream(t)
+			b := kvstore.OpenSyncBucket(js)
+			rev, err := b.CompareAndSwap(t.Context(), "k", []byte("a"), 0)
+			require.NoError(t, err)
+			stop(t, ns)
+
+			ops := map[string]func(context.Context) error{
+				"load": func(ctx context.Context) error { _, _, err := b.Load(ctx, "k"); return err },
+				"swap": func(ctx context.Context) error {
+					_, err := b.CompareAndSwap(ctx, "k", []byte("b"), rev)
+					return err
+				},
+				"delete": func(ctx context.Context) error { return b.CompareAndDelete(ctx, "k", rev) },
+			}
+			for op, call := range ops {
+				// Bounded because a gone server answers nothing at all.
+				ctx, cancel := context.WithTimeout(t.Context(), 500*time.Millisecond)
+				err := call(ctx)
+				cancel()
+				assert.ErrorIs(t, err, kvsync.ErrUnavailable, op)
+				assert.NotErrorIs(t, err, kvsync.ErrRevisionMismatch, op)
+			}
+		})
+	}
+}
+
+// A recreated sync stream restarts its revisions, so a stale holder's revision
+// could match a new key. Losing the stream must leave every write failing.
+func TestSyncBucket_LostStreamIsNeverRecreated(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	b := kvstore.OpenSyncBucket(js)
+	rev, err := b.CompareAndSwap(t.Context(), "k", []byte("a"), 0)
+	require.NoError(t, err)
+	require.NoError(t, js.DeleteKeyValue(t.Context(), "spinifex-sync"))
+
+	for range 3 {
+		_, err = b.CompareAndSwap(t.Context(), "k", []byte("b"), 0)
+		require.Error(t, err)
+		_, err = b.CompareAndSwap(t.Context(), "k", []byte("b"), rev)
+		require.Error(t, err)
+		_, _, err = b.Load(t.Context(), "k")
+		require.Error(t, err)
+	}
+	_, err = js.KeyValue(t.Context(), "spinifex-sync")
+	require.ErrorIs(t, err, jetstream.ErrBucketNotFound, "the sync bucket must still be absent")
+}
+
+// stallingJS holds every bucket lookup until gate closes, standing in for an
+// open stuck on a server that has not formed quorum.
+type stallingJS struct {
+	jetstream.JetStream
+
+	entered chan struct{}
+	once    sync.Once
+	gate    chan struct{}
+}
+
+func (s *stallingJS) KeyValue(ctx context.Context, name string) (jetstream.KeyValue, error) {
+	s.once.Do(func() { close(s.entered) })
+	<-s.gate
+	return s.JetStream.KeyValue(ctx, name)
+}
+
+// One caller's stuck open must not hold another past its own context, or a
+// lease renewal waiting behind it could outlive its deadline.
+func TestSyncBucket_AWaiterOnAStuckOpenStopsAtItsContext(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	stall := &stallingJS{JetStream: js, entered: make(chan struct{}), gate: make(chan struct{})}
+	t.Cleanup(func() { close(stall.gate) })
+	b := kvstore.OpenSyncBucket(stall)
+
+	go func() { _, _, _ = b.Load(context.Background(), "k") }()
+	<-stall.entered
+
+	done := make(chan error, 1)
+	go func() {
+		ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
+		defer cancel()
+		_, err := b.CompareAndSwap(ctx, "k", []byte("a"), 0)
+		done <- err
+	}()
+	select {
+	case err := <-done:
+		require.ErrorIs(t, err, kvsync.ErrUnavailable)
+	case <-time.After(5 * time.Second):
+		t.Fatal("the swap waited on another caller's open past its own context")
+	}
+}
+
+// leaderlessJS answers every bucket lookup and create with the 503 a cluster
+// gives while JetStream has no meta leader.
+type leaderlessJS struct {
+	jetstream.JetStream
+}
+
+func errNoMetaLeader() error {
+	return &jetstream.APIError{Code: 503, ErrorCode: 10008, Description: "JetStream system temporarily unavailable"}
+}
+
+func (leaderlessJS) KeyValue(context.Context, string) (jetstream.KeyValue, error) {
+	return nil, errNoMetaLeader()
+}
+
+func (leaderlessJS) CreateKeyValue(context.Context, jetstream.KeyValueConfig) (jetstream.KeyValue, error) {
+	return nil, errNoMetaLeader()
+}
+
+// A cold cluster still electing its meta leader is an outage a lock caller
+// waits out, not a fault that ends its election.
+func TestSyncBucket_ALeaderlessJetStreamIsUnavailable(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	b := kvstore.OpenSyncBucket(leaderlessJS{JetStream: js})
+
+	_, _, err := b.Load(t.Context(), "k")
+	require.ErrorIs(t, err, kvsync.ErrUnavailable)
+	_, err = b.CompareAndSwap(t.Context(), "k", []byte("a"), 0)
+	require.ErrorIs(t, err, kvsync.ErrUnavailable)
+}

spinifex/foundation/state/kvsync/claim.go

@@ -0,0 +1,131 @@
+package kvsync
+
+import (
+	"context"
+	"errors"
+	"fmt"
+)
+
+// Claim is ownership of a key with no expiry: it is held until its holder
+// unlocks it or another holder takes it over. It keeps no state, so it gives no
+// exclusion between goroutines of one holder; its callers serialise their use.
+type Claim struct {
+	store  KVStore
+	key    string
+	holder string
+}
+
+// NewClaim returns the claim on key for holder. It panics if key is invalid or
+// already constructed against store, or if holder is empty.
+func NewClaim(store KVStore, key, holder string) *Claim {
+	mustWire(errors.Join(validHolder(holder), registerKey(store, key)))
+	return &Claim{store: store, key: key, holder: holder}
+}
+
+// TryLock takes the key when it is absent, and adopts it when it already names
+// this holder, keeping its generation. Otherwise it returns *HeldError naming
+// the holder.
+func (c *Claim) TryLock(ctx context.Context) (Token, error) {
+	var err error
+	for range casAttempts {
+		var rev uint64
+		rev, err = c.store.CompareAndSwap(ctx, c.key, encode(c.holder, 0), 0)
+		if err == nil {
+			return Token{Holder: c.holder, Generation: rev}, nil
+		}
+		if !errors.Is(err, ErrRevisionMismatch) {
+			return Token{}, fmt.Errorf("kvsync: claim %s: %w", c.key, err)
+		}
+		cur, rev, ok, rerr := read(ctx, c.store, c.key)
+		if rerr != nil {
+			return Token{}, rerr
+		}
+		if !ok {
+			// Unlocked between the create and the read; try again.
+			continue
+		}
+		if cur.Holder != c.holder {
+			return Token{}, &HeldError{Current: cur}
+		}
+		// A read can be answered by a lagging replica, so adoption is confirmed
+		// by a write at the revision read, which only the leader accepts.
+		_, err = c.store.CompareAndSwap(ctx, c.key, encode(c.holder, cur.Generation), rev)
+		if err == nil {
+			return cur, nil
+		}
+		if !errors.Is(err, ErrRevisionMismatch) {
+			return Token{}, fmt.Errorf("kvsync: claim %s: %w", c.key, err)
+		}
+	}
+	return Token{}, fmt.Errorf("kvsync: claim %s: %w", c.key, err)
+}
+
+// Unlock deletes the key while it still holds held, and returns ErrLost when
+// it does not.
+func (c *Claim) Unlock(ctx context.Context, held Token) error {
+	for range casAttempts {
+		cur, rev, ok, err := read(ctx, c.store, c.key)
+		if err != nil {
+			return err
+		}
+		if !ok || cur != held {
+			return ErrLost
+		}
+		err = c.store.CompareAndDelete(ctx, c.key, rev)
+		if err == nil {
+			return nil
+		}
+		if !errors.Is(err, ErrRevisionMismatch) {
+			return fmt.Errorf("kvsync: unlock %s: %w", c.key, err)
+		}
+	}
+	return ErrLost
+}
+
+// TakeOver moves the key to this holder only while it still holds from, the
+// token the caller judged dead from its own evidence. It returns *HeldError
+// when the key no longer holds from.
+func (c *Claim) TakeOver(ctx context.Context, from Token) (Token, error) {
+	cur, rev, ok, err := read(ctx, c.store, c.key)
+	if err != nil {
+		return Token{}, err
+	}
+	if !ok || cur != from {
+		return Token{}, &HeldError{Current: cur}
+	}
+	next, err := c.store.CompareAndSwap(ctx, c.key, encode(c.holder, 0), rev)
+	if errors.Is(err, ErrRevisionMismatch) {
+		return Token{}, heldAfterConflict(ctx, c.store, c.key)
+	}
+	if err != nil {
+		return Token{}, fmt.Errorf("kvsync: take over %s: %w", c.key, err)
+	}
+	return Token{Holder: c.holder, Generation: next}, nil
+}
+
+// Holder reports the token of whoever holds the key, and false when nobody does.
+func (c *Claim) Holder(ctx context.Context) (Token, bool, error) {
+	cur, _, ok, err := read(ctx, c.store, c.key)
+	return cur, ok, err
+}
+
+// ClaimGroup hands out per-resource claims under one registered prefix.
+type ClaimGroup struct {
+	store  KVStore
+	prefix string
+	holder string
+}
+
+// NewClaimGroup registers prefix, which ends in ".", against store. It panics
+// on an invalid or overlapping prefix, or an empty holder.
+func NewClaimGroup(store KVStore, prefix, holder string) *ClaimGroup {
+	mustWire(errors.Join(validHolder(holder), registerPrefix(store, prefix)))
+	return &ClaimGroup{store: store, prefix: prefix, holder: holder}
+}
+
+// Get returns the claim on prefix+name. A Claim keeps no state, so a fresh
+// value for the same name is equivalent to an earlier one.
+func (g *ClaimGroup) Get(name string) *Claim {
+	mustWire(validName("name", name))
+	return &Claim{store: g.store, key: g.prefix + name, holder: g.holder}
+}

spinifex/foundation/state/kvsync/claim_test.go

@@ -0,0 +1,160 @@
+package kvsync_test
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestClaim_SecondHolderIsRefusedAndSameHolderAdoptsWithoutAWrite(t *testing.T) {
+	s := stores(t, 3)
+	a := kvsync.NewClaim(s[0], "claim.x", "node-a")
+	b := kvsync.NewClaim(s[1], "claim.x", "node-b")
+	restarted := kvsync.NewClaim(s[2], "claim.x", "node-a")
+
+	held, err := a.TryLock(t.Context())
+	require.NoError(t, err)
+
+	_, err = b.TryLock(t.Context())
+	he, ok := asHeld(err)
+	require.True(t, ok, "want *HeldError, got %v", err)
+	assert.Equal(t, held, he.Current, "the refusal must name the holder and its generation")
+
+	adopted, err := restarted.TryLock(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, held, adopted, "adoption keeps the generation")
+	cur, ok, err := a.Holder(t.Context())
+	require.NoError(t, err)
+	require.True(t, ok)
+	assert.Equal(t, held, cur, "the adoption's confirming write keeps the generation in the key")
+}
+
+// A replica that has not seen a take-over still names the old holder. Its read
+// alone must not let that holder adopt the key back from the new one.
+func TestClaim_AStaleReadCannotAdoptATakenOverKey(t *testing.T) {
+	s := stores(t, 3)
+	a := kvsync.NewClaim(s[0], "claim.x", "node-a")
+	b := kvsync.NewClaim(s[1], "claim.x", "node-b")
+
+	first, err := a.TryLock(t.Context())
+	require.NoError(t, err)
+	value, rev, err := s[0].Load(t.Context(), "claim.x")
+	require.NoError(t, err)
+	second, err := b.TakeOver(t.Context(), first)
+	require.NoError(t, err)
+
+	stale := &staleStore{KVStore: s[2], value: value, rev: rev}
+	restarted := kvsync.NewClaim(stale, "claim.x", "node-a")
+	_, err = restarted.TryLock(t.Context())
+	_, held := asHeld(err)
+	assert.True(t, held, "a stale read must not adopt the key, got %v", err)
+	cur, ok, err := b.Holder(t.Context())
+	require.NoError(t, err)
+	require.True(t, ok)
+	assert.Equal(t, second, cur, "the key must still name the holder that took it over")
+}
+
+func TestClaim_TakeOverNeedsTheCurrentToken(t *testing.T) {
+	s := stores(t, 3)
+	a := kvsync.NewClaim(s[0], "claim.x", "node-a")
+	b := kvsync.NewClaim(s[1], "claim.x", "node-b")
+	c := kvsync.NewClaim(s[2], "claim.x", "node-c")
+
+	first, err := a.TryLock(t.Context())
+	require.NoError(t, err)
+
+	second, err := b.TakeOver(t.Context(), first)
+	require.NoError(t, err)
+	assert.Equal(t, "node-b", second.Holder)
+	assert.Greater(t, second.Generation, first.Generation)
+
+	// c judged node-a dead, but node-b has moved the key on since.
+	_, err = c.TakeOver(t.Context(), first)
+	he, ok := asHeld(err)
+	require.True(t, ok, "want *HeldError, got %v", err)
+	assert.Equal(t, second, he.Current)
+}
+
+func TestClaim_UnlockAfterTakeOverIsLostAndLeavesTheNewHolder(t *testing.T) {
+	s := stores(t, 2)
+	a := kvsync.NewClaim(s[0], "claim.x", "node-a")
+	b := kvsync.NewClaim(s[1], "claim.x", "node-b")
+
+	first, err := a.TryLock(t.Context())
+	require.NoError(t, err)
+	second, err := b.TakeOver(t.Context(), first)
+	require.NoError(t, err)
+
+	require.ErrorIs(t, a.Unlock(t.Context(), first), kvsync.ErrLost)
+
+	cur, ok, err := a.Holder(t.Context())
+	require.NoError(t, err)
+	require.True(t, ok, "the stale unlock deleted the new holder's key")
+	assert.Equal(t, second, cur)
+}
+
+func TestClaim_RelockAfterUnlockGetsAHigherGeneration(t *testing.T) {
+	s := stores(t, 1)
+	a := kvsync.NewClaim(s[0], "claim.x", "node-a")
+
+	first, err := a.TryLock(t.Context())
+	require.NoError(t, err)
+	require.NoError(t, a.Unlock(t.Context(), first))
+
+	_, ok, err := a.Holder(t.Context())
+	require.NoError(t, err)
+	require.False(t, ok, "unlock must free the key")
+
+	second, err := a.TryLock(t.Context())
+	require.NoError(t, err)
+	assert.Greater(t, second.Generation, first.Generation, "a generation must outrank every earlier one, across a delete")
+}
+
+func TestClaimGroup_HandsOutKeysUnderItsPrefix(t *testing.T) {
+	s := stores(t, 2)
+	ga := kvsync.NewClaimGroup(s[0], "uefi.", "node-a")
+	gb := kvsync.NewClaimGroup(s[1], "uefi.", "node-b")
+
+	_, err := ga.Get("i-1").TryLock(t.Context())
+	require.NoError(t, err)
+	_, err = gb.Get("i-1").TryLock(t.Context())
+	_, ok := asHeld(err)
+	assert.True(t, ok, "the same name in two groups is the same key")
+	_, err = gb.Get("i-2").TryLock(t.Context())
+	assert.NoError(t, err, "a different name is a different key")
+	assert.NotZero(t, revision(t, s[0], "uefi.i-1"))
+}
+
+// A peer that moves the key between the take-over's read and its write wins,
+// and the refusal names that peer rather than the holder judged dead.
+func TestClaim_TakeOverThatLosesTheRaceNamesTheWinner(t *testing.T) {
+	s := stores(t, 3)
+	a := kvsync.NewClaim(s[0], "claim.x", "node-a")
+	b := kvsync.NewClaim(s[1], "claim.x", "node-b")
+
+	first, err := a.TryLock(t.Context())
+	require.NoError(t, err)
+	var winner kvsync.Token
+	racing := &racingStore{KVStore: s[2], race: func() {
+		winner, err = b.TakeOver(t.Context(), first)
+		require.NoError(t, err)
+	}}
+	c := kvsync.NewClaim(racing, "claim.x", "node-c")
+
+	_, err = c.TakeOver(t.Context(), first)
+	he, ok := asHeld(err)
+	require.True(t, ok, "want *HeldError, got %v", err)
+	assert.Equal(t, winner, he.Current)
+}
+
+func TestClaim_UnlockOfAnAbsentKeyIsLost(t *testing.T) {
+	s := stores(t, 1)
+	a := kvsync.NewClaim(s[0], "claim.x", "node-a")
+
+	first, err := a.TryLock(t.Context())
+	require.NoError(t, err)
+	require.NoError(t, a.Unlock(t.Context(), first))
+	require.ErrorIs(t, a.Unlock(t.Context(), first), kvsync.ErrLost, "a key already gone is not this holder's to free")
+}

spinifex/foundation/state/kvsync/clock.go

@@ -0,0 +1,21 @@
+package kvsync
+
+import "time"
+
+// clock is the monotonic time source a Lease measures expiry and renewal on.
+type clock interface {
+	Now() time.Time
+	After(d time.Duration) <-chan time.Time
+	// Deadline is After with a stop, for a bound that is usually not reached.
+	Deadline(d time.Duration) (<-chan time.Time, func())
+}
+
+type realClock struct{}
+
+func (realClock) Now() time.Time                         { return time.Now() }
+func (realClock) After(d time.Duration) <-chan time.Time { return time.After(d) }
+
+func (realClock) Deadline(d time.Duration) (<-chan time.Time, func()) {
+	t := time.NewTimer(d)
+	return t.C, func() { t.Stop() }
+}

spinifex/foundation/state/kvsync/election.go

@@ -0,0 +1,71 @@
+package kvsync
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"time"
+)
+
+// RunElection holds l across many passes until ctx is cancelled, retrying
+// every ttl/3 while another holder leads. onGained fires once per acquisition
+// and onLost once per loss, neither under a lock; either may be nil.
+//
+// An onGained that fails stands the lease down, because a leader that cannot
+// set up its work is worse than none; onLost does not fire for it. On return
+// the lease is unlocked.
+func RunElection(ctx context.Context, l *Lease, onGained func(context.Context) error, onLost func()) {
+	// lost is nil while the lease is not held.
+	var (
+		tok  Token
+		lost <-chan struct{}
+	)
+	try := func() {
+		t, err := l.TryLock(ctx)
+		if err != nil {
+			if he, ok := errors.AsType[*HeldError](err); ok {
+				slog.DebugContext(ctx, "kvsync: election held elsewhere", "key", l.key, "node", l.holder, "holder", he.Current.Holder)
+			} else {
+				slog.WarnContext(ctx, "kvsync: election attempt failed", "key", l.key, "node", l.holder, "err", err)
+			}
+			return
+		}
+		ch := l.Lost()
+		slog.InfoContext(ctx, "kvsync: elected", "key", l.key, "node", l.holder, "generation", t.Generation)
+		if onGained != nil {
+			if err := onGained(ctx); err != nil {
+				slog.ErrorContext(ctx, "kvsync: onGained failed, standing down", "key", l.key, "node", l.holder, "err", err)
+				l.Unlock(ctx, t)
+				return
+			}
+		}
+		tok, lost = t, ch
+	}
+	stepDown := func() {
+		lost = nil
+		if onLost != nil {
+			onLost()
+		}
+	}
+
+	try()
+	for {
+		var retry <-chan time.Time
+		if lost == nil {
+			retry = l.clock.After(l.ttl / 3)
+		}
+		select {
+		case <-ctx.Done():
+			if lost != nil {
+				l.Unlock(ctx, tok)
+				stepDown()
+			}
+			return
+		case <-lost:
+			slog.WarnContext(ctx, "kvsync: election lost", "key", l.key, "node", l.holder, "generation", tok.Generation)
+			stepDown()
+		case <-retry:
+			try()
+		}
+	}
+}

spinifex/foundation/state/kvsync/election_test.go

@@ -0,0 +1,89 @@
+package kvsync_test
+
+import (
+	"context"
+	"errors"
+	"sync/atomic"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRunElection_FiresEachEdgeOncePerTransition(t *testing.T) {
+	s := stores(t, 2)
+	c := kvsync.NewFakeClock()
+	l := kvsync.NewLeaseWithClock(s[0], leaseKey, "node-a", testTTL, c)
+
+	gained := make(chan struct{}, 4)
+	lostEdge := make(chan struct{}, 4)
+	var gains, losses atomic.Int32
+	ctx, cancel := context.WithCancel(t.Context())
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		kvsync.RunElection(ctx, l,
+			func(context.Context) error { gains.Add(1); gained <- struct{}{}; return nil },
+			func() { losses.Add(1); lostEdge <- struct{}{} })
+	}()
+
+	requireClosedRecv(t, gained, "the first attempt on a free key must gain")
+
+	// Another holder takes the key, so the next renewal loses it.
+	takeKey(t, s[1])
+	waitForWaiters(t, c, 1)
+	c.Advance(testTTL / 3)
+	requireClosedRecv(t, lostEdge, "a lost renewal must fire onLost")
+
+	// The election keeps retrying every ttl/3 and takes the taker's key once
+	// it has seen it unchanged for a TTL.
+	for range 4 {
+		waitForWaiters(t, c, 1)
+		c.Advance(testTTL / 3)
+	}
+	requireClosedRecv(t, gained, "the election must regain the lease after a TTL")
+
+	cancel()
+	<-done
+	requireClosedRecv(t, lostEdge, "stopping the election must fire onLost for the held lease")
+	assert.Equal(t, int32(2), gains.Load())
+	assert.Equal(t, int32(2), losses.Load())
+	assert.Zero(t, revision(t, s[0], leaseKey), "stopping the election must release the key")
+}
+
+func TestRunElection_FailedOnGainedStandsDownWithoutOnLost(t *testing.T) {
+	s := stores(t, 1)
+	c := kvsync.NewFakeClock()
+	l := kvsync.NewLeaseWithClock(s[0], leaseKey, "node-a", testTTL, c)
+
+	attempted := make(chan struct{}, 1)
+	var losses atomic.Int32
+	ctx, cancel := context.WithCancel(t.Context())
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		kvsync.RunElection(ctx, l,
+			func(context.Context) error { attempted <- struct{}{}; return errors.New("cannot lead") },
+			func() { losses.Add(1) })
+	}()
+	requireClosedRecv(t, attempted, "the first attempt must reach onGained")
+	// The stopped renewal leaves its one timer behind, and the retry timer is
+	// registered only after the stand-down has unlocked.
+	waitForWaiters(t, c, 2)
+	assert.False(t, l.Held(), "a failed onGained must stand the lease down")
+	assert.Zero(t, revision(t, s[0], leaseKey), "standing down must free the key for a healthy node")
+
+	cancel()
+	<-done
+	assert.Zero(t, losses.Load(), "a lease never gained is never lost")
+}
+
+func requireClosedRecv(t *testing.T, ch <-chan struct{}, msg string) {
+	t.Helper()
+	select {
+	case <-ch:
+	case <-waitCtx(t).Done():
+		require.FailNow(t, msg)
+	}
+}

spinifex/foundation/state/kvsync/export_test.go

@@ -0,0 +1,143 @@
+package kvsync
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"time"
+)
+
+// FakeClock is a manual clock: time moves only on Advance, and a waiter fires
+// once Advance has carried the clock to its deadline.
+type FakeClock struct {
+	mu      sync.Mutex
+	now     time.Time
+	waiters []fakeWaiter
+	changed chan struct{}
+}
+
+type fakeWaiter struct {
+	at time.Time
+	ch chan time.Time
+	// deadline marks a Deadline timer, which the waiter counts leave out.
+	deadline bool
+}
+
+// NewFakeClock returns a clock reading an arbitrary fixed instant.
+func NewFakeClock() *FakeClock {
+	return &FakeClock{now: time.Unix(1_000_000, 0), changed: make(chan struct{})}
+}
+
+func (c *FakeClock) Now() time.Time {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.now
+}
+
+func (c *FakeClock) After(d time.Duration) <-chan time.Time {
+	return c.add(d, false)
+}
+
+func (c *FakeClock) Deadline(d time.Duration) (<-chan time.Time, func()) {
+	ch := c.add(d, true)
+	return ch, func() {
+		c.mu.Lock()
+		defer c.mu.Unlock()
+		kept := c.waiters[:0]
+		for _, w := range c.waiters {
+			if w.ch != ch {
+				kept = append(kept, w)
+			}
+		}
+		c.waiters = kept
+		c.notifyLocked()
+	}
+}
+
+func (c *FakeClock) add(d time.Duration, deadline bool) chan time.Time {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	ch := make(chan time.Time, 1)
+	if d <= 0 {
+		ch <- c.now
+		return ch
+	}
+	c.waiters = append(c.waiters, fakeWaiter{at: c.now.Add(d), ch: ch, deadline: deadline})
+	c.notifyLocked()
+	return ch
+}
+
+// Advance moves the clock on by d and fires every waiter now due.
+func (c *FakeClock) Advance(d time.Duration) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.now = c.now.Add(d)
+	kept := c.waiters[:0]
+	for _, w := range c.waiters {
+		if w.at.After(c.now) {
+			kept = append(kept, w)
+			continue
+		}
+		w.ch <- c.now
+	}
+	c.waiters = kept
+	c.notifyLocked()
+}
+
+// WaitForWaiters blocks until at least n After waiters are pending or ctx
+// ends, and reports whether they were.
+func (c *FakeClock) WaitForWaiters(ctx context.Context, n int) bool {
+	return c.waitFor(ctx, n, false)
+}
+
+// WaitForDeadlines is WaitForWaiters for Deadline timers.
+func (c *FakeClock) WaitForDeadlines(ctx context.Context, n int) bool {
+	return c.waitFor(ctx, n, true)
+}
+
+func (c *FakeClock) waitFor(ctx context.Context, n int, deadline bool) bool {
+	for {
+		c.mu.Lock()
+		count, changed := c.countLocked(deadline), c.changed
+		c.mu.Unlock()
+		if count >= n {
+			return true
+		}
+		select {
+		case <-ctx.Done():
+			return false
+		case <-changed:
+		}
+	}
+}
+
+// Waiters reports how many After waiters are pending.
+func (c *FakeClock) Waiters() int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.countLocked(false)
+}
+
+func (c *FakeClock) countLocked(deadline bool) int {
+	n := 0
+	for _, w := range c.waiters {
+		if w.deadline == deadline {
+			n++
+		}
+	}
+	return n
+}
+
+func (c *FakeClock) notifyLocked() {
+	close(c.changed)
+	c.changed = make(chan struct{})
+}
+
+// NewLeaseWithClock is NewLease measuring time on c.
+func NewLeaseWithClock(store KVStore, key, holder string, ttl time.Duration, c *FakeClock) *Lease {
+	mustWire(errors.Join(validHolder(holder), validTTL(ttl), registerKey(store, key)))
+	return newLease(store, key, holder, ttl, c)
+}
+
+// SetGroupClock makes every Lease g hands out from now on measure time on c.
+func SetGroupClock(g *LeaseGroup, c *FakeClock) { g.clock = c }

spinifex/foundation/state/kvsync/helpers_test.go

@@ -0,0 +1,216 @@
+package kvsync_test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/stretchr/testify/require"
+)
+
+// testTTL is a lease's ttl in the fake-clock tests, where only its ratio to
+// the renewal and retry intervals matters.
+const testTTL = 60 * time.Second
+
+// stores returns n distinct store values over one sync bucket, each standing
+// in for a separate process.
+func stores(t *testing.T, n int) []*kvstore.Bucket {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	out := make([]*kvstore.Bucket, n)
+	for i := range out {
+		out[i] = kvstore.OpenSyncBucket(js)
+	}
+	return out
+}
+
+// revision is the key's current revision, or 0 when it is absent.
+func revision(t *testing.T, s kvsync.KVStore, key string) uint64 {
+	t.Helper()
+	_, rev, err := s.Load(t.Context(), key)
+	if err != nil {
+		require.ErrorIs(t, err, kvsync.ErrNotFound)
+		return 0
+	}
+	return rev
+}
+
+// waitCtx bounds a wait on a condition so a broken test fails rather than
+// hanging; it is never what makes the condition true.
+func waitCtx(t *testing.T) context.Context {
+	t.Helper()
+	ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
+	t.Cleanup(cancel)
+	return ctx
+}
+
+// waitForWaiters fails the test unless c has n pending waiters in time.
+func waitForWaiters(t *testing.T, c *kvsync.FakeClock, n int) {
+	t.Helper()
+	require.True(t, c.WaitForWaiters(waitCtx(t), n), "clock never reached %d waiters", n)
+}
+
+// renewOnce advances c by one renewal interval and waits for the renewal to
+// finish and schedule the next.
+func renewOnce(t *testing.T, c *kvsync.FakeClock) {
+	t.Helper()
+	waitForWaiters(t, c, 1)
+	c.Advance(testTTL / 3)
+	waitForWaiters(t, c, 1)
+}
+
+func requireClosed(t *testing.T, ch <-chan struct{}, msg string) {
+	t.Helper()
+	select {
+	case <-ch:
+	case <-waitCtx(t).Done():
+		t.Fatal(msg)
+	}
+}
+
+func requireOpen(t *testing.T, ch <-chan struct{}, msg string) {
+	t.Helper()
+	select {
+	case <-ch:
+		t.Fatal(msg)
+	default:
+	}
+}
+
+// faultyStore wraps a store, failing writes with ErrUnavailable while down is
+// set and counting every CompareAndSwap.
+type faultyStore struct {
+	kvsync.KVStore
+
+	down  atomic.Bool
+	swaps atomic.Int64
+}
+
+func (f *faultyStore) CompareAndSwap(ctx context.Context, key string, value []byte, rev uint64) (uint64, error) {
+	f.swaps.Add(1)
+	if f.down.Load() {
+		return 0, kvsync.ErrUnavailable
+	}
+	return f.KVStore.CompareAndSwap(ctx, key, value, rev)
+}
+
+// gatedStore wraps a store so that its CompareAndSwap waits at a shared
+// barrier, holding contenders until every one has read the same revision.
+type gatedStore struct {
+	kvsync.KVStore
+
+	gate *barrier
+}
+
+func (g *gatedStore) CompareAndSwap(ctx context.Context, key string, value []byte, rev uint64) (uint64, error) {
+	if err := g.gate.wait(); err != nil {
+		return 0, err
+	}
+	return g.KVStore.CompareAndSwap(ctx, key, value, rev)
+}
+
+// barrier releases its callers once n have arrived, and fails each of them
+// instead when ctx ends first, so a contender that never arrives fails the test.
+type barrier struct {
+	ctx     context.Context
+	mu      sync.Mutex
+	pending int
+	all     chan struct{}
+}
+
+func newBarrier(ctx context.Context, n int) *barrier {
+	return &barrier{ctx: ctx, pending: n, all: make(chan struct{})}
+}
+
+func (b *barrier) wait() error {
+	b.mu.Lock()
+	b.pending--
+	if b.pending == 0 {
+		close(b.all)
+	}
+	b.mu.Unlock()
+	select {
+	case <-b.all:
+		return nil
+	case <-b.ctx.Done():
+		return fmt.Errorf("barrier: not every contender arrived: %w", b.ctx.Err())
+	}
+}
+
+// racingStore runs race once, before the first CompareAndSwap, standing in for
+// a peer whose write lands between this caller's read and its write.
+type racingStore struct {
+	kvsync.KVStore
+
+	race func()
+	once sync.Once
+}
+
+func (r *racingStore) CompareAndSwap(ctx context.Context, key string, value []byte, rev uint64) (uint64, error) {
+	r.once.Do(r.race)
+	return r.KVStore.CompareAndSwap(ctx, key, value, rev)
+}
+
+func asHeld(err error) (*kvsync.HeldError, bool) {
+	return errors.AsType[*kvsync.HeldError](err)
+}
+
+// slowStore wraps a store so that a CompareAndSwap can run beforeSwap before it
+// is sent, standing in for a stalled request, and afterSwap once it has
+// committed but before its response returns, standing in for a delayed reply.
+type slowStore struct {
+	kvsync.KVStore
+
+	calls      atomic.Int64
+	swaps      atomic.Int64
+	beforeSwap func(n int64)
+	afterSwap  func(n int64)
+}
+
+func (s *slowStore) CompareAndSwap(ctx context.Context, key string, value []byte, rev uint64) (uint64, error) {
+	if n := s.calls.Add(1); s.beforeSwap != nil {
+		s.beforeSwap(n)
+	}
+	next, err := s.KVStore.CompareAndSwap(ctx, key, value, rev)
+	if n := s.swaps.Add(1); err == nil && s.afterSwap != nil {
+		s.afterSwap(n)
+	}
+	return next, err
+}
+
+// waitForSwaps fails the test unless s has seen n CompareAndSwap calls in time.
+func waitForSwaps(t *testing.T, s *slowStore, n int64) {
+	t.Helper()
+	ctx := waitCtx(t)
+	for s.swaps.Load() < n {
+		select {
+		case <-ctx.Done():
+			t.Fatalf("only %d of %d swaps happened", s.swaps.Load(), n)
+		case <-time.After(time.Millisecond):
+		}
+	}
+}
+
+// staleStore answers its first Load with value at rev, as a replica behind the
+// leader would, and every later Load from the store itself.
+type staleStore struct {
+	kvsync.KVStore
+
+	value []byte
+	rev   uint64
+	used  atomic.Bool
+}
+
+func (s *staleStore) Load(ctx context.Context, key string) ([]byte, uint64, error) {
+	if s.used.CompareAndSwap(false, true) {
+		return s.value, s.rev, nil
+	}
+	return s.KVStore.Load(ctx, key)
+}

spinifex/foundation/state/kvsync/kvsync.go

@@ -0,0 +1,109 @@
+// Package kvsync provides ownership of a key over a compare-and-swap store, in
+// the manner of the standard sync package: a Claim is held until its holder
+// unlocks it or another takes it over, and a Lease lapses unless renewed.
+//
+// Every acquisition returns a Token whose generation is the revision of the
+// write that acquired the key. A store revision only grows, so a later holder's
+// generation always exceeds an earlier one's, and a caller writing elsewhere
+// fences on it there. Nothing in this package fences on its own.
+package kvsync
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+)
+
+// KVStore is the compare-and-swap surface every kvsync lock is built on. A
+// revision must only grow, including across a delete. The store value keys the
+// registry of constructed locks, so it must be comparable, such as a pointer.
+type KVStore interface {
+	Load(ctx context.Context, key string) (value []byte, rev uint64, err error)
+	CompareAndSwap(ctx context.Context, key string, value []byte, rev uint64) (uint64, error)
+	CompareAndDelete(ctx context.Context, key string, rev uint64) error
+}
+
+var (
+	// ErrNotFound is Load's report that no such key exists.
+	ErrNotFound = errors.New("kvsync: key not found")
+	// ErrRevisionMismatch reports that the key is not at the revision given;
+	// revision 0 means the key must be absent.
+	ErrRevisionMismatch = errors.New("kvsync: revision mismatch")
+	// ErrUnavailable reports that the store could not be reached.
+	ErrUnavailable = errors.New("kvsync: store unavailable")
+	// ErrLost reports that the lock was taken over or lapsed.
+	ErrLost = errors.New("kvsync: lock lost")
+)
+
+// Token names one acquisition of a key.
+type Token struct {
+	Holder     string
+	Generation uint64
+}
+
+// HeldError reports that another holder has the key. Current is zero when the
+// key was found absent after the conflict that produced this error.
+type HeldError struct{ Current Token }
+
+func (e *HeldError) Error() string {
+	if e.Current.Holder == "" {
+		return "kvsync: lock held by another holder"
+	}
+	return fmt.Sprintf("kvsync: lock held by %s at generation %d", e.Current.Holder, e.Current.Generation)
+}
+
+// record is the stored value. An acquiring write cannot know its own revision,
+// so it stores generation 0, which a reader takes to mean the entry's revision.
+type record struct {
+	Holder     string `json:"holder"`
+	Generation uint64 `json:"generation"`
+}
+
+func encode(holder string, generation uint64) []byte {
+	data, _ := json.Marshal(record{Holder: holder, Generation: generation}) //nolint:errchkjson // A struct of a string and an integer cannot fail to marshal.
+	return data
+}
+
+func decode(key string, value []byte, rev uint64) (Token, error) {
+	var r record
+	if err := json.Unmarshal(value, &r); err != nil {
+		return Token{}, fmt.Errorf("kvsync: decode %s: %w", key, err)
+	}
+	if r.Holder == "" {
+		return Token{}, fmt.Errorf("kvsync: decode %s: no holder", key)
+	}
+	if r.Generation == 0 {
+		r.Generation = rev
+	}
+	return Token(r), nil
+}
+
+// read loads and decodes key. ok is false when the key is absent.
+func read(ctx context.Context, store KVStore, key string) (tok Token, rev uint64, ok bool, err error) {
+	value, rev, err := store.Load(ctx, key)
+	if errors.Is(err, ErrNotFound) {
+		return Token{}, 0, false, nil
+	}
+	if err != nil {
+		return Token{}, 0, false, fmt.Errorf("kvsync: load %s: %w", key, err)
+	}
+	tok, err = decode(key, value, rev)
+	if err != nil {
+		return Token{}, 0, false, err
+	}
+	return tok, rev, true, nil
+}
+
+// heldAfterConflict reports who has the key after a write lost its CAS.
+func heldAfterConflict(ctx context.Context, store KVStore, key string) error {
+	cur, _, _, err := read(ctx, store, key)
+	if err != nil {
+		return err
+	}
+	return &HeldError{Current: cur}
+}
+
+// casAttempts bounds the read-then-write loops that retry when the key changes
+// between the read and the write.
+const casAttempts = 3

spinifex/foundation/state/kvsync/lease.go

@@ -0,0 +1,353 @@
+package kvsync
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"sync"
+	"time"
+)
+
+// releaseTimeout bounds Unlock's delete, which runs on a detached context so
+// that a cancelled caller still frees the key for the next holder.
+const releaseTimeout = 5 * time.Second
+
+// Lease is ownership of a key that lapses unless its holder renews it. Within
+// a process it excludes like a sync.Mutex: while held, TryLock from any call
+// site returns *HeldError. A Lease is reusable after Unlock or after Lost closes.
+//
+// Expiry is never read from the store. A key another holder has is taken over
+// only once this Lease has seen the same revision for a full TTL on its own
+// monotonic clock, so skew between nodes cannot shorten a lease.
+type Lease struct {
+	store  KVStore
+	key    string
+	holder string
+	ttl    time.Duration
+	clock  clock
+
+	// mu guards every field below.
+	mu sync.Mutex
+	// acquiring is set while a TryLock is between its read and its write.
+	acquiring bool
+	cur       *acquisition
+	// lost is the current or most recent acquisition's channel.
+	lost chan struct{}
+	// obsRev is another holder's revision as first seen at obsAt.
+	obsRev uint64
+	obsAt  time.Time
+}
+
+// acquisition is one holding of a Lease, from TryLock to Unlock or loss.
+type acquisition struct {
+	gen  uint64
+	lost chan struct{}
+	done chan struct{}
+	// rev and landed, guarded by Lease.mu, are the last write that landed and
+	// when it was sent.
+	rev    uint64
+	landed time.Time
+}
+
+// NewLease returns the lease on key for holder. It panics if key is invalid or
+// already constructed against store, if holder is empty, or if ttl is not positive.
+func NewLease(store KVStore, key, holder string, ttl time.Duration) *Lease {
+	mustWire(errors.Join(validHolder(holder), validTTL(ttl), registerKey(store, key)))
+	return newLease(store, key, holder, ttl, realClock{})
+}
+
+// newLease builds a Lease whose holder and ttl the caller has checked.
+func newLease(store KVStore, key, holder string, ttl time.Duration, c clock) *Lease {
+	closed := make(chan struct{})
+	close(closed)
+	return &Lease{store: store, key: key, holder: holder, ttl: ttl, clock: c, lost: closed}
+}
+
+// Key returns the store key the lease is held on.
+func (l *Lease) Key() string { return l.key }
+
+// TTL returns the lease's time to live, which a contender retries a third of.
+func (l *Lease) TTL() time.Duration { return l.ttl }
+
+// TryLock makes one attempt to acquire the key, returning the token of its own
+// write. Absent, the key is created; naming this holder, it is adopted by CAS;
+// naming another, it is taken over once its revision has been seen for a TTL.
+func (l *Lease) TryLock(ctx context.Context) (Token, error) {
+	l.mu.Lock()
+	if l.cur != nil || l.acquiring {
+		cur := Token{Holder: l.holder}
+		if l.cur != nil {
+			cur.Generation = l.cur.gen
+		}
+		l.mu.Unlock()
+		return Token{}, &HeldError{Current: cur}
+	}
+	l.acquiring = true
+	l.mu.Unlock()
+	defer func() {
+		l.mu.Lock()
+		l.acquiring = false
+		l.mu.Unlock()
+	}()
+
+	rev, sent, err := l.acquire(ctx)
+	if err != nil {
+		return Token{}, err
+	}
+	return l.install(ctx, rev, sent), nil
+}
+
+// acquire returns the revision of its write and when the write was sent. A
+// peer's TTL starts no earlier than the send, so validity is counted from it,
+// and a grant whose response took 2*ttl/3 is refused rather than installed.
+func (l *Lease) acquire(ctx context.Context) (uint64, time.Time, error) {
+	cur, rev, ok, err := read(ctx, l.store, l.key)
+	if err != nil {
+		return 0, time.Time{}, err
+	}
+	if ok && cur.Holder != l.holder && !l.observedFor(rev) {
+		return 0, time.Time{}, &HeldError{Current: cur}
+	}
+	validity := 2 * l.ttl / 3
+	sent := l.clock.Now()
+	casCtx, cancel := context.WithTimeout(ctx, validity)
+	next, err := l.store.CompareAndSwap(casCtx, l.key, encode(l.holder, 0), rev)
+	cancel()
+	if errors.Is(err, ErrRevisionMismatch) {
+		return 0, time.Time{}, heldAfterConflict(ctx, l.store, l.key)
+	}
+	if err != nil {
+		return 0, time.Time{}, fmt.Errorf("kvsync: lock %s: %w", l.key, err)
+	}
+	if l.clock.Now().Sub(sent) >= validity {
+		return 0, time.Time{}, fmt.Errorf("kvsync: lock %s: grant arrived after its validity: %w", l.key, ErrUnavailable)
+	}
+	return next, sent, nil
+}
+
+// observedFor records rev as seen now if it is new, and reports whether it has
+// been seen for a full TTL.
+func (l *Lease) observedFor(rev uint64) bool {
+	now := l.clock.Now()
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if l.obsRev != rev {
+		l.obsRev, l.obsAt = rev, now
+		return false
+	}
+	return now.Sub(l.obsAt) >= l.ttl
+}
+
+// install records an acquisition at rev, valid from sent, and starts its
+// renewal, which ignores the caller's cancellation and stops when Lost closes.
+func (l *Lease) install(ctx context.Context, rev uint64, sent time.Time) Token {
+	a := &acquisition{
+		gen:    rev,
+		rev:    rev,
+		landed: sent,
+		lost:   make(chan struct{}),
+		done:   make(chan struct{}),
+	}
+	l.mu.Lock()
+	l.cur, l.lost, l.obsRev = a, a.lost, 0
+	l.mu.Unlock()
+	go l.renew(context.WithoutCancel(ctx), a)
+	return Token{Holder: l.holder, Generation: rev}
+}
+
+// renew rewrites the key by CAS ttl/3 after each write that landed was sent,
+// until Lost closes. A mismatch loses the lease at once. An unreachable store is
+// retried, and the lease is lost at 2*ttl/3 after that send, however long a CAS has been stuck.
+func (l *Lease) renew(ctx context.Context, a *acquisition) {
+	defer close(a.done)
+	interval := l.ttl / 3
+	validity := 2 * l.ttl / 3
+	retry := max(l.ttl/12, time.Millisecond)
+	var lastErr error
+	for {
+		l.mu.Lock()
+		rev, landed := a.rev, a.landed
+		l.mu.Unlock()
+		deadline := landed.Add(validity)
+		wait := max(0, landed.Add(interval).Sub(l.clock.Now()))
+		if lastErr != nil {
+			wait = min(retry, deadline.Sub(l.clock.Now()))
+		}
+		select {
+		case <-a.lost:
+			return
+		case <-l.clock.After(wait):
+		}
+		budget := deadline.Sub(l.clock.Now())
+		if budget <= 0 {
+			l.lose(ctx, a, "renewal unavailable for two thirds of the ttl", lastErr)
+			return
+		}
+		sent := l.clock.Now()
+		next, err := l.swapWithin(ctx, a, rev, budget, min(budget, interval))
+		switch {
+		case errors.Is(err, errRenewalStopped):
+			return
+		case errors.Is(err, errRenewalLate):
+			l.lose(ctx, a, "renewal did not answer within two thirds of the ttl", nil)
+			return
+		case err == nil:
+			l.mu.Lock()
+			// Stamped at the send: a peer may have seen the revision that early.
+			a.rev, a.landed = next, sent
+			l.mu.Unlock()
+			lastErr = nil
+		case errors.Is(err, ErrRevisionMismatch):
+			l.lose(ctx, a, "renewal lost the key", err)
+			return
+		default:
+			lastErr = err
+		}
+	}
+}
+
+var (
+	// errRenewalStopped reports a renewal CAS abandoned because Unlock stopped it.
+	errRenewalStopped = errors.New("kvsync: renewal stopped")
+	// errRenewalLate reports a renewal CAS that did not answer within its budget.
+	errRenewalLate = errors.New("kvsync: renewal late")
+)
+
+// swapWithin runs one renewal CAS, giving up on it at Unlock or once budget has
+// passed on the lease's clock, since a CAS can block past its own context. A
+// late result is dropped; it cannot land after Unlock's delete at a.rev.
+func (l *Lease) swapWithin(ctx context.Context, a *acquisition, rev uint64, budget, timeout time.Duration) (uint64, error) {
+	type result struct {
+		next uint64
+		err  error
+	}
+	done := make(chan result, 1)
+	attemptCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+	go func() {
+		next, err := l.store.CompareAndSwap(attemptCtx, l.key, encode(l.holder, a.gen), rev)
+		done <- result{next, err}
+	}()
+	expired, stop := l.clock.Deadline(budget)
+	defer stop()
+	select {
+	case r := <-done:
+		return r.next, r.err
+	case <-expired:
+		return 0, errRenewalLate
+	case <-a.lost:
+		return 0, errRenewalStopped
+	}
+}
+
+// lose ends a if it is still the current acquisition.
+func (l *Lease) lose(ctx context.Context, a *acquisition, why string, err error) {
+	l.mu.Lock()
+	current := l.cur == a
+	if current {
+		l.cur = nil
+		close(a.lost)
+	}
+	l.mu.Unlock()
+	if current {
+		slog.WarnContext(ctx, "kvsync: lease lost", "key", l.key, "holder", l.holder,
+			"generation", a.gen, "reason", why, "err", err)
+	}
+}
+
+// Unlock releases the acquisition held is the token of, and does nothing
+// unless this Lease is still held at that generation, so a stale release never
+// frees a later acquisition. It deletes at the last revision that landed.
+func (l *Lease) Unlock(ctx context.Context, held Token) {
+	l.mu.Lock()
+	a := l.cur
+	if a == nil || held.Holder != l.holder || held.Generation != a.gen {
+		l.mu.Unlock()
+		return
+	}
+	l.cur = nil
+	close(a.lost)
+	l.mu.Unlock()
+
+	// Waited for so that no renewal lands after the revision is read; a renewal
+	// in flight is abandoned rather than waited out.
+	<-a.done
+	l.mu.Lock()
+	rev := a.rev
+	l.mu.Unlock()
+
+	releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), releaseTimeout)
+	defer cancel()
+	if err := l.store.CompareAndDelete(releaseCtx, l.key, rev); err != nil {
+		slog.WarnContext(releaseCtx, "kvsync: lease release did not delete the key",
+			"key", l.key, "holder", l.holder, "generation", a.gen, "err", err)
+	}
+}
+
+// Lost returns a channel closed when the current acquisition ends, by loss or
+// Unlock. Each acquisition has its own; before the first it is already closed.
+func (l *Lease) Lost() <-chan struct{} {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return l.lost
+}
+
+// Held reports whether this Lease is currently held in this process.
+func (l *Lease) Held() bool {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return l.cur != nil
+}
+
+// LeaseGroup hands out per-resource leases under one registered prefix, one
+// Lease per name for as long as the owner keeps it.
+type LeaseGroup struct {
+	store  KVStore
+	prefix string
+	holder string
+	ttl    time.Duration
+	clock  clock
+
+	// mu guards leases.
+	mu     sync.Mutex
+	leases map[string]*Lease
+}
+
+// NewLeaseGroup registers prefix, which ends in ".", against store. It panics
+// on an invalid or overlapping prefix, an empty holder or a non-positive ttl.
+func NewLeaseGroup(store KVStore, prefix, holder string, ttl time.Duration) *LeaseGroup {
+	mustWire(errors.Join(validHolder(holder), validTTL(ttl), registerPrefix(store, prefix)))
+	return &LeaseGroup{store: store, prefix: prefix, holder: holder, ttl: ttl, clock: realClock{}, leases: map[string]*Lease{}}
+}
+
+// Get returns the Lease for prefix+name, the same one until Forget.
+func (g *LeaseGroup) Get(name string) *Lease {
+	mustWire(validName("name", name))
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	if l, ok := g.leases[name]; ok {
+		return l
+	}
+	l := newLease(g.store, g.prefix+name, g.holder, g.ttl, g.clock)
+	g.leases[name] = l
+	return l
+}
+
+// Forget unlocks the name's current acquisition, if any, and drops the name,
+// so a resource recreated under it starts from a fresh Lease.
+func (g *LeaseGroup) Forget(name string) {
+	g.mu.Lock()
+	l := g.leases[name]
+	delete(g.leases, name)
+	g.mu.Unlock()
+	if l == nil {
+		return
+	}
+	l.mu.Lock()
+	a := l.cur
+	l.mu.Unlock()
+	if a != nil {
+		l.Unlock(context.Background(), Token{Holder: l.holder, Generation: a.gen})
+	}
+}

spinifex/foundation/state/kvsync/lease_test.go

@@ -0,0 +1,443 @@
+package kvsync_test
+
+import (
+	"context"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const leaseKey = "test.lease"
+
+// stopRenewal unlocks a lease whose clock was frozen to stand in for a dead
+// process, so its renewal goroutine ends with the test.
+func stopRenewal(t *testing.T, l *kvsync.Lease, tok kvsync.Token) {
+	t.Cleanup(func() { l.Unlock(context.Background(), tok) })
+}
+
+func TestLease_TakeOverOnlyAfterAFullTTLOfTheLastRevision(t *testing.T) {
+	s := stores(t, 2)
+	ca, cb := kvsync.NewFakeClock(), kvsync.NewFakeClock()
+	a := kvsync.NewLeaseWithClock(s[0], leaseKey, "node-a", testTTL, ca)
+	b := kvsync.NewLeaseWithClock(s[1], leaseKey, "node-b", testTTL, cb)
+
+	tokA, err := a.TryLock(t.Context())
+	require.NoError(t, err)
+	stopRenewal(t, a, tokA)
+
+	// Several TTLs pass on both clocks while a renews, and b never gets it.
+	last := revision(t, s[0], leaseKey)
+	for range 9 {
+		renewOnce(t, ca)
+		rev := revision(t, s[0], leaseKey)
+		require.Greater(t, rev, last, "a renewal must rewrite the key")
+		last = rev
+		cb.Advance(testTTL / 3)
+		_, err := b.TryLock(t.Context())
+		_, held := asHeld(err)
+		require.True(t, held, "b took a lease that is being renewed: %v", err)
+	}
+
+	// a stops renewing without unlocking; b has seen its last revision once.
+	cb.Advance(testTTL - time.Nanosecond)
+	_, err = b.TryLock(t.Context())
+	_, held := asHeld(err)
+	require.True(t, held, "b took over before a full TTL of the last revision: %v", err)
+
+	cb.Advance(time.Nanosecond)
+	tokB, err := b.TryLock(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, "node-b", tokB.Holder)
+	assert.Greater(t, tokB.Generation, tokA.Generation)
+	b.Unlock(t.Context(), tokB)
+}
+
+func TestLease_LateStarterWaitsAFullTTLFromItsOwnObservation(t *testing.T) {
+	s := stores(t, 2)
+	ca, cb := kvsync.NewFakeClock(), kvsync.NewFakeClock()
+	a := kvsync.NewLeaseWithClock(s[0], leaseKey, "node-a", testTTL, ca)
+	b := kvsync.NewLeaseWithClock(s[1], leaseKey, "node-b", testTTL, cb)
+
+	tokA, err := a.TryLock(t.Context())
+	require.NoError(t, err)
+	stopRenewal(t, a, tokA)
+	renewOnce(t, ca)
+
+	// b starts long after a's last renewal, and has no earlier observation.
+	cb.Advance(10 * testTTL)
+	_, err = b.TryLock(t.Context())
+	_, held := asHeld(err)
+	require.True(t, held, "b's first look must start its clock, not take over: %v", err)
+
+	cb.Advance(testTTL - time.Nanosecond)
+	_, err = b.TryLock(t.Context())
+	_, held = asHeld(err)
+	require.True(t, held, "b took over before a full TTL of its own observation: %v", err)
+
+	cb.Advance(time.Nanosecond)
+	tokB, err := b.TryLock(t.Context())
+	require.NoError(t, err)
+	b.Unlock(t.Context(), tokB)
+}
+
+func TestLease_RestartedHolderAndFollowerRaceOnOneRevision(t *testing.T) {
+	s := stores(t, 3)
+	// Each contender's write waits until both have read, so both CAS at one revision.
+	gate := newBarrier(waitCtx(t), 2)
+	cDead, cNew, cFollow := kvsync.NewFakeClock(), kvsync.NewFakeClock(), kvsync.NewFakeClock()
+	dead := kvsync.NewLeaseWithClock(s[0], leaseKey, "node-a", testTTL, cDead)
+	restarted := kvsync.NewLeaseWithClock(&gatedStore{KVStore: s[1], gate: gate}, leaseKey, "node-a", testTTL, cNew)
+	follower := kvsync.NewLeaseWithClock(&gatedStore{KVStore: s[2], gate: gate}, leaseKey, "node-b", testTTL, cFollow)
+
+	old, err := dead.TryLock(t.Context())
+	require.NoError(t, err)
+	stopRenewal(t, dead, old)
+	rev := revision(t, s[0], leaseKey)
+
+	// The follower's first look only observes, so it never reaches the gate.
+	_, err = follower.TryLock(t.Context())
+	_, held := asHeld(err)
+	require.True(t, held)
+	cFollow.Advance(testTTL)
+
+	type result struct {
+		tok kvsync.Token
+		err error
+	}
+	results := make([]result, 2)
+	var wg sync.WaitGroup
+	for i, l := range []*kvsync.Lease{restarted, follower} {
+		wg.Go(func() {
+			tok, err := l.TryLock(t.Context())
+			results[i] = result{tok, err}
+		})
+	}
+	wg.Wait()
+
+	var winners int
+	for _, r := range results {
+		if r.err == nil {
+			winners++
+			assert.Greater(t, r.tok.Generation, rev)
+			continue
+		}
+		_, held := asHeld(r.err)
+		assert.True(t, held, "the loser must get HeldError, got %v", r.err)
+	}
+	require.Equal(t, 1, winners, "exactly one CAS at the shared revision wins")
+	for i, l := range []*kvsync.Lease{restarted, follower} {
+		if results[i].err == nil {
+			l.Unlock(t.Context(), results[i].tok)
+		}
+	}
+}
+
+func TestLease_ExcludesCallSitesInOneProcess(t *testing.T) {
+	s := stores(t, 1)
+	l := kvsync.NewLeaseWithClock(s[0], leaseKey, "node-a", testTTL, kvsync.NewFakeClock())
+
+	siteA, err := l.TryLock(t.Context())
+	require.NoError(t, err)
+
+	_, err = l.TryLock(t.Context())
+	he, held := asHeld(err)
+	require.True(t, held, "a second site took a held Lease: %v", err)
+	assert.Equal(t, siteA, he.Current, "the refusal names this holder's acquisition")
+
+	l.Unlock(t.Context(), siteA)
+	siteB, err := l.TryLock(t.Context())
+	require.NoError(t, err)
+	assert.Greater(t, siteB.Generation, siteA.Generation)
+	l.Unlock(t.Context(), siteB)
+}
+
+// takeKey overwrites the key from another store, standing in for a holder
+// that took the lease over.
+func takeKey(t *testing.T, other kvsync.KVStore) {
+	t.Helper()
+	_, rev, err := other.Load(t.Context(), leaseKey)
+	require.NoError(t, err)
+	_, err = other.CompareAndSwap(t.Context(), leaseKey, []byte(`{"holder":"node-b"}`), rev)
+	require.NoError(t, err)
+}
+
+func TestLease_StaleUnlockLeavesTheLaterAcquisition(t *testing.T) {
+	s := stores(t, 2)
+	c := kvsync.NewFakeClock()
+	l := kvsync.NewLeaseWithClock(s[0], leaseKey, "node-a", testTTL, c)
+
+	siteA, err := l.TryLock(t.Context())
+	require.NoError(t, err)
+	lostA := l.Lost()
+	takeKey(t, s[1])
+	waitForWaiters(t, c, 1)
+	c.Advance(testTTL / 3)
+	requireClosed(t, lostA, "site A's acquisition must be lost")
+
+	// The key names another holder, so site B waits out a TTL before taking it.
+	_, err = l.TryLock(t.Context())
+	_, held := asHeld(err)
+	require.True(t, held)
+	c.Advance(testTTL)
+	siteB, err := l.TryLock(t.Context())
+	require.NoError(t, err)
+
+	l.Unlock(t.Context(), siteA)
+	assert.True(t, l.Held(), "site A's stale unlock released site B's acquisition")
+	assert.Equal(t, siteB.Generation, revision(t, s[0], leaseKey), "site A's stale unlock deleted site B's key")
+	l.Unlock(t.Context(), siteB)
+}
+
+// A Lease is reusable after Unlock and after a renewal loses the key, each
+// acquisition with a fresh Lost and a higher generation.
+func TestLease_MismatchClosesLostOnceAndStopsRenewing(t *testing.T) {
+	s := stores(t, 2)
+	f := &faultyStore{KVStore: s[0]}
+	c := kvsync.NewFakeClock()
+	l := kvsync.NewLeaseWithClock(f, leaseKey, "node-a", testTTL, c)
+
+	first, err := l.TryLock(t.Context())
+	require.NoError(t, err)
+	firstLost := l.Lost()
+	l.Unlock(t.Context(), first)
+	requireClosed(t, firstLost, "Unlock must close the acquisition's Lost")
+
+	second, err := l.TryLock(t.Context())
+	require.NoError(t, err)
+	lost := l.Lost()
+	requireOpen(t, lost, "a new acquisition needs a fresh Lost channel")
+	takeKey(t, s[1])
+	waitForWaiters(t, c, 1)
+	c.Advance(testTTL / 3)
+	requireClosed(t, lost, "a renewal that lost its CAS must close Lost")
+	require.False(t, l.Held())
+
+	swaps := f.swaps.Load()
+	c.Advance(10 * testTTL)
+	assert.Zero(t, c.Waiters(), "a lost lease must not schedule another renewal")
+	assert.Equal(t, swaps, f.swaps.Load(), "a lost lease must not renew again")
+
+	// With the taker's key deleted the key is free, so the lease takes it at once.
+	require.NoError(t, s[1].CompareAndDelete(t.Context(), leaseKey, revision(t, s[1], leaseKey)))
+	third, err := l.TryLock(t.Context())
+	require.NoError(t, err)
+	requireOpen(t, l.Lost(), "a new acquisition needs a fresh Lost channel")
+	assert.Greater(t, third.Generation, second.Generation)
+	l.Unlock(t.Context(), third)
+}
+
+// failRenewalsUntilDeadline drives a lease whose last renewal landed now
+// through failing renewals up to 2*ttl/3, checking Lost stays open before it.
+// The first attempt comes at ttl/3 and each retry ttl/12 after the last.
+func failRenewalsUntilDeadline(t *testing.T, c *kvsync.FakeClock, lost <-chan struct{}) {
+	t.Helper()
+	step := testTTL / 12
+	waitForWaiters(t, c, 1)
+	c.Advance(testTTL / 3)
+	for elapsed := testTTL / 3; elapsed < 2*testTTL/3; elapsed += step {
+		waitForWaiters(t, c, 1)
+		requireOpen(t, lost, "Lost closed before 2*ttl/3 without a landed renewal")
+		c.Advance(step)
+	}
+}
+
+func TestLease_UnavailableRenewalsLoseAtTwoThirdsOfTheTTL(t *testing.T) {
+	s := stores(t, 1)
+	f := &faultyStore{KVStore: s[0]}
+	c := kvsync.NewFakeClock()
+	l := kvsync.NewLeaseWithClock(f, leaseKey, "node-a", testTTL, c)
+
+	gone, err := l.TryLock(t.Context())
+	require.NoError(t, err)
+	lost := l.Lost()
+	renewOnce(t, c) // lands, so the deadline counts from here
+
+	f.down.Store(true)
+	before := f.swaps.Load()
+	failRenewalsUntilDeadline(t, c, lost)
+	requireClosed(t, lost, "Lost must close once 2*ttl/3 has passed since the last landed renewal")
+	assert.False(t, l.Held())
+	assert.Greater(t, f.swaps.Load()-before, int64(1), "an unavailable renewal must be retried before the lease is given up")
+	f.down.Store(false)
+
+	readopted, err := l.TryLock(t.Context())
+	require.NoError(t, err, "the surviving key names this holder, so it is adopted at once")
+	assert.Greater(t, readopted.Generation, gone.Generation, "an adoption takes the generation of its own write")
+	l.Unlock(t.Context(), gone)
+	assert.Equal(t, readopted.Generation, revision(t, f, leaseKey), "the lost acquisition's unlock deleted the readopted key")
+	l.Unlock(t.Context(), readopted)
+}
+
+func TestLeaseGroup_SameLeaseUntilForget(t *testing.T) {
+	s := stores(t, 1)
+	g := kvsync.NewLeaseGroup(s[0], "eks.cluster.", "node-a", testTTL)
+	kvsync.SetGroupClock(g, kvsync.NewFakeClock())
+
+	l := g.Get("123/alpha")
+	require.Same(t, l, g.Get("123/alpha"), "Get must return the one Lease a name has")
+	assert.Equal(t, "eks.cluster.123/alpha", l.Key())
+
+	tok, err := l.TryLock(t.Context())
+	require.NoError(t, err)
+	g.Forget("123/alpha")
+	assert.False(t, l.Held(), "Forget unlocks the current acquisition")
+	assert.Zero(t, revision(t, s[0], "eks.cluster.123/alpha"), "Forget deletes the key")
+
+	fresh := g.Get("123/alpha")
+	require.NotSame(t, l, fresh, "a forgotten name starts from a fresh Lease")
+	again, err := fresh.TryLock(t.Context())
+	require.NoError(t, err)
+	assert.Greater(t, again.Generation, tok.Generation)
+	g.Forget("123/alpha")
+}
+
+// A grant whose reply comes back after a peer has already waited out a TTL and
+// taken over must not be reported as held: only one node may think it leads.
+func TestLease_DelayedGrantIsRefusedAfterATakeover(t *testing.T) {
+	s := stores(t, 2)
+	ca, cb := kvsync.NewFakeClock(), kvsync.NewFakeClock()
+	slow := &slowStore{KVStore: s[0]}
+	a := kvsync.NewLeaseWithClock(slow, leaseKey, "node-a", testTTL, ca)
+	b := kvsync.NewLeaseWithClock(s[1], leaseKey, "node-b", testTTL, cb)
+
+	var tokB kvsync.Token
+	slow.afterSwap = func(int64) {
+		_, err := b.TryLock(t.Context())
+		_, held := asHeld(err)
+		require.True(t, held, "b must first observe a's committed write: %v", err)
+		cb.Advance(testTTL)
+		tokB, err = b.TryLock(t.Context())
+		require.NoError(t, err, "b takes over after a full TTL of a's revision")
+		ca.Advance(testTTL)
+	}
+
+	_, err := a.TryLock(t.Context())
+	require.ErrorIs(t, err, kvsync.ErrUnavailable)
+	assert.False(t, a.Held(), "a must not lead on a grant a peer has since taken over")
+	assert.True(t, b.Held())
+	b.Unlock(t.Context(), tokB)
+}
+
+// A renewal counts from when it was sent, since a peer can have seen the
+// revision that early, so the next renewal is due ttl/3 after that send.
+func TestLease_DelayedRenewalCountsFromItsSend(t *testing.T) {
+	s := stores(t, 1)
+	c := kvsync.NewFakeClock()
+	slow := &slowStore{KVStore: s[0]}
+	a := kvsync.NewLeaseWithClock(slow, leaseKey, "node-a", testTTL, c)
+	tok, err := a.TryLock(t.Context())
+	require.NoError(t, err)
+	stopRenewal(t, a, tok)
+
+	slow.afterSwap = func(n int64) {
+		if n == 2 {
+			c.Advance(testTTL / 4)
+		}
+	}
+	renewOnce(t, c)
+	c.Advance(testTTL/3 - testTTL/4)
+	waitForSwaps(t, slow, 3)
+	requireOpen(t, a.Lost(), "a renewal that landed in time keeps the lease")
+}
+
+// A grant that took most of its validity to answer is renewed at once,
+// rather than ttl/3 after install, when its validity would already be gone.
+func TestLease_SlowGrantIsRenewedFromItsSend(t *testing.T) {
+	s := stores(t, 1)
+	c := kvsync.NewFakeClock()
+	slow := &slowStore{KVStore: s[0]}
+	slow.afterSwap = func(n int64) {
+		if n == 1 {
+			c.Advance(testTTL / 2)
+		}
+	}
+	a := kvsync.NewLeaseWithClock(slow, leaseKey, "node-a", testTTL, c)
+	tok, err := a.TryLock(t.Context())
+	require.NoError(t, err, "a grant inside its validity is installed")
+	stopRenewal(t, a, tok)
+
+	waitForSwaps(t, slow, 2)
+	requireOpen(t, a.Lost(), "the prompt renewal keeps the lease")
+}
+
+// A renewal stuck where its context cannot reach still loses the lease at the
+// deadline, before a peer that waited a TTL can take over.
+func TestLease_StuckRenewalLosesAtTheDeadline(t *testing.T) {
+	s := stores(t, 1)
+	c := kvsync.NewFakeClock()
+	slow := &slowStore{KVStore: s[0]}
+	a := kvsync.NewLeaseWithClock(slow, leaseKey, "node-a", testTTL, c)
+	tok, err := a.TryLock(t.Context())
+	require.NoError(t, err)
+	stopRenewal(t, a, tok)
+
+	stuck := make(chan struct{})
+	t.Cleanup(func() { close(stuck) })
+	slow.afterSwap = func(n int64) {
+		if n == 2 {
+			<-stuck
+		}
+	}
+	waitForWaiters(t, c, 1)
+	c.Advance(testTTL / 3)
+	require.True(t, c.WaitForDeadlines(waitCtx(t), 1), "the renewal must arm its deadline")
+	c.Advance(testTTL / 3)
+	requireClosed(t, a.Lost(), "the lease must be lost at 2*ttl/3 after the last send, stuck CAS or not")
+	assert.False(t, a.Held())
+}
+
+// Unlock does not wait out a renewal stuck in flight: it deletes the key at the
+// last revision that landed, and the stuck write cannot bring the key back.
+func TestLease_UnlockAbandonsAStuckRenewal(t *testing.T) {
+	s := stores(t, 1)
+	c := kvsync.NewFakeClock()
+	slow := &slowStore{KVStore: s[0]}
+	a := kvsync.NewLeaseWithClock(slow, leaseKey, "node-a", testTTL, c)
+	tok, err := a.TryLock(t.Context())
+	require.NoError(t, err)
+
+	entered, stuck := make(chan struct{}), make(chan struct{})
+	slow.beforeSwap = func(n int64) {
+		if n == 2 {
+			close(entered)
+			<-stuck
+		}
+	}
+	waitForWaiters(t, c, 1)
+	c.Advance(testTTL / 3)
+	select {
+	case <-entered:
+	case <-waitCtx(t).Done():
+		t.Fatal("the renewal never started")
+	}
+
+	unlocked := make(chan struct{})
+	go func() { defer close(unlocked); a.Unlock(t.Context(), tok) }()
+	select {
+	case <-unlocked:
+	case <-waitCtx(t).Done():
+		close(stuck)
+		t.Fatal("Unlock waited for a renewal stuck in flight")
+	}
+	_, _, err = s[0].Load(t.Context(), leaseKey)
+	require.ErrorIs(t, err, kvsync.ErrNotFound, "Unlock must delete the key at the last landed revision")
+
+	close(stuck)
+	waitForSwaps(t, slow, 2)
+	_, _, err = s[0].Load(t.Context(), leaseKey)
+	require.ErrorIs(t, err, kvsync.ErrNotFound, "the abandoned renewal must not recreate the key")
+}
+
+func TestLease_ConstructionRefusesAnEmptyHolderOrANonPositiveTTL(t *testing.T) {
+	s := stores(t, 1)
+	assert.PanicsWithValue(t, "kvsync: empty holder", func() { kvsync.NewLease(s[0], "lease.noholder", "", testTTL) })
+	assert.PanicsWithValue(t, "kvsync: lease ttl must be positive", func() { kvsync.NewLease(s[0], "lease.zerottl", "node-a", 0) })
+	assert.PanicsWithValue(t, "kvsync: lease ttl must be positive", func() { kvsync.NewLease(s[0], "lease.negttl", "node-a", -time.Second) })
+	assert.PanicsWithValue(t, "kvsync: empty holder", func() { kvsync.NewLeaseGroup(s[0], "group.noholder.", "", testTTL) })
+	assert.PanicsWithValue(t, "kvsync: lease ttl must be positive", func() { kvsync.NewLeaseGroup(s[0], "group.zerottl.", "node-a", 0) })
+	assert.PanicsWithValue(t, "kvsync: lease ttl must be positive", func() { kvsync.NewLeaseGroup(s[0], "group.negttl.", "node-a", -time.Second) })
+}

spinifex/foundation/state/kvsync/registry.go

@@ -0,0 +1,148 @@
+package kvsync
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+)
+
+// registries holds, per store value, every key and prefix constructed against
+// it, so two owners in one process can never share a key by accident.
+var registries = struct {
+	mu sync.Mutex
+	m  map[KVStore]*keySet
+}{m: map[KVStore]*keySet{}}
+
+type keySet struct {
+	keys     map[string]struct{}
+	prefixes map[string]struct{}
+}
+
+// registerKey records key against store unless it is invalid, already
+// registered, or under a registered prefix.
+func registerKey(store KVStore, key string) error {
+	if err := validKey(key); err != nil {
+		return err
+	}
+	set, err := keySetFor(store)
+	if err != nil {
+		return err
+	}
+	defer registries.mu.Unlock()
+	if _, dup := set.keys[key]; dup {
+		return fmt.Errorf("kvsync: key %q constructed twice against one store", key)
+	}
+	for p := range set.prefixes {
+		if strings.HasPrefix(key, p) {
+			return fmt.Errorf("kvsync: key %q is under registered prefix %q", key, p)
+		}
+	}
+	set.keys[key] = struct{}{}
+	return nil
+}
+
+// registerPrefix records prefix against store unless it is invalid, overlaps a
+// registered prefix, or covers a registered key.
+func registerPrefix(store KVStore, prefix string) error {
+	if err := validPrefix(prefix); err != nil {
+		return err
+	}
+	set, err := keySetFor(store)
+	if err != nil {
+		return err
+	}
+	defer registries.mu.Unlock()
+	for p := range set.prefixes {
+		if strings.HasPrefix(prefix, p) || strings.HasPrefix(p, prefix) {
+			return fmt.Errorf("kvsync: prefix %q overlaps registered prefix %q", prefix, p)
+		}
+	}
+	for k := range set.keys {
+		if strings.HasPrefix(k, prefix) {
+			return fmt.Errorf("kvsync: prefix %q covers registered key %q", prefix, k)
+		}
+	}
+	set.prefixes[prefix] = struct{}{}
+	return nil
+}
+
+// keySetFor returns store's key set with registries.mu held; the caller unlocks.
+func keySetFor(store KVStore) (*keySet, error) {
+	if store == nil {
+		return nil, errors.New("kvsync: nil store")
+	}
+	registries.mu.Lock()
+	set, ok := registries.m[store]
+	if !ok {
+		set = &keySet{keys: map[string]struct{}{}, prefixes: map[string]struct{}{}}
+		registries.m[store] = set
+	}
+	return set, nil
+}
+
+// mustWire panics on a construction error, the one place this package panics.
+func mustWire(err error) {
+	if err != nil {
+		panic(err.Error()) //nolint:forbidigo // Constructing a lock wrongly is a wiring error that must fail at startup, not at the first election.
+	}
+}
+
+func validHolder(holder string) error {
+	if holder == "" {
+		return errors.New("kvsync: empty holder")
+	}
+	return nil
+}
+
+func validTTL(ttl time.Duration) error {
+	if ttl <= 0 {
+		return errors.New("kvsync: lease ttl must be positive")
+	}
+	return nil
+}
+
+// validChars reports whether s holds only characters JetStream KV accepts in a key.
+func validChars(s string) bool {
+	for _, c := range s {
+		switch {
+		case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9':
+		case c == '-', c == '/', c == '_', c == '=', c == '.':
+		default:
+			return false
+		}
+	}
+	return true
+}
+
+// validName checks the rules every key and group name shares: non-empty,
+// characters JetStream accepts, and no empty dot-separated token.
+func validName(kind, s string) error {
+	switch {
+	case s == "":
+		return fmt.Errorf("kvsync: empty %s", kind)
+	case !validChars(s):
+		return fmt.Errorf("kvsync: %s %q has a character JetStream KV refuses", kind, s)
+	case strings.HasPrefix(s, "."), strings.HasSuffix(s, "."), strings.Contains(s, ".."):
+		return fmt.Errorf("kvsync: %s %q has an empty token", kind, s)
+	}
+	return nil
+}
+
+func validKey(key string) error {
+	if err := validName("key", key); err != nil {
+		return err
+	}
+	if strings.HasPrefix(key, "_") {
+		return fmt.Errorf("kvsync: key %q starts with _, which the bucket's schema version key reserves", key)
+	}
+	return nil
+}
+
+func validPrefix(prefix string) error {
+	if !strings.HasSuffix(prefix, ".") {
+		return fmt.Errorf("kvsync: prefix %q must end in a dot", prefix)
+	}
+	return validKey(strings.TrimSuffix(prefix, "."))
+}

spinifex/foundation/state/kvsync/registry_test.go

@@ -0,0 +1,88 @@
+package kvsync_test
+
+import (
+	"context"
+	"go/build"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/stretchr/testify/assert"
+)
+
+// nopStore is a store value for registry tests, which never touch the store.
+type nopStore struct{ _ byte }
+
+func (*nopStore) Load(context.Context, string) ([]byte, uint64, error) {
+	return nil, 0, kvsync.ErrNotFound
+}
+func (*nopStore) CompareAndSwap(context.Context, string, []byte, uint64) (uint64, error) {
+	return 0, kvsync.ErrUnavailable
+}
+func (*nopStore) CompareAndDelete(context.Context, string, uint64) error {
+	return kvsync.ErrUnavailable
+}
+
+var _ kvsync.KVStore = (*nopStore)(nil)
+
+func TestRegistry_PanicsOnACollisionWithinOneStore(t *testing.T) {
+	tests := []struct {
+		name   string
+		first  func(kvsync.KVStore)
+		second func(kvsync.KVStore)
+	}{
+		{"duplicate key",
+			func(s kvsync.KVStore) { kvsync.NewLease(s, "vpcd.reconcile", "h", time.Minute) },
+			func(s kvsync.KVStore) { kvsync.NewClaim(s, "vpcd.reconcile", "h") }},
+		{"key under a registered prefix",
+			func(s kvsync.KVStore) { kvsync.NewLeaseGroup(s, "eks.cluster.", "h", time.Minute) },
+			func(s kvsync.KVStore) { kvsync.NewLease(s, "eks.cluster.x", "h", time.Minute) }},
+		{"prefix over a registered key",
+			func(s kvsync.KVStore) { kvsync.NewLease(s, "eks.cluster.x", "h", time.Minute) },
+			func(s kvsync.KVStore) { kvsync.NewClaimGroup(s, "eks.", "h") }},
+		{"prefix inside a registered prefix",
+			func(s kvsync.KVStore) { kvsync.NewLeaseGroup(s, "eks.", "h", time.Minute) },
+			func(s kvsync.KVStore) { kvsync.NewClaimGroup(s, "eks.cluster.", "h") }},
+		{"prefix containing a registered prefix",
+			func(s kvsync.KVStore) { kvsync.NewClaimGroup(s, "eks.cluster.", "h") },
+			func(s kvsync.KVStore) { kvsync.NewLeaseGroup(s, "eks.", "h", time.Minute) }},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			one, two := &nopStore{}, &nopStore{}
+			tt.first(one)
+			assert.Panics(t, func() { tt.second(one) }, "against one store")
+			assert.NotPanics(t, func() { tt.second(two) }, "against another store")
+		})
+	}
+}
+
+func TestRegistry_PanicsOnAKeyJetStreamRefuses(t *testing.T) {
+	for _, key := range []string{"", "has space", "star.*", "gt.>", ".lead", "trail.", "two..dots", "_version", "_reserved"} {
+		t.Run(key, func(t *testing.T) {
+			assert.Panics(t, func() { kvsync.NewClaim(&nopStore{}, key, "h") })
+		})
+	}
+	for _, prefix := range []string{"noDot", "_v.", ".x.", "a b."} {
+		t.Run("prefix "+prefix, func(t *testing.T) {
+			assert.Panics(t, func() { kvsync.NewLeaseGroup(&nopStore{}, prefix, "h", time.Minute) })
+		})
+	}
+	g := kvsync.NewLeaseGroup(&nopStore{}, "ok.", "h", time.Minute)
+	assert.Panics(t, func() { g.Get("bad name") })
+	assert.NotPanics(t, func() { g.Get("123456789012/my-cluster/teardown") })
+}
+
+func TestPackage_ImportsOnlyTheStandardLibrary(t *testing.T) {
+	pkg, err := build.ImportDir(".", 0)
+	if err != nil {
+		t.Fatalf("read package: %v", err)
+	}
+	for _, imp := range pkg.Imports {
+		// A standard library path has no dot in its first element.
+		if first, _, _ := strings.Cut(imp, "/"); strings.Contains(first, ".") {
+			t.Errorf("kvsync imports %s; it must depend on nothing but the standard library", imp)
+		}
+	}
+}

spinifex/runtime/daemon/daemon.go

@@ -156,6 +156,7 @@ type Daemon struct {
 	instanceObjects   objectstore.ObjectStore
 	dnsWriter         *dns.Writer
 	dnsReconciler     *dns.Reconciler
+	locks             *daemonLocks
 	dnsBaseDomain     string
 	dnsInternalDomain string
 	keyService        *ec2key.KeyServiceImpl
@@ -2024,12 +2025,12 @@ func (d *Daemon) startCluster() error {
 		return fmt.Errorf("EBS provider lock: %w", err)
 	}
 
-	// Remove the obsolete spinifex-dhcp-leases bucket (idempotent).
+	syncJS, err := jetstream.New(d.natsConn)
-	if js, jsErr := jetstream.New(d.natsConn); jsErr == nil {
+	if err != nil {
-		if err := kvstore.DeleteBucketIfExists(d.ctx, js, "spinifex-dhcp-leases"); err != nil {
+		return fmt.Errorf("initialize JetStream for leases: %w", err)
-			slog.Warn("Failed to delete obsolete spinifex-dhcp-leases KV bucket", "err", err)
-		}
 	}
+	deleteObsoleteBuckets(d.ctx, syncJS)
+	d.locks = newDaemonLocks(kvstore.OpenSyncBucket(syncJS), d.node)
 
 	// Reconcile OVN native IPsec both ways (idempotent): a node that does not
 	// use it should not be left running charon on 500/4500 either. Runs in the
@@ -2062,7 +2063,7 @@ func (d *Daemon) startCluster() error {
 	d.instanceObjects = store
 	d.instanceService = ec2instance.NewInstanceServiceImpl(d.config, d.resourceMgr.instanceTypes, d.natsConn, store, d.vmMgr, d.resourceMgr, d.jsManager)
 	d.dnsWriter = dns.NewWriter(d.config, d.clusterConfig, d.natsConn)
-	d.dnsReconciler = dns.NewReconciler(d.config, d.clusterConfig, d.natsConn, d.dnsWriter, d.dnsDesiredSet, d.dnsWatchSources()...)
+	d.dnsReconciler = dns.NewReconciler(d.config, d.clusterConfig, d.locks.dns, d.dnsWriter, d.dnsDesiredSet, d.dnsWatchSources()...)
 	d.dnsBaseDomain = dns.ResolveBaseDomain(d.config)
 	d.dnsInternalDomain = dns.ResolveInternalDomain(d.config)
 	d.keyService = ec2key.NewKeyServiceImpl(d.config)
@@ -2333,7 +2334,7 @@ func (d *Daemon) startCluster() error {
 	}
 
 	d.elbv2Service.StartLifecycleReaper(context.Background())
-	d.elbv2Service.StartDataPathReconciler(d.ctx, d.node)
+	d.elbv2Service.StartDataPathReconciler(d.ctx, d.locks.elbv2)
 
 	d.eksService, err = initServiceWithRetry("EKS service", func() (*ekscluster.EKSServiceImpl, error) {
 		return ekscluster.NewEKSServiceImpl(d.buildEKSServiceDeps())
@@ -2347,21 +2348,15 @@ func (d *Daemon) startCluster() error {
 	// reaper. The scheduler is disabled (handlers still serve) when JetStream is
 	// unavailable.
 	d.ecsService = ecsdomain.NewService(d.natsConn, d.config.Region, d.clusterConfig.AWS.ServicesDomain).WithDeps(d.buildECSServiceDeps())
-	if js, jsErr := jetstream.New(d.natsConn); jsErr != nil {
+	d.ecsScheduler = ecsdomain.NewScheduler(d.natsConn, d.ecsService, d.locks.ecs)
-		slog.Warn("ECS scheduler disabled: JetStream unavailable", "err", jsErr)
+	d.shutdownWg.Go(func() {
-	} else if _, lbErr := ecsdomain.InitLeaderBucket(d.ctx, js); lbErr != nil {
+		defer func() {
-		slog.Warn("ECS scheduler disabled: leader bucket init failed", "err", lbErr)
+			if r := recover(); r != nil {
-	} else {
+				slog.Error("ECS scheduler goroutine panicked", "recover", r)
-		d.ecsScheduler = ecsdomain.NewScheduler(d.natsConn, d.ecsService, d.node)
+			}
-		d.shutdownWg.Go(func() {
+		}()
-			defer func() {
+		d.ecsScheduler.Run(d.ctx)
-				if r := recover(); r != nil {
+	})
-					slog.Error("ECS scheduler goroutine panicked", "recover", r)
-				}
-			}()
-			d.ecsScheduler.Run(d.ctx)
-		})
-	}
 
 	// RDS control plane: KV-backed agent-protocol handlers plus the customer
 	// actions. The cluster CA signs the per-instance serving certs, minted per
@@ -2387,7 +2382,7 @@ func (d *Daemon) startCluster() error {
 
 	// One leader across the cluster derives DB instance status from the agent
 	// heartbeat; every node keeps serving the API.
-	d.rdsReconciler = rdsinstance.NewReconciler(d.rdsService, d.node)
+	d.rdsReconciler = rdsinstance.NewReconciler(d.rdsService, d.locks.rds)
 	d.shutdownWg.Go(func() {
 		defer func() {
 			if r := recover(); r != nil {
@@ -2405,7 +2400,7 @@ func (d *Daemon) startCluster() error {
 	// One leader across the cluster scrapes each serving endpoint's vLLM metrics
 	// and hands an idle GPU back; every node keeps serving the API. Without it a
 	// launched model owns its device until an operator deletes the endpoint.
-	d.bedrockReaper = ochre.NewReaper(d.bedrockService, d.node, ochre.ReaperDeps{})
+	d.bedrockReaper = ochre.NewReaper(d.bedrockService, d.locks.ochre, ochre.ReaperDeps{})
 	d.shutdownWg.Go(func() {
 		defer func() {
 			if r := recover(); r != nil {

spinifex/runtime/daemon/daemon_handlers_eks_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit"
 	ekscluster "github.com/mulgadc/spinifex/spinifex/domains/eks/cluster"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -19,8 +20,8 @@ import (
 // the rest of the Daemon graph can stay nil.
 func setupEKSDaemon(t *testing.T) (*Daemon, *nats.Conn) {
 	t.Helper()
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, nc, js := testutil.StartTestJetStream(t)
-	svc, err := ekscluster.NewEKSServiceImpl(ekscluster.EKSServiceDeps{NATSConn: nc})
+	svc, err := ekscluster.NewEKSServiceImpl(ekscluster.EKSServiceDeps{NATSConn: nc, Leases: ekscluster.NewClusterLeases(kvstore.OpenSyncBucket(js), "test")})
 	require.NoError(t, err)
 	return &Daemon{eksService: svc}, nc
 }

spinifex/runtime/daemon/dns_reconcile_services_test.go

@@ -10,6 +10,7 @@ import (
 	ekscluster "github.com/mulgadc/spinifex/spinifex/domains/eks/cluster"
 	elbv2domain "github.com/mulgadc/spinifex/spinifex/domains/elbv2"
 	rdsinstance "github.com/mulgadc/spinifex/spinifex/domains/rds/instance"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -33,14 +34,14 @@ func TestDNSDesiredSet_ServicesWithoutBaseDomain(t *testing.T) {
 
 func TestDNSDesiredSet_ServicesEnumerateCompletely(t *testing.T) {
 	t.Parallel()
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, nc, js := testutil.StartTestJetStream(t)
 	cfg := &config.Config{Region: "ap-southeast-2"}
 	cfg.Northstar.DefaultDomain = "example.internal"
 
 	elb, err := elbv2domain.NewELBv2ServiceImplWithNATS(cfg, nc, bytes.Repeat([]byte{0x42}, masterkey.MasterKeySize))
 	require.NoError(t, err)
 	t.Cleanup(elb.Close)
-	eks, err := ekscluster.NewEKSServiceImpl(ekscluster.EKSServiceDeps{NATSConn: nc, Config: cfg})
+	eks, err := ekscluster.NewEKSServiceImpl(ekscluster.EKSServiceDeps{NATSConn: nc, Config: cfg, Leases: ekscluster.NewClusterLeases(kvstore.OpenSyncBucket(js), "test")})
 	require.NoError(t, err)
 	t.Cleanup(eks.Shutdown)
 	rds := rdsinstance.NewService(nc, cfg.Region).WithDeps(rdsinstance.Deps{BaseDomain: "example.internal"})

spinifex/runtime/daemon/eks_deps.go

@@ -78,6 +78,7 @@ func (d *Daemon) buildEKSServiceDeps() ekscluster.EKSServiceDeps {
 		GatewayBaseURL:      d.resolveGatewayBaseURL(),
 		Region:              d.config.Region,
 		HolderID:            d.node,
+		Leases:              d.locks.eks,
 		InternalSuffix:      internalSuffix,
 		RegistryHosts:       d.registryHosts,
 		SystemGatewayURL:    d.resolveSystemGatewayBaseURL(),

spinifex/runtime/daemon/eks_deps_test.go

@@ -149,6 +149,7 @@ func TestResolveSystemPredastoreURL(t *testing.T) {
 func TestBuildEKSServiceDeps(t *testing.T) {
 	t.Parallel()
 	d := &Daemon{
+		locks:        &daemonLocks{},
 		mgmtBridgeIP: "10.15.8.1",
 		config: &config.Config{
 			Predastore: config.PredastoreConfig{
@@ -213,6 +214,7 @@ func TestBuildEKSServiceDeps_ConfigDerivedFields(t *testing.T) {
 
 	base := func() *Daemon {
 		return &Daemon{
+			locks:         &daemonLocks{},
 			mgmtBridgeIP:  "10.15.8.1",
 			clusterConfig: &config.ClusterConfig{AWS: config.AWSConfig{ServicesDomain: "svc.example"}},
 			volumeService: &ec2volume.VolumeServiceImpl{},
@@ -247,6 +249,7 @@ func TestBuildEKSServiceDeps_ConfigDerivedFields(t *testing.T) {
 func TestBuildEKSServiceDeps_WiresLaunchTemplates(t *testing.T) {
 	t.Parallel()
 	d := &Daemon{
+		locks:                 &daemonLocks{},
 		config:                &config.Config{},
 		launchTemplateService: &ec2launchtemplate.LaunchTemplateServiceImpl{},
 	}

spinifex/runtime/daemon/locks.go

@@ -0,0 +1,70 @@
+package daemon
+
+import (
+	"context"
+	"log/slog"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/dns"
+	ecsdomain "github.com/mulgadc/spinifex/spinifex/domains/ecs"
+	ekscluster "github.com/mulgadc/spinifex/spinifex/domains/eks/cluster"
+	elbv2domain "github.com/mulgadc/spinifex/spinifex/domains/elbv2"
+	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
+	"github.com/mulgadc/spinifex/spinifex/domains/ochre"
+	rdsinstance "github.com/mulgadc/spinifex/spinifex/domains/rds/instance"
+	route53domain "github.com/mulgadc/spinifex/spinifex/domains/route53"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// daemonLocks is every lease the daemon owns, each constructed once against the
+// process's sync bucket and handed to the sites that run under it.
+type daemonLocks struct {
+	dns            *kvsync.Lease
+	route53Realize *kvsync.Lease
+	route53Verify  *kvsync.Lease
+	rds            *kvsync.Lease
+	ecs            *kvsync.Lease
+	ochre          *kvsync.Lease
+	elbv2          *kvsync.Lease
+	eks            *kvsync.LeaseGroup
+}
+
+func newDaemonLocks(store kvsync.KVStore, node string) *daemonLocks {
+	return &daemonLocks{
+		dns:            dns.NewReconcileLease(store, node),
+		route53Realize: reconcilelock.NewLeaderLease(store, route53domain.KeyRealize, node),
+		route53Verify:  reconcilelock.NewLeaderLease(store, route53domain.KeyVerify, node),
+		rds:            rdsinstance.NewReconcilerLease(store, node),
+		ecs:            ecsdomain.NewSchedulerLease(store, node),
+		ochre:          ochre.NewReaperLease(store, node),
+		elbv2:          elbv2domain.NewDataPathLease(store, node),
+		eks:            ekscluster.NewClusterLeases(store, node),
+	}
+}
+
+// obsoleteBuckets held only keys nothing reads any more: DHCP leases, and the
+// lease buckets every election ran in before they moved to the sync bucket.
+var obsoleteBuckets = []string{
+	"spinifex-dhcp-leases",
+	"spinifex-vpcd-reconcile",
+	"spinifex-dns-reconcile",
+	"spinifex-route53-realize",
+	"spinifex-route53-verify",
+	"spinifex-quota-reconcile",
+	"spinifex-rds-leader",
+	"spinifex-ecs-leader",
+	"bedrock-leader",
+	"spinifex-eks-leader",
+	"spinifex-elbv2-leader",
+}
+
+// deleteObsoleteBuckets removes each obsolete bucket that exists. A failure is
+// logged and the rest are still tried; the next start retries it.
+func deleteObsoleteBuckets(ctx context.Context, js jetstream.JetStream) {
+	for _, name := range obsoleteBuckets {
+		if err := kvstore.DeleteBucketIfExists(ctx, js, name); err != nil {
+			slog.WarnContext(ctx, "Failed to delete obsolete KV bucket", "bucket", name, "err", err)
+		}
+	}
+}

spinifex/runtime/daemon/locks_test.go

@@ -0,0 +1,66 @@
+package daemon
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/internal/testkit"
+	admissionquota "github.com/mulgadc/spinifex/spinifex/domains/admission/quota"
+	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Every lease owner's key is constructed against one store here, which the
+// registry panics on if two of them collide. vpcd and the gateway run in
+// processes of their own, so theirs are added by hand.
+func TestDaemonLocks_EveryOwnerHasItsOwnKey(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	store := kvstore.OpenSyncBucket(testutil.NewJetStream(t, nc))
+	assert.NotPanics(t, func() {
+		locks := newDaemonLocks(store, "node-1")
+		reconcilelock.NewLeaderLease(store, reconcilelock.KeyVPCDReconcile, "node-1")
+		reconcilelock.NewLeaderLease(store, admissionquota.KeyReconcile, "node-1")
+		locks.eks.Get("123456789012/alpha")
+		locks.eks.Get("123456789012/alpha/teardown")
+	})
+}
+
+// upgradedBuckets are the buckets a cluster upgraded from per-election leases
+// carries, named here rather than read off obsoleteBuckets, so dropping a name
+// from that list fails the test instead of leaving the bucket behind.
+var upgradedBuckets = []string{
+	"spinifex-dhcp-leases",
+	"spinifex-vpcd-reconcile",
+	"spinifex-dns-reconcile",
+	"spinifex-route53-realize",
+	"spinifex-route53-verify",
+	"spinifex-quota-reconcile",
+	"spinifex-rds-leader",
+	"spinifex-ecs-leader",
+	"bedrock-leader",
+	"spinifex-eks-leader",
+	"spinifex-elbv2-leader",
+}
+
+// A cluster upgraded from the per-election buckets carries each of them, and
+// start removes every one while leaving other buckets alone.
+func TestDeleteObsoleteBuckets_RemovesEveryOldLeaseBucket(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	js := testutil.NewJetStream(t, nc)
+	for _, name := range append([]string{"spinifex-instance-state"}, upgradedBuckets...) {
+		_, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: name})
+		require.NoError(t, err, name)
+	}
+
+	deleteObsoleteBuckets(t.Context(), js)
+	deleteObsoleteBuckets(t.Context(), js) // idempotent once they are gone
+
+	for _, name := range upgradedBuckets {
+		_, err := js.KeyValue(t.Context(), name)
+		assert.ErrorIs(t, err, jetstream.ErrBucketNotFound, "%s must be deleted", name)
+	}
+	_, err := js.KeyValue(t.Context(), "spinifex-instance-state")
+	assert.NoError(t, err, "a bucket not on the list must survive")
+}

spinifex/runtime/daemon/route53.go

@@ -3,6 +3,7 @@ package daemon
 import (
 	"context"
 	"log/slog"
+	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/dns"
@@ -51,8 +52,8 @@ func (d *Daemon) startRoute53Realizer() {
 		return
 	}
 	r := route53domain.NewRealizer(d.route53Service.Store(), pub)
-	r.Elect = func(ctx context.Context) (func(), bool) {
+	r.Elect = func(ctx context.Context) (func(), time.Duration, bool) {
-		return reconcilelock.AcquireLeader(ctx, d.natsConn, route53domain.KVBucketRoute53Realize, d.node)
+		return reconcilelock.ElectPass(ctx, d.locks.route53Realize)
 	}
 	r.Reserved = d.route53Service.Reserved
 	go r.Run(d.ctx)
@@ -75,8 +76,8 @@ func (d *Daemon) newRoute53Verifier() *route53domain.Verifier {
 		servers, parent, err := lookup.Delegation(ctx, name)
 		return route53domain.Delegation{Source: "the " + parent + " name servers", Servers: servers}, err
 	}
-	v.Elect = func(ctx context.Context) (func(), bool) {
+	v.Elect = func(ctx context.Context) (func(), time.Duration, bool) {
-		return reconcilelock.AcquireLeader(ctx, d.natsConn, route53domain.KVBucketRoute53Verify, d.node)
+		return reconcilelock.ElectPass(ctx, d.locks.route53Verify)
 	}
 	return v
 }
@@ -87,7 +88,7 @@ func (d *Daemon) route53Delegate(ctx context.Context, in *route53domain.AdminDel
 	if len(in.Names) == 0 {
 		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "no names to delegate")
 	}
-	res, err := d.dnsWriter.Delegate(ctx, d.natsConn, d.node, in.Names, in.Remove)
+	res, err := d.dnsWriter.Delegate(ctx, d.locks.dns, in.Names, in.Remove)
 	if err != nil {
 		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "%v", err)
 	}

spinifex/runtime/roles/awsgw/awsgw.go

@@ -58,6 +58,7 @@ import (
 	taggingawsapi "github.com/mulgadc/spinifex/spinifex/domains/tagging/awsapi"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/reconciler"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/pipeline"
 	ingresshttp "github.com/mulgadc/spinifex/spinifex/ingress/http"
@@ -68,7 +69,6 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
 	"github.com/mulgadc/spinifex/spinifex/runtime/daemon"
 	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
-	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 	toml "github.com/pelletier/go-toml/v2"
 )
@@ -775,7 +775,9 @@ func launchService(config *config.ClusterConfig) error {
 	// terminations free quota. Started only when quotas are enabled so default-off
 	// gateways spin no ticker.
 	if quotaCfg.Enabled {
-		go runQuotaReconcile(janitorCtx, quotaSvc, natsConn, js, activeAccountIDs(iamService), len(config.Nodes))
+		holder, _ := os.Hostname()
+		lease := reconcile.NewLeaderLease(kvstore.OpenSyncBucket(js), admissionquota.KeyReconcile, holder)
+		go runQuotaReconcile(janitorCtx, quotaSvc, lease, js, activeAccountIDs(iamService), len(config.Nodes))
 	}
 
 	// Bedrock RPM enforcement is local and immediate (never gated on
@@ -820,13 +822,12 @@ func launchService(config *config.ClusterConfig) error {
 // runQuotaReconcile drives the per-account vCPU reconcile off changes to the
 // instance record space rather than a ticker: a change to one instance costs
 // that one account's recompute, and the resync sweeps every account as the
-// backstop. Each pass is guarded by the dedicated quota reconcile leader lock
+// backstop. Each pass is guarded by lease, the quota reconcile leader lease, so
-// so exactly one gateway sweeps at a time across a multi-gateway deployment.
+// exactly one gateway sweeps at a time across a multi-gateway deployment. The
-// The lock is distinct from vpcd's network-reconcile lock so the two loops
+// lease is distinct from vpcd's so the two loops never block each other. It
-// never block each other. It runs until ctx is cancelled.
+// runs until ctx is cancelled.
-func runQuotaReconcile(ctx context.Context, quota *admissionquota.Service, natsConn *nats.Conn,
+func runQuotaReconcile(ctx context.Context, quota *admissionquota.Service, lease *kvsync.Lease,
 	js jetstream.JetStream, accounts admissionquota.AccountLister, replicas int) {
-	holder, _ := os.Hostname()
 	cfg := kvstore.Config{Name: daemon.InstanceStateBucket, History: 1}
 	list := admissionquota.RecordVCPULister(
 		kvstore.New[vm.InstanceRecord](js, cfg), daemon.InstanceRecordPrefix)
@@ -835,7 +836,7 @@ func runQuotaReconcile(ctx context.Context, quota *admissionquota.Service, natsC
 	// reconcile loops: the lock keeps two gateways off the same counters, it is
 	// not what keeps this loop alive.
 	underLeader := func(run func() error) error {
-		release, elected := reconcile.AcquireLeader(ctx, natsConn, admissionquota.KVBucketQuotaReconcile, holder)
+		release, elected := reconcile.AcquireLeader(ctx, lease)
 		if !elected {
 			return nil
 		}

spinifex/runtime/roles/awsgw/quota_reconcile_test.go

@@ -9,7 +9,10 @@ import (
 
 	"github.com/mulgadc/spinifex/internal/testkit"
 	admissionquota "github.com/mulgadc/spinifex/spinifex/domains/admission/quota"
+	"github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/resource"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/require"
@@ -31,6 +34,11 @@ func TestOpenAccountUsageBucketPreservesExistingConfiguration(t *testing.T) {
 	require.EqualValues(t, 5, status.History())
 }
 
+// quotaLease is one gateway's quota reconcile lease on its own sync bucket.
+func quotaLease(js jetstream.JetStream) *kvsync.Lease {
+	return reconcile.NewLeaderLease(kvstore.OpenSyncBucket(js), admissionquota.KeyReconcile, "gw-test")
+}
+
 // putInstanceRecord writes one instance record at the key the sweep reads, so a
 // test can state an account's holdings without a node or a describe responder.
 func putInstanceRecord(t *testing.T, records jetstream.KeyValue, id, accountID, instanceType string, state vm.InstanceState) {
@@ -78,7 +86,7 @@ func TestRunQuotaReconcileCountsFromTheRecordSpace(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	done := make(chan struct{})
 	go func() {
-		runQuotaReconcile(ctx, quota, nc, js, accounts, 1)
+		runQuotaReconcile(ctx, quota, quotaLease(js), js, accounts, 1)
 		close(done)
 	}()
 
@@ -110,7 +118,7 @@ func TestRunQuotaReconcileFollowsARecordChange(t *testing.T) {
 	accounts := func() ([]string, error) { return []string{account}, nil }
 	ctx, cancel := context.WithCancel(context.Background())
 	t.Cleanup(cancel)
-	go runQuotaReconcile(ctx, quota, nc, js, accounts, 1)
+	go runQuotaReconcile(ctx, quota, quotaLease(js), js, accounts, 1)
 
 	require.Eventually(t, func() bool {
 		return readUsageCounter(t, bucket, account) == 4
@@ -146,7 +154,7 @@ func TestRunQuotaReconcileRecomputesOnlyTheChangedAccount(t *testing.T) {
 	accounts := func() ([]string, error) { return []string{changed, untouched}, nil }
 	ctx, cancel := context.WithCancel(context.Background())
 	t.Cleanup(cancel)
-	go runQuotaReconcile(ctx, quota, nc, js, accounts, 1)
+	go runQuotaReconcile(ctx, quota, quotaLease(js), js, accounts, 1)
 
 	require.Eventually(t, func() bool {
 		return readUsageCounter(t, bucket, changed) == 4
@@ -164,6 +172,42 @@ func TestRunQuotaReconcileRecomputesOnlyTheChangedAccount(t *testing.T) {
 		"a change to one account must not recompute another; the resync is what eventually corrects it")
 }
 
+// The quota counters are written by one gateway at a time: while a peer holds
+// the reconcile lease this gateway leaves them alone, even when a record change
+// wakes it, and takes the work up once the lease is free.
+func TestRunQuotaReconcileLeavesTheCountersToTheLeaseHolder(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	js := testutil.NewJetStream(t, nc)
+	bucket, err := openAccountUsageBucket(t.Context(), js)
+	require.NoError(t, err)
+	quota := admissionquota.New(admissionquota.Limits{Enabled: true, VCPUs: 100}, bucket)
+
+	syncKV, err := kvstore.OpenSyncBucket(js).KV(t.Context())
+	require.NoError(t, err)
+	_, err = syncKV.Put(t.Context(), admissionquota.KeyReconcile, []byte(`{"holder":"gw-peer","generation":1}`))
+	require.NoError(t, err)
+
+	const account = "123456789012"
+	records := instanceRecordBucket(t, js)
+	putInstanceRecord(t, records, "i-aaa", account, "m5.xlarge", vm.StateRunning)
+
+	accounts := func() ([]string, error) { return []string{account}, nil }
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+	go runQuotaReconcile(ctx, quota, quotaLease(js), js, accounts, 1)
+
+	putInstanceRecord(t, records, "i-bbb", account, "t3.micro", vm.StateRunning)
+	require.Never(t, func() bool {
+		return readUsageCounter(t, bucket, account) != -1
+	}, 500*time.Millisecond, 20*time.Millisecond, "a gateway that does not hold the lease must not write the counter")
+
+	require.NoError(t, syncKV.Delete(t.Context(), admissionquota.KeyReconcile))
+	putInstanceRecord(t, records, "i-ccc", account, "t3.micro", vm.StateRunning)
+	require.Eventually(t, func() bool {
+		return readUsageCounter(t, bucket, account) == 8
+	}, 10*time.Second, 20*time.Millisecond, "with the lease free, the next change reconciles the account")
+}
+
 // readUsageCounter reads an account's integer vCPU counter from the usage bucket,
 // returning -1 while the key is still absent so Eventually keeps polling.
 func readUsageCounter(t *testing.T, bucket jetstream.KeyValue, account string) int {

spinifex/runtime/roles/vpcd/vpcd.go

@@ -33,6 +33,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
 	"github.com/mulgadc/spinifex/spinifex/domains/network/subscribers"
 	"github.com/mulgadc/spinifex/spinifex/domains/network/topology"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
 	hostcommand "github.com/mulgadc/spinifex/spinifex/runtime/host/command"
 	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
@@ -702,9 +703,11 @@ func launchService(cfg *Config) error {
 	}
 
 	// Elect one vpcd for startup reconcile; without this, concurrent Get-then-Create on Logical_Router
-	// produces duplicate rows that ovn-nbctl rejects. Runtime events still fan out via queue groups.
+	// produces duplicate rows that ovn-nbctl rejects. The drift loop shares the lease, so the two
+	// passes exclude each other on this node too. Runtime events still fan out via queue groups.
 	holder, _ := os.Hostname()
-	releaseLeader, isLeader := reconcile.AcquireLeader(ctx, nc, reconcile.KVBucketVPCDReconcile, holder)
+	leader := reconcile.NewLeaderLease(kvstore.OpenSyncBucket(js), reconcile.KeyVPCDReconcile, holder)
+	releaseLeader, isLeader := reconcile.AcquireLeaderAtStart(ctx, leader)
 
 	subscriber, err := subscribers.New(subscribers.Config{
 		Topology: topoMgr,
@@ -788,7 +791,7 @@ func launchService(cfg *Config) error {
 	defer loopCancel()
 	loopDone := make(chan struct{})
 	go func() {
-		reconcile.DriftLoop(loopCtx, rec, nc, cfg.AZ, holder, startupErr)
+		reconcile.DriftLoop(loopCtx, rec, nc, leader, cfg.AZ, startupErr)
 		close(loopDone)
 	}()
 

tests/e2e/kvquorum/replication_test.go

@@ -41,24 +41,48 @@ func runBucketsAreReplicated(t *testing.T, fix *Fixture) {
 	}
 	harness.RequireKVQuorum(t, fix.Env, nodes)
 
-	// The leader-lease buckets are named as well as swept. They were the worst
+	// The sync bucket is named as well as swept: every leader election runs in
-	// case of the defect this suite guards — created with no replica count at
+	// it, so if it loses quorum, no reconciler in the cluster can lead.
-	// all — and a regression that reached only them would otherwise be one more
+	harness.Step(t, "confirm the sync bucket is replicated and the old lease buckets are gone")
-	// line in a long list rather than the headline.
+	sync, found := findBucket(reports, syncBucket)
-	harness.Step(t, "confirm the leader-lease buckets are among those checked")
+	require.True(t, found, "no %s bucket was found; either no election has run or the audit missed it", syncBucket)
-	leases := leaseBuckets(reports)
+	require.Equalf(t, want, sync.Replicas, "%s holds %d replicas, want %d", syncBucket, sync.Replicas, want)
-	require.NotEmpty(t, leases,
+	require.Empty(t, presentBuckets(reports, obsoleteLeaseBuckets),
-		"no leader-lease bucket was found; either the cluster has never run a reconciler or the audit missed them")
+		"the daemon deletes the per-caller lease buckets at start, so none may survive")
-	harness.Detail(t, "lease_buckets", strings.Join(leases, ","))
 }
 
-// leaseBuckets returns the names of the reconciler leader-lease buckets, which
+// syncBucket is where every kvsync claim and lease lives.
-// are the ones a reconciler must acquire before it can do anything at all.
+const syncBucket = "spinifex-sync"
-func leaseBuckets(reports []kvstore.BucketReport) []string {
+
-	var out []string
+// obsoleteLeaseBuckets are the per-caller lease buckets the sync bucket replaced.
+var obsoleteLeaseBuckets = []string{
+	"spinifex-vpcd-reconcile",
+	"spinifex-dns-reconcile",
+	"spinifex-route53-realize",
+	"spinifex-route53-verify",
+	"spinifex-quota-reconcile",
+	"spinifex-rds-leader",
+	"spinifex-ecs-leader",
+	"bedrock-leader",
+	"spinifex-eks-leader",
+	"spinifex-elbv2-leader",
+}
+
+func findBucket(reports []kvstore.BucketReport, name string) (kvstore.BucketReport, bool) {
 	for _, r := range reports {
-		if strings.HasSuffix(r.Bucket, "-leader") || strings.HasSuffix(r.Bucket, "-reconcile") {
+		if r.Bucket == name {
-			out = append(out, r.Bucket)
+			return r, true
+		}
+	}
+	return kvstore.BucketReport{}, false
+}
+
+// presentBuckets returns those of names that appear in reports, sorted.
+func presentBuckets(reports []kvstore.BucketReport, names []string) []string {
+	var out []string
+	for _, name := range names {
+		if _, ok := findBucket(reports, name); ok {
+			out = append(out, name)
 		}
 	}
 	sort.Strings(out)