fix: confirm a stale or absent UEFI holder heartbeat with the stream leader before takeover

A start checks the pointer's firmware before it touches the claim, reuses
vars.fd whenever seeded.json names the claim's generation, and refuses as
retryable, keeping the file and the claim, when seeded.json cannot be read.
A rollback releases the claim only when QEMU never started and no local
copy may owe a write-back. An EFI-volume instance reads no store record,
and a pointer move to the current key is still confirmed by CAS.

.github/workflows/actions.lock

@@ -65,7 +65,7 @@ workflows:
         - 'actions/setup-go@v7.0.0'
         - 'actions/upload-artifact@v7.0.2'
         - 'docker/setup-buildx-action@v4.4.1'
-        - 'softprops/action-gh-release@v3.0.4'
+        - 'softprops/action-gh-release@v3.0.3'
     '.github/workflows/segscan-oracle.yml':
         - 'actions/checkout@v7.0.1'
         - 'actions/create-github-app-token@v3.3.0'
@@ -79,7 +79,7 @@ workflows:
         - 'actions/checkout@v7.0.1'
         - 'actions/setup-node@v7.1.0'
         - 'actions/upload-artifact@v7.0.2'
-        - 'pnpm/action-setup@v6.1.1'
+        - 'pnpm/action-setup@v6.1.0'
 dependencies:
     'actions/cache@v6.1.0':
         ref: 'v6.1.0'
@@ -131,13 +131,13 @@ dependencies:
         commit: 'sha1-dfe3c3f87815947d99a8997f908cb6525fc44e9e'
         owner_id: 761456
         repo_id: 258271226
-    'pnpm/action-setup@v6.1.1':
-        ref: 'v6.1.1'
-        commit: 'sha1-73e46992e05b19abd31f2f3c20e0e7370d762115'
+    'pnpm/action-setup@v6.1.0':
+        ref: 'v6.1.0'
+        commit: 'sha1-ea17c68df8912ef543352723c149a84f56e3d413'
         owner_id: 21320719
         repo_id: 262289747
-    'softprops/action-gh-release@v3.0.4':
-        ref: 'v3.0.4'
-        commit: 'sha1-8a0cd651e5b2365bd640854a3fa5dfc70e4875f3'
+    'softprops/action-gh-release@v3.0.3':
+        ref: 'v3.0.3'
+        commit: 'sha1-efb35369e0ad2afab669f228072c1b0d510eae64'
         owner_id: 2242
         repo_id: 204253808

.github/workflows/e2e-nightly.yml

@@ -578,8 +578,6 @@ jobs:
              echo \"::group::suite ${SUITE}\"; \
              date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-${SUITE}.start'; \
              GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
-               SPINIFEX_NODES='${{ steps.vm.outputs.wan_ip }}' SPINIFEX_SSH_USER=tf-user \
-               SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
                ARTIFACT_DIR='${ARTIFACT_DIR}' \
                ./${SUITE}.test -test.v -test.timeout=${TMO} \
                2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log'; \

.github/workflows/release.yml

@@ -150,7 +150,7 @@ jobs:
           path: artifacts
 
       - name: Create Draft GitHub Release
-        uses: softprops/action-gh-release@v3.0.4
+        uses: softprops/action-gh-release@v3.0.3
         with:
           draft: true
           files: |

.github/workflows/ui.yml

@@ -51,7 +51,7 @@ jobs:
           sparse-checkout: spinifex/runtime/roles/spinifexui/frontend
 
       - name: Setup pnpm
-        uses: pnpm/action-setup@v6.1.1
+        uses: pnpm/action-setup@v6.1.0
         with:
           package_json_file: spinifex/runtime/roles/spinifexui/frontend/package.json
 
@@ -127,7 +127,7 @@ jobs:
           sparse-checkout: spinifex/runtime/roles/spinifexui/frontend
 
       - name: Setup pnpm
-        uses: pnpm/action-setup@v6.1.1
+        uses: pnpm/action-setup@v6.1.0
         with:
           package_json_file: spinifex/runtime/roles/spinifexui/frontend/package.json
 

Makefile

@@ -90,7 +90,6 @@ go_build:
 LAMBDA_TESTLINK_DIR := ./bin/lambda-testlink
 LAMBDA_TESTLINK_PKG := github.com/mulgadc/spinifex/spinifex/runtime/daemon
 LAMBDA_TESTLINK_PIN := $(LAMBDA_TESTLINK_PKG).lambdaHardKillSHA256
-LAMBDA_TESTLINK_GW := github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw
 LAMBDA_DIGEST_RE := ^[0-9a-f]{64}$$
 build-lambda-testlink: override export LAMBDA_IMAGE_DIGEST := $(value LAMBDA_IMAGE_DIGEST)
 build-lambda-testlink:
@@ -105,8 +104,8 @@ build-lambda-testlink:
 	@digest=$$(sha256sum $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so | cut -d' ' -f1); \
 	test $${#digest} -eq 64 || { echo "initializer digest missing"; exit 1; }; \
 	echo "initializer sha256 $$digest"; \
-	pin="-X $(LAMBDA_TESTLINK_PIN)=$$digest -X $(LAMBDA_TESTLINK_PKG).lambdaImageDigest=$$LAMBDA_IMAGE_DIGEST -X $(LAMBDA_TESTLINK_GW).lambdaImageDigest=$$LAMBDA_IMAGE_DIGEST"; \
-	LAMBDA_TESTLINK_INITIALIZER=$(abspath $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so) go test -count=1 -tags lambdatestlink -ldflags "$$pin" -run '^TestLambda(HardKillPin|ImageDigestPin)$$' ./spinifex/runtime/daemon/ ./spinifex/runtime/roles/awsgw/ && \
+	pin="-X $(LAMBDA_TESTLINK_PIN)=$$digest -X $(LAMBDA_TESTLINK_PKG).lambdaImageDigest=$$LAMBDA_IMAGE_DIGEST"; \
+	LAMBDA_TESTLINK_INITIALIZER=$(abspath $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so) go test -count=1 -tags lambdatestlink -ldflags "$$pin" -run '^TestLambda(HardKillPin|ImageDigestPin)$$' ./spinifex/runtime/daemon/ && \
 	go build $(GO_BUILD_MOD) -tags lambdatestlink -ldflags "$(subst -s -w ,,$(LDFLAGS)) $$pin" -o $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME) cmd/spinifex/main.go
 	@mkdir -p $(LAMBDA_TESTLINK_DIR)
 	mv $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME) $(LAMBDA_TESTLINK_DIR)/

architecture/layering/allowlist_test.go

@@ -17,6 +17,7 @@ var allowlist = []allowance{
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/instance", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/routetable", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/snapshot", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
+	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/volume", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/vpc", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 
 	{From: "agents/ecs/agent", To: "domains/ecs", Debt: "agents/ecs/agent", Reason: "assignment/state wire types from the ECS implementation; needs a guest/controller contract"},

build/microvm/init.sh

@@ -32,8 +32,6 @@ load_mod qemu_fw_cfg || load_mod fw_cfg_sysfs || true
 load_mod failover || true
 load_mod net_failover || true
 load_mod virtio_net || true
-# lb-agent probes each data NIC's gateway on an AF_PACKET socket.
-load_mod af_packet || true
 
 NETCFG=/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
 i=0

build/microvm/lambda-runtime/init.sh

@@ -16,8 +16,8 @@ load_mod() {
     done
     return 1
 }
-# lambda-agent reads its identity from fw_cfg, or the configuration disk, and
-# seals it before anything else runs. virtio_net needs net_failover and failover, and the
+# lambda-agent reads its identity from fw_cfg and unloads qemu_fw_cfg before
+# anything else runs. virtio_net needs net_failover and failover, and the
 # insmod fallback resolves no dependencies, so load the chain bottom-up.
 load_mod qemu_fw_cfg || echo "[init] qemu_fw_cfg did not load" >/dev/console
 load_mod failover || true
@@ -26,30 +26,8 @@ load_mod virtio_net || echo "[init] virtio_net did not load" >/dev/console
 
 ip link set lo up
 
-# Without fw_cfg data the executor attaches the same entries as a tar archive
-# on the second disk; lambda-agent reads them here and erases the disk before
-# the function starts. qemu_fw_cfg loads without a device, so test the entry.
-CFG_ROOT=/spinifex-cfg
-FWCFG_ROOT=/sys/firmware/qemu_fw_cfg
-if [ ! -f "$FWCFG_ROOT/by_name/opt/spinifex/netcfg/raw" ]; then
-    load_mod virtio_blk || echo "[init] virtio_blk did not load" >/dev/console
-    i=0
-    while [ "$i" -lt 25 ] && [ ! -b /dev/vdb ]; do
-        sleep 0.04
-        i=$((i + 1))
-    done
-    if [ -b /dev/vdb ]; then
-        mkdir -m 0700 "$CFG_ROOT"
-        if ! mount -t tmpfs -o mode=0700,size=1m tmpfs "$CFG_ROOT" ||
-            ! tar -xf /dev/vdb -C "$CFG_ROOT"; then
-            echo "[init] configuration disk not readable" >/dev/console
-        fi
-        FWCFG_ROOT=$CFG_ROOT
-    fi
-fi
-
 # The guest's addresses come from the same netcfg entry as other system VMs.
-NETCFG=$FWCFG_ROOT/by_name/opt/spinifex/netcfg/raw
+NETCFG=/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
 i=0
 while [ "$i" -lt 25 ] && [ ! -f "$NETCFG" ]; do
     sleep 0.04

cmd/lambda-agent/connect.go

@@ -5,7 +5,6 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
-	"os"
 
 	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
@@ -26,22 +25,9 @@ type startup struct {
 }
 
 func defaultStartup() startup {
-	if _, err := os.Stat(hostlink.ConfigDiskRoot); err == nil {
-		return configDiskStartup(hostlink.ConfigDiskRoot, hostlink.ConfigDiskDevice)
-	}
 	return startup{fwCfgDir: hostlink.FwCfgDir, fwCfgRoot: hostlink.FwCfgRoot, unload: hostlink.UnloadFwCfg}
 }
 
-// configDiskStartup reads the entries init extracted from the configuration
-// disk, and seals by erasing the disk and removing the extracted tree.
-func configDiskStartup(root, device string) startup {
-	return startup{
-		fwCfgDir:  root + "/by_name/" + lambdav1.FwCfgPrefix,
-		fwCfgRoot: root,
-		unload:    func() error { return hostlink.SealConfigDisk(device, root) },
-	}
-}
-
 // connect loads the identity, seals fw_cfg whether or not the load succeeded,
 // then dials the host. Under TLS 1.3 a host refusal is seen only on first read.
 func (s startup) connect(ctx context.Context) (*tls.Conn, lambdav1.Scope, int, error) {

cmd/lambda-agent/connect_configdisk_test.go

@@ -1,43 +0,0 @@
-//test:in-package — package main cannot be imported by an external test package.
-
-package main
-
-import (
-	"bytes"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-// An identity that does not load still seals the configuration disk: the
-// guest key on it is erased before the agent refuses to start.
-func TestConfigDiskStartup_RefusedLoadErasesDisk(t *testing.T) {
-	dir := t.TempDir()
-	root := filepath.Join(dir, "cfg")
-	require.NoError(t, os.MkdirAll(filepath.Join(root, "by_name"), 0o700))
-	device := filepath.Join(dir, "vdb")
-	require.NoError(t, os.WriteFile(device, bytes.Repeat([]byte("guest-key"), 512), 0o600))
-
-	_, _, code, err := configDiskStartup(root, device).connect(t.Context())
-	require.Error(t, err)
-	require.Equal(t, exitRefused, code)
-	got, rerr := os.ReadFile(device)
-	require.NoError(t, rerr)
-	require.Len(t, got, 9*512)
-	require.Equal(t, make([]byte, len(got)), got, "configuration disk erased")
-}
-
-// A disk that cannot be erased fails closed: the agent refuses, and the
-// extracted tree is kept so the seal check sees it.
-func TestConfigDiskStartup_UnerasableDiskRefuses(t *testing.T) {
-	dir := t.TempDir()
-	root := filepath.Join(dir, "cfg")
-	require.NoError(t, os.MkdirAll(filepath.Join(root, "by_name"), 0o700))
-
-	_, _, code, err := configDiskStartup(root, filepath.Join(dir, "missing")).connect(t.Context())
-	require.ErrorContains(t, err, "erase configuration disk")
-	require.Equal(t, exitRefused, code)
-	require.DirExists(t, root)
-}

cmd/lambda-agent/sdk_config_testlink.go

@@ -1,65 +0,0 @@
-//go:build lambdatestlink
-
-package main
-
-import (
-	"crypto/x509"
-	"errors"
-	"fmt"
-	"net"
-	"net/url"
-	"os"
-	"path/filepath"
-)
-
-const (
-	sdkEndpointEntry = "test-sdk-endpoint"
-	sdkCAEntry       = "test-sdk-ca"
-	sdkBundlePath    = "/run/lambda-sdk/ca-bundle.pem"
-	systemBundlePath = "/etc/ssl/certs/ca-certificates.crt"
-)
-
-type sdkConfig struct {
-	endpoint string
-	ca       []byte
-}
-
-// readSDKConfig reads optional public settings before the boot config is sealed.
-func readSDKConfig(dir string) (*sdkConfig, error) {
-	endpoint, endpointErr := os.ReadFile(filepath.Join(dir, sdkEndpointEntry, "raw"))
-	ca, caErr := os.ReadFile(filepath.Join(dir, sdkCAEntry, "raw"))
-	if errors.Is(endpointErr, os.ErrNotExist) && errors.Is(caErr, os.ErrNotExist) {
-		return nil, nil
-	}
-	if endpointErr != nil || caErr != nil {
-		return nil, errors.New("incomplete SDK gateway settings")
-	}
-	u, err := url.Parse(string(endpoint))
-	if err != nil || u.Scheme != "https" || u.User != nil || u.Path != "" || u.RawQuery != "" || u.Fragment != "" ||
-		net.ParseIP(u.Hostname()) == nil || u.Port() == "" {
-		return nil, errors.New("invalid SDK gateway endpoint")
-	}
-	if len(ca) > 64<<10 || !x509.NewCertPool().AppendCertsFromPEM(ca) {
-		return nil, errors.New("invalid SDK gateway CA")
-	}
-	return &sdkConfig{endpoint: string(endpoint), ca: ca}, nil
-}
-
-// writeBundle combines the system roots with the cluster's public root.
-func (c *sdkConfig) writeBundle(path, systemPath string) error {
-	system, err := os.ReadFile(systemPath)
-	if err != nil && !errors.Is(err, os.ErrNotExist) {
-		return fmt.Errorf("read system CA bundle: %w", err)
-	}
-	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
-		return fmt.Errorf("create SDK CA directory: %w", err)
-	}
-	if err := os.Chmod(filepath.Dir(path), 0o755); err != nil {
-		return fmt.Errorf("make SDK CA directory readable: %w", err)
-	}
-	bundle := append(append(system, '\n'), c.ca...)
-	if err := os.WriteFile(path, bundle, 0o644); err != nil {
-		return fmt.Errorf("write SDK CA bundle: %w", err)
-	}
-	return os.Chmod(path, 0o644)
-}

cmd/lambda-agent/sdk_config_testlink_test.go

@@ -1,106 +0,0 @@
-//go:build lambdatestlink
-
-//test:in-package exercises the unexported boot configuration reader
-package main
-
-import (
-	"crypto/ecdsa"
-	"crypto/elliptic"
-	"crypto/rand"
-	"crypto/tls"
-	"crypto/x509"
-	"crypto/x509/pkix"
-	"encoding/pem"
-	"math/big"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"path/filepath"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestSDKConfig_OptionalAndInvalid(t *testing.T) {
-	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
-	defer server.Close()
-	ca := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw})
-	for _, tc := range []struct {
-		name     string
-		endpoint []byte
-		ca       []byte
-		wantErr  bool
-		want     bool
-	}{
-		{name: "absent"},
-		{name: "valid", endpoint: []byte(server.URL), ca: ca, want: true},
-		{name: "missing ca", endpoint: []byte(server.URL), wantErr: true},
-		{name: "missing endpoint", ca: ca, wantErr: true},
-		{name: "invalid endpoint", endpoint: []byte("http://127.0.0.1:1234"), ca: ca, wantErr: true},
-		{name: "invalid ca", endpoint: []byte(server.URL), ca: []byte("not a certificate"), wantErr: true},
-	} {
-		t.Run(tc.name, func(t *testing.T) {
-			dir := t.TempDir()
-			for name, value := range map[string][]byte{sdkEndpointEntry: tc.endpoint, sdkCAEntry: tc.ca} {
-				if value != nil {
-					require.NoError(t, os.MkdirAll(filepath.Join(dir, name), 0o700))
-					require.NoError(t, os.WriteFile(filepath.Join(dir, name, "raw"), value, 0o600))
-				}
-			}
-			cfg, err := readSDKConfig(dir)
-			if tc.wantErr {
-				require.Error(t, err)
-				require.Nil(t, cfg)
-				return
-			}
-			require.NoError(t, err)
-			if !tc.want {
-				require.Nil(t, cfg)
-				return
-			}
-			bundle := filepath.Join(t.TempDir(), "sdk", "ca.pem")
-			require.NoError(t, cfg.writeBundle(bundle, filepath.Join(t.TempDir(), "absent")))
-			data, err := os.ReadFile(bundle)
-			require.NoError(t, err)
-			roots := x509.NewCertPool()
-			require.True(t, roots.AppendCertsFromPEM(data))
-			client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: roots}}}
-			resp, err := client.Get(cfg.endpoint)
-			require.NoError(t, err)
-			defer resp.Body.Close()
-			assert.Equal(t, http.StatusOK, resp.StatusCode)
-		})
-	}
-}
-
-func TestSDKConfig_BundleKeepsSystemRoots(t *testing.T) {
-	cluster := httptest.NewTLSServer(nil)
-	defer cluster.Close()
-	clusterCA := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cluster.Certificate().Raw})
-	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
-	require.NoError(t, err)
-	template := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test system root"},
-		NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour),
-		IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign}
-	der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
-	require.NoError(t, err)
-	systemCA := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
-	dir := t.TempDir()
-	systemPath, bundlePath := filepath.Join(dir, "system.pem"), filepath.Join(dir, "sdk", "ca.pem")
-	require.NoError(t, os.WriteFile(systemPath, systemCA, 0o600))
-	require.NoError(t, (&sdkConfig{ca: clusterCA}).writeBundle(bundlePath, systemPath))
-	bundle, err := os.ReadFile(bundlePath)
-	require.NoError(t, err)
-	roots := x509.NewCertPool()
-	require.True(t, roots.AppendCertsFromPEM(bundle))
-	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: roots}}}
-	resp, err := client.Get(cluster.URL)
-	require.NoError(t, err)
-	require.NoError(t, resp.Body.Close())
-	systemCert, err := x509.ParseCertificate(der)
-	require.NoError(t, err)
-	_, err = systemCert.Verify(x509.VerifyOptions{Roots: roots})
-	require.NoError(t, err)
-}

cmd/lambda-agent/start_testlink.go

@@ -12,14 +12,11 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
-	"sort"
-	"strings"
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/agent"
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/runtimeapi"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/lambdaenv"
 )
 
 const (
@@ -34,17 +31,13 @@ type testlink struct {
 	pkg            taskPackage
 	runtimeAddr    string
 	startBootstrap func(ctx context.Context, env []string) (*exec.Cmd, error)
-	caBundlePath   string
-	systemCAPath   string
 }
 
 func start(ctx context.Context) int {
 	t := testlink{
-		startup:      defaultStartup(),
-		pkg:          devTaskPackage(),
-		runtimeAddr:  runtimeAddr,
-		caBundlePath: sdkBundlePath,
-		systemCAPath: systemBundlePath,
+		startup:     defaultStartup(),
+		pkg:         devTaskPackage(),
+		runtimeAddr: runtimeAddr,
 		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
 			return agent.StartBootstrap(ctx, taskRoot+"/bootstrap", env, os.Stdout, os.Stderr)
 		},
@@ -60,7 +53,6 @@ func (t testlink) run(parent context.Context) (int, error) {
 	// The package values are fw_cfg entries, so read them before connect seals it.
 	spec, specErr := readPackageSpec(t.fwCfgDir)
 	grant, grantErr := readGrant(t.fwCfgDir)
-	sdk, sdkErr := readSDKConfig(t.fwCfgDir)
 	conn, scope, code, err := t.connect(parent)
 	if err != nil {
 		return code, errors.Join(err, specErr, grantErr)
@@ -93,18 +85,6 @@ func (t testlink) run(parent context.Context) (int, error) {
 	}
 	cred := *first.Credential
 	first.Credential = nil
-	if err := lambdaenv.Validate(cred.Environment); err != nil {
-		return exitRefused, errors.New("first frame carries an invalid function environment")
-	}
-	if sdkErr != nil {
-		slog.Warn("lambda-agent: SDK gateway settings unavailable", "err", sdkErr)
-	}
-	if sdk != nil {
-		if err := sdk.writeBundle(t.caBundlePath, t.systemCAPath); err != nil {
-			slog.Warn("lambda-agent: SDK gateway settings unavailable", "err", err)
-			sdk = nil
-		}
-	}
 	if err := t.pkg.install(spec); err != nil {
 		return exitRefused, fmt.Errorf("install package: %w", err)
 	}
@@ -120,8 +100,7 @@ func (t testlink) run(parent context.Context) (int, error) {
 
 	ctx, cancel := context.WithCancelCause(parent)
 	defer cancel(nil)
-	region, _, _ := strings.Cut(string(scope), "/")
-	cmd, err := t.startBootstrap(ctx, bootstrapEnv(ln.Addr().String(), spec.handler, cred, region, sdk, t.caBundlePath))
+	cmd, err := t.startBootstrap(ctx, bootstrapEnv(ln.Addr().String(), spec.handler, cred))
 	if err != nil {
 		return exitLink, fmt.Errorf("start bootstrap: %w", err)
 	}
@@ -145,35 +124,18 @@ func (t testlink) run(parent context.Context) (int, error) {
 	return exitClosed, nil
 }
 
-// bootstrapEnv supplies the runtime, execution-role and SDK settings.
-func bootstrapEnv(runtimeAPI, handler string, cred hostlink.Credential, region string, sdk *sdkConfig, caPath string) []string {
-	vars := map[string]string{
-		"AWS_LAMBDA_RUNTIME_API": runtimeAPI,
-		"_HANDLER":               handler,
-		"LAMBDA_TASK_ROOT":       taskRoot,
-		"PATH":                   "/bin:/sbin:/usr/bin",
-		"AWS_ACCESS_KEY_ID":      cred.AccessKeyID,
-		"AWS_SECRET_ACCESS_KEY":  cred.SecretAccessKey,
-		"AWS_SESSION_TOKEN":      cred.SessionToken,
-		"AWS_REGION":             region,
-		"AWS_DEFAULT_REGION":     region,
+// bootstrapEnv is the bootstrap's whole environment: the runtime variables and
+// the execution role's three credential variables.
+func bootstrapEnv(runtimeAPI, handler string, cred hostlink.Credential) []string {
+	return []string{
+		"AWS_LAMBDA_RUNTIME_API=" + runtimeAPI,
+		"_HANDLER=" + handler,
+		"LAMBDA_TASK_ROOT=" + taskRoot,
+		"PATH=/bin:/sbin:/usr/bin",
+		"AWS_ACCESS_KEY_ID=" + cred.AccessKeyID,
+		"AWS_SECRET_ACCESS_KEY=" + cred.SecretAccessKey,
+		"AWS_SESSION_TOKEN=" + cred.SessionToken,
 	}
-	if sdk != nil {
-		vars["AWS_ENDPOINT_URL"], vars["AWS_CA_BUNDLE"] = sdk.endpoint, caPath
-	}
-	for k, v := range cred.Environment {
-		vars[k] = v
-	}
-	keys := make([]string, 0, len(vars))
-	for k := range vars {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys)
-	env := make([]string, 0, len(keys))
-	for _, k := range keys {
-		env = append(env, k+"="+vars[k])
-	}
-	return env
 }
 
 // shutdownGrace is how long past the invocation deadline the agent waits for

cmd/lambda-agent/start_testlink_test.go

@@ -7,10 +7,8 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
-	"encoding/pem"
 	"io"
 	"net/http"
-	"net/http/httptest"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -72,7 +70,6 @@ type bootstrapStarts struct {
 	sealed  bool
 	// handlerOut receives the _HANDLER the bootstrap process itself saw.
 	handlerOut string
-	caBundle   string
 	// env is the environment the agent gave the bootstrap.
 	env []string
 }
@@ -97,7 +94,6 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 	b.started = make(chan struct{}, 1)
 	b.task = t.TempDir()
 	b.handlerOut = filepath.Join(t.TempDir(), "handler")
-	b.caBundle = filepath.Join(t.TempDir(), "sdk", "ca-bundle.pem")
 	return testlink{
 		startup: g.startup(),
 		pkg: taskPackage{
@@ -106,9 +102,7 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 			mountDir:   func(string) error { return nil },
 			sealDir:    func(string) error { b.sealed = true; return nil },
 		},
-		runtimeAddr:  "127.0.0.1:0",
-		caBundlePath: b.caBundle,
-		systemCAPath: filepath.Join(t.TempDir(), "system-ca.pem"),
+		runtimeAddr: "127.0.0.1:0",
 		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
 			b.n++
 			b.env = env
@@ -182,87 +176,10 @@ func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
 		"AWS_ACCESS_KEY_ID=" + testCredential.AccessKeyID,
 		"AWS_SECRET_ACCESS_KEY=" + testCredential.SecretAccessKey,
 		"AWS_SESSION_TOKEN=" + testCredential.SessionToken,
-		"AWS_REGION=ap-southeast-2",
-		"AWS_DEFAULT_REGION=ap-southeast-2",
 	}, b.env, "the bootstrap gets exactly the runtime and credential variables")
 	assert.NotEmpty(t, envValue(b.env, "AWS_LAMBDA_RUNTIME_API"))
 }
 
-func TestTestlinkProvidesSDKSettingsToBootstrap(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
-	defer server.Close()
-	h := newHost(t)
-	h.accept(ctx)
-	entries := h.entries()
-	entries[sdkEndpointEntry] = []byte(server.URL)
-	entries[sdkCAEntry] = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw})
-	var b bootstrapStarts
-	done := runTestlinkAsync(ctx, b.testlink(t, entries))
-	conn := h.guest(t)
-	defer conn.Close()
-	hs := hostlink.NewTestStream(conn)
-	req := hostlink.Request{Credential: &testCredential, GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: []byte(`{}`)}
-	require.NoError(t, hs.HostSend(req))
-	_, err := hs.HostResult(ctx)
-	require.NoError(t, err)
-	res := wait(t, done, "agent to stop")
-	require.NoError(t, res.err)
-	assert.Equal(t, server.URL, envValue(b.env, "AWS_ENDPOINT_URL"))
-	assert.Equal(t, b.caBundle, envValue(b.env, "AWS_CA_BUNDLE"))
-	assert.Equal(t, "ap-southeast-2", envValue(b.env, "AWS_REGION"))
-	assert.Equal(t, "ap-southeast-2", envValue(b.env, "AWS_DEFAULT_REGION"))
-	ca, err := os.ReadFile(b.caBundle)
-	require.NoError(t, err)
-	assert.Contains(t, string(ca), string(entries[sdkCAEntry]))
-	info, err := os.Stat(b.caBundle)
-	require.NoError(t, err)
-	assert.Equal(t, os.FileMode(0o644), info.Mode().Perm())
-}
-
-func TestTestlinkProvidesFunctionEnvironment(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	h := newHost(t)
-	h.accept(ctx)
-	var b bootstrapStarts
-	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
-	conn := h.guest(t)
-	defer conn.Close()
-	hs := hostlink.NewTestStream(conn)
-	cred := testCredential
-	cred.Environment = map[string]string{"GREETING": "hello $WORLD", "PATH": "/my/bin"}
-	req := hostlink.Request{Credential: &cred, GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: []byte(`{}`)}
-	require.NoError(t, hs.HostSend(req))
-	_, err := hs.HostResult(ctx)
-	require.NoError(t, err)
-	require.NoError(t, wait(t, done, "agent to stop").err)
-	assert.Equal(t, "hello $WORLD", envValue(b.env, "GREETING"))
-	assert.Equal(t, "/my/bin", envValue(b.env, "PATH"))
-	assert.Equal(t, testCredential.SecretAccessKey, envValue(b.env, "AWS_SECRET_ACCESS_KEY"))
-}
-
-func TestTestlinkRejectsReservedFunctionEnvironment(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
-	h := newHost(t)
-	h.accept(ctx)
-	var b bootstrapStarts
-	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
-	conn := h.guest(t)
-	defer conn.Close()
-	cred := testCredential
-	cred.Environment = map[string]string{"AWS_SECRET_ACCESS_KEY": "forged"}
-	req := hostlink.Request{Credential: &cred, GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: []byte(`{}`)}
-	require.NoError(t, hostlink.NewTestStream(conn).HostSend(req))
-	result := wait(t, done, "agent to refuse the environment")
-	assert.Equal(t, exitRefused, result.code)
-	assert.Error(t, result.err)
-	assert.NotContains(t, result.err.Error(), "forged")
-	assert.Zero(t, b.n)
-}
-
 var testCredential = hostlink.Credential{AccessKeyID: "ASIAGUESTKEY", SecretAccessKey: "guest-secret", SessionToken: "guest-token"}
 
 func envValue(env []string, name string) string {

contracts/cluster/v1/heartbeat.go

@@ -1,6 +1,9 @@
 package clusterv1
 
-import "time"
+import (
+	"encoding/json"
+	"time"
+)
 
 // ClusterStateBucket is the KV bucket holding each daemon's heartbeat alongside
 // other cluster-wide state.
@@ -11,6 +14,27 @@ func HeartbeatKey(node string) string {
 	return "heartbeat." + node
 }
 
+// LivenessWindow is how recently a node must have heartbeated for a reader
+// deciding on a takeover to count it live. It is six heartbeats, compared
+// across node clocks, where NTP skew is small against it.
+const LivenessWindow = 60 * time.Second
+
+// LiveAt reports whether a heartbeat written at written still vouches for its
+// node at now: it does while it is under LivenessWindow old.
+func LiveAt(written, now time.Time) bool {
+	return now.Sub(written) < LivenessWindow
+}
+
+// HeartbeatWritten dates a stored heartbeat value. One that cannot be decoded
+// or dated vouches for nothing, which is an answer rather than a fault.
+func HeartbeatWritten(data []byte) (time.Time, bool) {
+	var beat HeartbeatTimestamp
+	if json.Unmarshal(data, &beat) != nil {
+		return time.Time{}, false
+	}
+	return beat.Time()
+}
+
 // HeartbeatTimestamp is the read side of a heartbeat for node liveness. It
 // decodes only the timestamp, so a reader does not depend on the capacity
 // figures the daemon writes beside it.

contracts/cluster/v1/heartbeat_test.go

@@ -33,3 +33,23 @@ func TestHeartbeatTimestamp_UndatedBeatVouchesForNothing(t *testing.T) {
 		assert.False(t, ok, "timestamp %q", ts)
 	}
 }
+
+// A beat is live strictly under the window, so one exactly the window old is
+// the first a takeover may treat as stopped.
+func TestLiveAt_IsStrictlyUnderTheWindow(t *testing.T) {
+	now := time.Date(2026, 11, 5, 12, 0, 0, 0, time.UTC)
+	assert.True(t, clusterv1.LiveAt(now.Add(-clusterv1.LivenessWindow+time.Nanosecond), now))
+	assert.False(t, clusterv1.LiveAt(now.Add(-clusterv1.LivenessWindow), now))
+	assert.True(t, clusterv1.LiveAt(now.Add(time.Second), now), "a beat from a clock running ahead is live")
+}
+
+func TestHeartbeatWritten_DatesAStoredBeatAndRefusesAnUndecodableOne(t *testing.T) {
+	written, ok := clusterv1.HeartbeatWritten([]byte(`{"node":"node-a","timestamp":"2026-10-30T01:02:03Z"}`))
+	require.True(t, ok)
+	assert.True(t, written.Equal(time.Date(2026, 10, 30, 1, 2, 3, 0, time.UTC)))
+
+	for _, raw := range []string{"not json", `{"node":"node-a"}`} {
+		_, ok := clusterv1.HeartbeatWritten([]byte(raw))
+		assert.False(t, ok, raw)
+	}
+}

contracts/ec2/v1/commands.go

@@ -12,11 +12,6 @@ const (
 	// this contract family. It is the NATS permission/subscription pattern for
 	// the per-instance command route, rather than the broader descendant match.
 	InstanceCommandSubjectWildcard = InstanceCommandSubjectPrefix + "*"
-
-	// HandledHeader is set to "1" on every reply an owner sends to a
-	// modify-instance-metadata-options or set-disable-api-termination command.
-	// An owner that predates those commands cannot set it.
-	HandledHeader = "Spinifex-Ec2-Cmd-Handled"
 )
 
 // InstanceCommandSubject returns the targeted command subject for instanceID.
@@ -28,9 +23,7 @@ func InstanceCommandSubject(instanceID string) string {
 }
 
 // EC2InstanceCommand is the NATS wire format for EC2 instance commands
-// (stop, terminate, start, reboot, attach/detach-volume, attach/detach-eni,
-// IAM profile, spot lineage, tags, monitoring, metadata options and
-// termination protection).
+// (stop, terminate, start, attach/detach-volume, attach/detach-eni).
 type EC2InstanceCommand struct {
 	ID                        string                     `json:"id"`
 	Attributes                EC2CommandAttributes       `json:"attributes"`
@@ -42,9 +35,6 @@ type EC2InstanceCommand struct {
 	SpotLineageData           *SpotLineageData           `json:"spot_lineage_data,omitempty"`
 	InstanceTagsData          *InstanceTagsData          `json:"instance_tags_data,omitempty"`
 	InstanceMonitoringData    *InstanceMonitoringData    `json:"instance_monitoring_data,omitempty"`
-
-	InstanceMetadataOptionsData *InstanceMetadataOptionsData `json:"instance_metadata_options_data,omitempty"`
-	DisableApiTerminationData   *DisableApiTerminationData   `json:"disable_api_termination_data,omitempty"`
 }
 
 // EC2CommandAttributes indicates which action the daemon should perform.
@@ -62,9 +52,6 @@ type EC2CommandAttributes struct {
 	SetInstanceTags             bool `json:"set_instance_tags,omitempty"`
 	RemoveInstanceTags          bool `json:"remove_instance_tags,omitempty"`
 	SetInstanceMonitoring       bool `json:"set_instance_monitoring,omitempty"`
-
-	ModifyInstanceMetadataOptions bool `json:"modify_instance_metadata_options,omitempty"`
-	SetDisableApiTermination      bool `json:"set_disable_api_termination,omitempty"`
 }
 
 // AttachVolumeData carries parameters for an attach-volume command.
@@ -116,23 +103,6 @@ type InstanceMonitoringData struct {
 	Enabled bool `json:"enabled"`
 }
 
-// InstanceMetadataOptionsData carries the requested metadata options for a
-// modify-instance-metadata-options command. Unset fields are left unchanged.
-type InstanceMetadataOptionsData struct {
-	HttpTokens              string `json:"http_tokens,omitempty"`
-	HttpEndpoint            string `json:"http_endpoint,omitempty"`
-	HttpProtocolIpv6        string `json:"http_protocol_ipv6,omitempty"`
-	InstanceMetadataTags    string `json:"instance_metadata_tags,omitempty"`
-	HttpPutResponseHopLimit *int64 `json:"http_put_response_hop_limit,omitempty"`
-}
-
-// DisableApiTerminationData carries the desired termination-protection value
-// for a set-disable-api-termination command. Carried as data so "false" is an
-// explicit value, not the absence of one.
-type DisableApiTerminationData struct {
-	Value bool `json:"value"`
-}
-
 // SpotLineageData carries the SIR id stamped onto a spot-launched instance.
 // Lifecycle is always "spot" for this command, so only the SIR id travels.
 type SpotLineageData struct {

contracts/ec2/v1/commands_test.go

@@ -24,9 +24,6 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 			SetInstanceTags:             true,
 			RemoveInstanceTags:          true,
 			SetInstanceMonitoring:       true,
-
-			ModifyInstanceMetadataOptions: true,
-			SetDisableApiTermination:      true,
 		},
 		AttachVolumeData:          &AttachVolumeData{VolumeID: "vol-1", Device: "/dev/sdf"},
 		DetachVolumeData:          &DetachVolumeData{VolumeID: "vol-2", Device: "/dev/sdg", Force: true},
@@ -36,15 +33,6 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 		SpotLineageData:           &SpotLineageData{SpotInstanceRequestId: "sir-1"},
 		InstanceTagsData:          &InstanceTagsData{Tags: map[string]string{"Name": "example"}, TagKeys: []string{"old"}},
 		InstanceMonitoringData:    &InstanceMonitoringData{Enabled: true},
-
-		InstanceMetadataOptionsData: &InstanceMetadataOptionsData{
-			HttpTokens:              "required",
-			HttpEndpoint:            "enabled",
-			HttpProtocolIpv6:        "disabled",
-			InstanceMetadataTags:    "enabled",
-			HttpPutResponseHopLimit: int64Ptr(3),
-		},
-		DisableApiTerminationData: &DisableApiTerminationData{Value: false},
 	}
 
 	data, err := json.Marshal(command)
@@ -64,9 +52,7 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 			"set_spot_lineage": true,
 			"set_instance_tags": true,
 			"remove_instance_tags": true,
-			"set_instance_monitoring": true,
-			"modify_instance_metadata_options": true,
-			"set_disable_api_termination": true
+			"set_instance_monitoring": true
 		},
 		"attach_volume_data": {"volume_id": "vol-1", "device": "/dev/sdf"},
 		"detach_volume_data": {"volume_id": "vol-2", "device": "/dev/sdg", "force": true},
@@ -75,62 +61,8 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 		"iam_profile_association_data": {"instance_profile_arn": "arn:aws:iam::123456789012:instance-profile/example"},
 		"spot_lineage_data": {"spot_instance_request_id": "sir-1"},
 		"instance_tags_data": {"tags": {"Name": "example"}, "tag_keys": ["old"]},
-		"instance_monitoring_data": {"enabled": true},
-		"instance_metadata_options_data": {
-			"http_tokens": "required",
-			"http_endpoint": "enabled",
-			"http_protocol_ipv6": "disabled",
-			"instance_metadata_tags": "enabled",
-			"http_put_response_hop_limit": 3
-		},
-		"disable_api_termination_data": {"value": false}
-	}`, string(data))
-}
-
-func int64Ptr(v int64) *int64 { return &v }
-
-// An owner that predates the metadata and termination-protection commands
-// sends and receives payloads without the new fields.
-func TestEC2InstanceCommandNewFieldsAreOptionalOnTheWire(t *testing.T) {
-	var n1 EC2InstanceCommand
-	require.NoError(t, json.Unmarshal([]byte(`{
-		"id": "i-0123456789abcdef0",
-		"attributes": {"stop_instance": true},
 		"instance_monitoring_data": {"enabled": true}
-	}`), &n1))
-	require.Equal(t, "i-0123456789abcdef0", n1.ID)
-	require.True(t, n1.Attributes.StopInstance)
-	require.False(t, n1.Attributes.ModifyInstanceMetadataOptions)
-	require.False(t, n1.Attributes.SetDisableApiTermination)
-	require.Nil(t, n1.InstanceMetadataOptionsData)
-	require.Nil(t, n1.DisableApiTerminationData)
-
-	unset, err := json.Marshal(EC2InstanceCommand{ID: "i-1", Attributes: EC2CommandAttributes{StopInstance: true}})
-	require.NoError(t, err)
-	require.NotContains(t, string(unset), "modify_instance_metadata_options")
-	require.NotContains(t, string(unset), "set_disable_api_termination")
-	require.NotContains(t, string(unset), "instance_metadata_options_data")
-	require.NotContains(t, string(unset), "disable_api_termination_data")
-}
-
-func TestInstanceMetadataOptionsDataOmitsUnsetFields(t *testing.T) {
-	data, err := json.Marshal(InstanceMetadataOptionsData{HttpPutResponseHopLimit: int64Ptr(1)})
-	require.NoError(t, err)
-	require.JSONEq(t, `{"http_put_response_hop_limit": 1}`, string(data))
-
-	data, err = json.Marshal(InstanceMetadataOptionsData{})
-	require.NoError(t, err)
-	require.JSONEq(t, `{}`, string(data))
-}
-
-func TestDisableApiTerminationDataKeepsExplicitFalse(t *testing.T) {
-	data, err := json.Marshal(DisableApiTerminationData{Value: false})
-	require.NoError(t, err)
-	require.JSONEq(t, `{"value": false}`, string(data))
-}
-
-func TestHandledHeaderName(t *testing.T) {
-	require.Equal(t, "Spinifex-Ec2-Cmd-Handled", HandledHeader)
+	}`, string(data))
 }
 
 func TestInstanceCommandSubjects(t *testing.T) {

contracts/lambda/v1/README.md

@@ -21,16 +21,8 @@ Tenant code may be able to reach the listener, but it cannot open the agent's co
 **Authentication:** mutual TLS 1.3, bound to the environment ID and realization generation, using material the user runtime cannot read (ADR-0009:S17).
 For each generation the executor creates ephemeral self-signed host and guest certificates whose single URI SAN is `spinifex-lambda://<Region>/<account>/<function ID>/<environment ID>/<generation>/<role>`, with role `host` or `guest`.
 Each end validates the peer certificate against the one pinned peer certificate and separately requires that URI SAN with the opposite role; hostname verification is not relied on.
-ALPN carries the L-GUEST major version, `spinifex-lambda-guest/1`, and an unsupported version fails the handshake. Builds with the `lambdatestlink` tag offer only `spinifex-lambda-testlink/3` on both host and guest, whose first frame carries the execution-role credential and function environment, so a test-link end and a release end fail the handshake; release builds are unchanged.
-In the same builds, the gateway reaches the host daemon on the local abstract socket `spinifex-lambda-testlink-control/4`, and each request carries the runtime-image digest the gateway was built with; the daemon refuses a missing or different digest before starting the attempt, and a gateway and daemon on different socket names never connect.
-On that socket a request's raw payload is at most 6 MiB, checked by the gateway before encoding and by the daemon after decoding.
-Its encoded frame is at most the base64 length of 6 MiB plus 64 KiB of metadata headroom and 64 bytes of field slack, and a receiver checks the frame length before allocating; metadata has no separate bound within the frame.
-The credential and environment travel as their own frame of at most 40 KiB, and the reply has the same payload and frame bounds as the request.
-The daemon holds at most two connections at once, from accept to close, and leaves the rest unaccepted in the listen backlog.
+ALPN carries the L-GUEST major version, `spinifex-lambda-guest/1`, and an unsupported version fails the handshake. Builds with the `lambdatestlink` tag offer only `spinifex-lambda-testlink/2` on both host and guest, whose first frame carries the execution-role credential, so a test-link end and a release end fail the handshake; release builds are unchanged.
 Under TLS 1.3 the guest's handshake can complete before the host refuses its certificate, so the guest treats the connection as accepted only after the host's first message.
-In the same builds, the function owner moves a created function with stored code from `Pending` to `Active` once this node's daemon serves the pinned image and the cluster configuration has exactly one node; a code copy over the test-link bound stays `Pending`, and `StateReason` names the unmet condition.
-Release builds never leave `Pending`.
-The Mulga deploy role installs a test-link build with `deploy_lambda_testlink=true`; this profile does not meet Q-199 fail-closed revocation, and its guests take management addresses whose record can expire (mulga-0hsub).
 
 **Bootstrap material:** the executor passes these fw_cfg entries under `opt/spinifex/lambda/`, and the agent reads them as root and unloads `qemu_fw_cfg` before user code starts.
 
@@ -44,8 +36,6 @@ The Mulga deploy role installs a test-link build with `deploy_lambda_testlink=tr
 
 The guest's own interface settings come from `opt/spinifex/netcfg`, the entry other system VMs use.
 The agent unloads `qemu_fw_cfg` even when an entry is missing, and refuses to start if the fw_cfg tree is still readable afterwards.
-On a VMM without fw_cfg (the test-link Firecracker executor), the executor attaches the same entries as a tar archive on the second, writable virtio-blk disk, laid out as `by_name/<entry>/raw`.
-The runtime image's init extracts it to a root-only tmpfs at `/spinifex-cfg`, and the agent reads the entries there, then overwrites the whole disk with zeros and removes the tree; it refuses to start if the tree is still present.
 Until the message format exists, the agent authenticates and holds the connection but accepts no host message and starts no user code.
 
 **Authority:** one environment's attempt only.

docs/testing/e2e-coverage.md

@@ -108,8 +108,6 @@ Three entry points, in the order they run:
 
 **Its own suite and its own cell, for instancerecovery's reasons plus one.** It takes a node away, so anything sharing the environment reports that outage as its own failure; it needs its own VPC, public subnet and a backend placement no shared fixture describes; and a failure here has to read as "the load balancer did not survive its host" rather than as one more red subtest in a suite about load balancer features.
 
-**A dead data NIC is the other half, and there the state does change.** `LBSelfHealsADeadDataNIC` disables one data NIC of a running load balancer guest through a second QMP client and runs for an internet-facing NLB and an internal ALB. The backends and an unrelated customer instance in the same subnet are launched more than 21 minutes before the fault, so a repair that reached a customer instance would find one the platform has long stopped expecting to change. A state watch samples `DescribeLoadBalancers` every two seconds and requires the sequence `active`, `active_impaired`, `active` with nothing in between, while the internet-facing run also watches the name continuously and serves traffic through it within five minutes. Afterwards the guest, its interfaces and its address are unchanged but its process is not, the backends and the bystander keep their instance ID, launch time and process (ADR-0003:S7), and the daemon journals exactly one `ELBv2 data-path repair started` line carrying the fields an operator needs. The internal ALB has no client inside the VPC, so it asserts the state sequence and target health rather than traffic through its name.
-
 ### `iam` — everything IAM and STS
 
 Policy evaluation against live requests rather than unit-level: a deny applying to the next request without a new credential, resource-scoped grants naming one key pair, a user rename needing `iam:UpdateUser` on both the old and the new name, `aws:SourceIp` seeing the real client address, assumed-role sessions carrying role policies, and revocation stopping one key alone.
@@ -131,7 +129,7 @@ The client reports on its serial console because the runner has no route into th
 
 ### `eks`, `ecs`, `ecr` — container services
 
-`eks`: cluster create/delete, access entries, addon delivery, `GetToken`, kubeconfig artifacts, the EBS CSI volume path, an IRSA web-identity exchange under the stock AWS trust policy (`StringEquals` on the issuer host-path `:sub`/`:aud`, a mismatched `:sub` denied with `AccessDenied`, a `StringLike` `:sub` granted), and an IRSA pod whose spec names only a ServiceAccount with the role-arn annotation, where the pod-identity webhook injects the token volume, `AWS_*` environment, STS endpoint and CA bundle and the pod assumes the role, while a pod of an unannotated ServiceAccount is left untouched. The unmodified aws-load-balancer-controller chart 1.11.0 installs with the values an AWS user passes plus the gateway endpoint override, the Shield and WAF switches and the single-subnet feature gate, and reconciles an Ingress into an active ALB. Worker IMDS is covered too: a managed nodegroup worker reports IMDSv2 required with hop limit 2, a pod with no IRSA resolves to the node role through IMDS, and a nodegroup built from a launch template takes the template's metadata options. The private-endpoint NIC of the cluster's endpoint NLB is disabled through QMP and every worker must be Ready again within ten minutes while the cluster stays `ACTIVE` and the NLB keeps its guest instance. Control-plane isolation is covered as well: no control-plane host registers as a Node, a DaemonSet tolerating every taint runs on workers only, metrics-server is Ready, and logs work for every kube-system pod. `ecs`: clusters, container instances, the three network modes (`awsvpc`, `bridge`, `host`), task-role credentials fetched from the container credentials endpoint, a service behind an ELB, and ENI release on terminate. `ecr`: control-plane CRUD plus a real OCI data plane — docker login/build/push/pull round trips, multi-layer images, and raw blob HTTP.
+`eks`: cluster create/delete, access entries, addon delivery, `GetToken`, kubeconfig artifacts, the EBS CSI volume path, an IRSA web-identity exchange under the stock AWS trust policy (`StringEquals` on the issuer host-path `:sub`/`:aud`, a mismatched `:sub` denied with `AccessDenied`, a `StringLike` `:sub` granted), and an IRSA pod whose spec names only a ServiceAccount with the role-arn annotation, where the pod-identity webhook injects the token volume, `AWS_*` environment, STS endpoint and CA bundle and the pod assumes the role, while a pod of an unannotated ServiceAccount is left untouched. The unmodified aws-load-balancer-controller chart 1.11.0 installs with the values an AWS user passes plus the gateway endpoint override, the Shield and WAF switches and the single-subnet feature gate, and reconciles an Ingress into an active ALB. Worker IMDS is covered too: a managed nodegroup worker reports IMDSv2 required with hop limit 2, a pod with no IRSA resolves to the node role through IMDS, and a nodegroup built from a launch template takes the template's metadata options. Control-plane isolation is covered as well: no control-plane host registers as a Node, a DaemonSet tolerating every taint runs on workers only, metrics-server is Ready, and logs work for every kube-system pod. `ecs`: clusters, container instances, the three network modes (`awsvpc`, `bridge`, `host`), task-role credentials fetched from the container credentials endpoint, a service behind an ELB, and ENI release on terminate. `ecr`: control-plane CRUD plus a real OCI data plane — docker login/build/push/pull round trips, multi-layer images, and raw blob HTTP.
 
 ### `rds` — managed databases
 

scripts/build-microvm-image.sh

@@ -209,7 +209,6 @@ echo "[build-microvm-image] nginx: binary + stream module present"
 #   virtio*.ko*    — virtio bus, net, blk, rng, console, and helpers
 #   *fw_cfg*.ko*   — QEMU fw_cfg sysfs driver (delivers boot blobs)
 #   *failover*.ko* — net_failover + failover (virtio_net deps for live migration)
-#   af_packet.ko*  — AF_PACKET sockets (lb-agent's per-NIC gateway ARP probe)
 # All other loadable modules are removed. Drivers compiled into the kernel
 # (CONFIG_VIRTIO_MMIO=y etc.) are unaffected — they live in vmlinuz, not here.
 echo "[build-microvm-image] stripping kernel modules to virtio+fw_cfg only..."
@@ -223,7 +222,6 @@ for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
     find "$kernel_dir" -name "virtio*.ko.gz" -o -name "virtio*.ko" \
                        -o -name "*fw_cfg*.ko.gz" -o -name "*fw_cfg*.ko" \
                        -o -name "*failover*.ko.gz" -o -name "*failover*.ko" \
-                       -o -name "af_packet.ko.gz" -o -name "af_packet.ko" \
         2>/dev/null | while read -r mod; do
         rel="${mod#"$kernel_dir/"}"
         dest_dir="$tmp_keep/$(dirname "$rel")"
@@ -242,15 +240,6 @@ for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
     done)
     rm -rf "$tmp_keep"
 
-    # lb-agent's data-path probe needs AF_PACKET; without it every LB would
-    # report a dead NIC. Accept a kernel with CONFIG_PACKET built in.
-    if ! find "$kernel_dir" -name "af_packet.ko*" | grep -q .; then
-        if ! grep -q "CONFIG_PACKET=y" "$KERNEL_CONFIG" 2>/dev/null; then
-            echo "ERROR: af_packet.ko missing from the kept modules — lb-agent's data-path probe cannot open its socket" >&2
-            exit 1
-        fi
-    fi
-
     # Decompress kept modules to plain .ko. Alpine ships .ko.gz, but a host
     # kmod that lacks gzip support (e.g. Debian trixie's kmod 34) silently
     # produces an EMPTY modules.dep from compressed modules — the guest's

spinifex/agents/elbv2/agent.go

@@ -21,7 +21,6 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
-	"sync"
 	"syscall"
 	"time"
 
@@ -95,18 +94,6 @@ type Agent struct {
 	statsFn func(socketPath string) ([]ServerStatus, error)
 	// For testing: override the active health prober (nginx/NLB).
 	probeFn func(targets []HealthTarget) []ServerStatus
-
-	// Data-path probe: netcfg and sysfs locations, the ARP exchange, the
-	// instance ID lookup and per-NIC miss counters (overridable in tests).
-	netcfgPath   string
-	sysNetDir    string
-	probeWait    time.Duration
-	arpProbeFn   arpProbeFn
-	instanceIDFn func() (string, error)
-	instanceID   string
-	idSkip       int
-	probeMu      sync.Mutex
-	probeStates  map[string]*nicProbeState
 }
 
 // New creates a new LB agent for the given load balancer.
@@ -163,12 +150,6 @@ func New(lbID, gatewayURL, accessKey, secretKey, region string) (*Agent, error)
 		reloadNginxFn: reloadNginx,
 		statsFn:       queryHAProxyStats,
 		probeFn:       probeHealthTargets,
-		netcfgPath:    netcfgPath,
-		sysNetDir:     sysClassNet,
-		probeWait:     probeWait,
-		arpProbeFn:    arpProbe,
-		instanceIDFn:  fetchInstanceID,
-		probeStates:   map[string]*nicProbeState{},
 	}, nil
 }
 
@@ -226,7 +207,7 @@ func (a *Agent) tick() {
 		}
 	}
 
-	resp, err := a.sendHeartbeat(servers, a.dataPathEvidence())
+	resp, err := a.sendHeartbeat(servers)
 	if err != nil {
 		// Include the dial target so the serial console disambiguates a
 		// transport failure (never reached AWSGW — "send request: dial ...")
@@ -307,7 +288,7 @@ func (a *Agent) enginePaths(engine string) (configPath, pidPath, certDir string,
 }
 
 // sendHeartbeat sends a heartbeat with health report to the gateway.
-func (a *Agent) sendHeartbeat(servers []ServerStatus, ev dataPathEvidence) (*heartbeatResponse, error) {
+func (a *Agent) sendHeartbeat(servers []ServerStatus) (*heartbeatResponse, error) {
 	params := url.Values{}
 	params.Set("Action", "LBAgentHeartbeat")
 	params.Set("Version", "2015-12-01")
@@ -319,7 +300,6 @@ func (a *Agent) sendHeartbeat(servers []ServerStatus, ev dataPathEvidence) (*hea
 		params.Set("Servers.member."+idx+".Server", s.Server)
 		params.Set("Servers.member."+idx+".Status", s.Status)
 	}
-	ev.addTo(params)
 
 	body, err := a.signedPost(params)
 	if err != nil {

spinifex/agents/elbv2/arp_linux.go

@@ -1,62 +0,0 @@
-//go:build linux
-
-package elbv2
-
-import (
-	"errors"
-	"fmt"
-	"net"
-	"net/netip"
-	"syscall"
-	"time"
-)
-
-// arpProbe sends one ARP request for gateway out of iface on an AF_PACKET
-// socket bound to it and waits for the gateway's reply on the same socket.
-func arpProbe(iface string, mac net.HardwareAddr, src, gateway netip.Addr, wait time.Duration) error {
-	ifi, err := net.InterfaceByName(iface)
-	if err != nil {
-		return fmt.Errorf("interface %s: %w", iface, err)
-	}
-
-	proto := htons(syscall.ETH_P_ARP)
-	fd, err := syscall.Socket(syscall.AF_PACKET, syscall.SOCK_RAW, int(proto))
-	if err != nil {
-		return fmt.Errorf("packet socket: %w", err)
-	}
-	defer syscall.Close(fd)
-
-	addr := &syscall.SockaddrLinklayer{Protocol: proto, Ifindex: ifi.Index}
-	if err := syscall.Bind(fd, addr); err != nil {
-		return fmt.Errorf("bind %s: %w", iface, err)
-	}
-
-	deadline := time.Now().Add(wait)
-	if err := syscall.Sendto(fd, buildARPRequest(mac, src, gateway), 0, addr); err != nil {
-		return fmt.Errorf("send ARP request: %w", err)
-	}
-
-	buf := make([]byte, 128)
-	for {
-		left := time.Until(deadline)
-		if left <= 0 {
-			return errARPNoReply
-		}
-		tv := syscall.NsecToTimeval(int64(left))
-		if err := syscall.SetsockoptTimeval(fd, syscall.SOL_SOCKET, syscall.SO_RCVTIMEO, &tv); err != nil {
-			return fmt.Errorf("set receive deadline: %w", err)
-		}
-		n, _, err := syscall.Recvfrom(fd, buf, 0)
-		if err != nil {
-			if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EINTR) {
-				continue
-			}
-			return fmt.Errorf("receive ARP reply: %w", err)
-		}
-		if isARPReplyFrom(buf[:n], mac, gateway) {
-			return nil
-		}
-	}
-}
-
-func htons(v uint16) uint16 { return v<<8 | v>>8 }

spinifex/agents/elbv2/arp_other.go

@@ -1,15 +0,0 @@
-//go:build !linux
-
-package elbv2
-
-import (
-	"errors"
-	"net"
-	"net/netip"
-	"time"
-)
-
-// arpProbe is only meaningful inside the Linux guest.
-func arpProbe(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error {
-	return errors.New("ARP probe is only supported on linux")
-}

spinifex/agents/elbv2/datapath.go

@@ -1,303 +0,0 @@
-package elbv2
-
-import (
-	"bufio"
-	"bytes"
-	"encoding/binary"
-	"errors"
-	"fmt"
-	"net"
-	"net/netip"
-	"os"
-	"path/filepath"
-	"sort"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-)
-
-const (
-	// netcfgPath is the fw_cfg blob the microVM init script reads to configure NICs.
-	netcfgPath = "/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw"
-
-	sysClassNet = "/sys/class/net"
-
-	// probeFailureThreshold is the number of consecutive missed probes after
-	// which a NIC is reported as failing (about 15 s at the poll interval).
-	probeFailureThreshold = 3
-
-	// arpExchangeWait bounds one ARP exchange. probeWait bounds how long a tick
-	// waits for all probes, so a stuck probe never delays the heartbeat.
-	arpExchangeWait = time.Second
-	probeWait       = 1500 * time.Millisecond
-)
-
-// nicConfig is one NIC of the netcfg blob that has a gateway to probe.
-type nicConfig struct {
-	MAC     string
-	Addr    netip.Addr
-	Gateway netip.Addr
-}
-
-// NICReport is the per-NIC data-path evidence carried in the heartbeat.
-type NICReport struct {
-	MAC                 string
-	ProbeOK             bool
-	ConsecutiveFailures int64
-	RxPackets           int64
-	TxPackets           int64
-	RxDropped           int64
-	TxDropped           int64
-}
-
-// arpProbeFn sends one ARP request for gateway out of iface and returns nil
-// when the gateway's reply arrives within timeout.
-type arpProbeFn func(iface string, mac net.HardwareAddr, src, gateway netip.Addr, timeout time.Duration) error
-
-// parseNetcfg reads the KEY=value netcfg blob and returns the NICs that have a
-// gateway, in NIC order. The mgmt NIC carries no gateway and is skipped, as
-// is any NIC without a usable MAC, address or gateway.
-func parseNetcfg(blob []byte) []nicConfig {
-	type raw struct{ mac, cidr, gw string }
-	byIndex := map[int]*raw{}
-
-	sc := bufio.NewScanner(bytes.NewReader(blob))
-	for sc.Scan() {
-		line := strings.TrimSpace(sc.Text())
-		key, val, ok := strings.Cut(line, "=")
-		if !ok || !strings.HasPrefix(key, "NIC") {
-			continue
-		}
-		idxStr, field, ok := strings.Cut(strings.TrimPrefix(key, "NIC"), "_")
-		if !ok {
-			continue
-		}
-		idx, err := strconv.Atoi(idxStr)
-		if err != nil {
-			continue
-		}
-		r := byIndex[idx]
-		if r == nil {
-			r = &raw{}
-			byIndex[idx] = r
-		}
-		switch field {
-		case "MAC":
-			r.mac = strings.ToLower(val)
-		case "CIDR":
-			r.cidr = val
-		case "GW":
-			r.gw = val
-		}
-	}
-
-	idxs := make([]int, 0, len(byIndex))
-	for i := range byIndex {
-		idxs = append(idxs, i)
-	}
-	sort.Ints(idxs)
-
-	var out []nicConfig
-	for _, i := range idxs {
-		r := byIndex[i]
-		if r.mac == "" || r.cidr == "" || r.gw == "" {
-			continue
-		}
-		prefix, err := netip.ParsePrefix(r.cidr)
-		if err != nil || !prefix.Addr().Is4() {
-			continue
-		}
-		gw, err := netip.ParseAddr(r.gw)
-		if err != nil || !gw.Is4() {
-			continue
-		}
-		out = append(out, nicConfig{MAC: r.mac, Addr: prefix.Addr(), Gateway: gw})
-	}
-	return out
-}
-
-// ifaceByMAC returns the interface under sysNetDir whose address is mac.
-func ifaceByMAC(sysNetDir, mac string) (string, error) {
-	entries, err := os.ReadDir(sysNetDir)
-	if err != nil {
-		return "", err
-	}
-	for _, e := range entries {
-		b, err := os.ReadFile(filepath.Join(sysNetDir, e.Name(), "address"))
-		if err != nil {
-			continue
-		}
-		if strings.EqualFold(strings.TrimSpace(string(b)), mac) {
-			return e.Name(), nil
-		}
-	}
-	return "", fmt.Errorf("no interface with MAC %s", mac)
-}
-
-// readCounters fills the sysfs statistics of iface into r.
-func readCounters(sysNetDir, iface string, r *NICReport) {
-	read := func(name string) int64 {
-		b, err := os.ReadFile(filepath.Join(sysNetDir, iface, "statistics", name))
-		if err != nil {
-			return 0
-		}
-		v, _ := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64)
-		return v
-	}
-	r.RxPackets = read("rx_packets")
-	r.TxPackets = read("tx_packets")
-	r.RxDropped = read("rx_dropped")
-	r.TxDropped = read("tx_dropped")
-}
-
-var errARPNoReply = errors.New("no ARP reply from the gateway")
-
-// buildARPRequest returns the Ethernet frame asking who has target.
-func buildARPRequest(mac net.HardwareAddr, src, target netip.Addr) []byte {
-	f := make([]byte, 0, 42)
-	f = append(f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff)
-	f = append(f, mac...)
-	f = binary.BigEndian.AppendUint16(f, 0x0806)
-	f = binary.BigEndian.AppendUint16(f, 1)      // Ethernet
-	f = binary.BigEndian.AppendUint16(f, 0x0800) // IPv4
-	f = append(f, 6, 4)
-	f = binary.BigEndian.AppendUint16(f, 1) // request
-	f = append(f, mac...)
-	s := src.As4()
-	f = append(f, s[:]...)
-	f = append(f, 0, 0, 0, 0, 0, 0)
-	t := target.As4()
-	f = append(f, t[:]...)
-	return f
-}
-
-// isARPReplyFrom reports whether frame is an ARP reply sent by sender to mac.
-func isARPReplyFrom(frame []byte, mac net.HardwareAddr, sender netip.Addr) bool {
-	if len(frame) < 42 || binary.BigEndian.Uint16(frame[12:14]) != 0x0806 {
-		return false
-	}
-	if binary.BigEndian.Uint16(frame[20:22]) != 2 || !bytes.Equal(frame[32:38], mac) {
-		return false
-	}
-	s := sender.As4()
-	return bytes.Equal(frame[28:32], s[:])
-}
-
-// nicProbeState carries the miss counter of one NIC across ticks. A probe that
-// has not returned is still in flight, and counts as a miss without starting
-// another goroutine behind it.
-type nicProbeState struct {
-	mu       sync.Mutex
-	failures int64
-	inflight bool
-}
-
-type probeResult struct {
-	ok  bool
-	set bool
-}
-
-// probeNICs probes every gateway-bearing data NIC once, in parallel, and
-// returns within probeWait however long a probe blocks.
-func (a *Agent) probeNICs() []NICReport {
-	blob, err := os.ReadFile(a.netcfgPath)
-	if err != nil {
-		return nil
-	}
-	nics := parseNetcfg(blob)
-	if len(nics) == 0 {
-		return nil
-	}
-
-	reports := make([]NICReport, len(nics))
-	results := make([]chan probeResult, len(nics))
-	for i, n := range nics {
-		results[i] = make(chan probeResult, 1)
-		st := a.probeState(n.MAC)
-
-		st.mu.Lock()
-		busy := st.inflight
-		if !busy {
-			st.inflight = true
-		}
-		st.mu.Unlock()
-		if busy {
-			results[i] <- probeResult{set: true}
-			continue
-		}
-
-		go func(n nicConfig, st *nicProbeState, out chan<- probeResult) {
-			ok := a.probeOne(n)
-			st.mu.Lock()
-			st.inflight = false
-			st.mu.Unlock()
-			out <- probeResult{ok: ok, set: true}
-		}(n, st, results[i])
-	}
-
-	deadline := time.NewTimer(a.probeWait)
-	defer deadline.Stop()
-	expired := false
-	for i, n := range nics {
-		var res probeResult
-		if expired {
-			select {
-			case res = <-results[i]:
-			default:
-			}
-		} else {
-			select {
-			case res = <-results[i]:
-			case <-deadline.C:
-				// Out of time: an unfinished probe is a miss for this tick.
-				expired = true
-			}
-		}
-
-		st := a.probeState(n.MAC)
-		st.mu.Lock()
-		if res.ok {
-			st.failures = 0
-		} else {
-			st.failures++
-		}
-		failures := st.failures
-		st.mu.Unlock()
-
-		r := NICReport{
-			MAC:                 n.MAC,
-			ProbeOK:             failures < probeFailureThreshold,
-			ConsecutiveFailures: failures,
-		}
-		if iface, err := ifaceByMAC(a.sysNetDir, n.MAC); err == nil {
-			readCounters(a.sysNetDir, iface, &r)
-		}
-		reports[i] = r
-	}
-	return reports
-}
-
-// probeOne runs one ARP exchange for n, false on any failure.
-func (a *Agent) probeOne(n nicConfig) bool {
-	iface, err := ifaceByMAC(a.sysNetDir, n.MAC)
-	if err != nil {
-		return false
-	}
-	hw, err := net.ParseMAC(n.MAC)
-	if err != nil {
-		return false
-	}
-	return a.arpProbeFn(iface, hw, n.Addr, n.Gateway, arpExchangeWait) == nil
-}
-
-func (a *Agent) probeState(mac string) *nicProbeState {
-	a.probeMu.Lock()
-	defer a.probeMu.Unlock()
-	st := a.probeStates[mac]
-	if st == nil {
-		st = &nicProbeState{}
-		a.probeStates[mac] = st
-	}
-	return st
-}

spinifex/agents/elbv2/datapath_test.go

@@ -1,484 +0,0 @@
-package elbv2
-
-import (
-	"errors"
-	"fmt"
-	"net"
-	"net/http"
-	"net/http/httptest"
-	"net/netip"
-	"net/url"
-	"os"
-	"path/filepath"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-)
-
-const (
-	testDataMAC = "02:00:00:aa:00:01"
-	testMgmtMAC = "02:00:00:aa:00:02"
-	testNetcfg  = "NIC0_MAC=" + testDataMAC + "\nNIC0_CIDR=10.0.1.5/24\nNIC0_GW=10.0.1.1\nNIC0_DEFAULT=1\n" +
-		"NIC1_MAC=" + testMgmtMAC + "\nNIC1_CIDR=10.255.0.9/24\nNIC1_DEFAULT=0\n"
-)
-
-// datapathAgent returns a test agent whose netcfg and sysfs live in a temp dir.
-func datapathAgent(t *testing.T, gwURL, netcfg string) *Agent {
-	t.Helper()
-	a := newTestAgent(t, gwURL)
-	dir := t.TempDir()
-
-	a.netcfgPath = filepath.Join(dir, "netcfg")
-	if err := os.WriteFile(a.netcfgPath, []byte(netcfg), 0o600); err != nil {
-		t.Fatal(err)
-	}
-
-	a.sysNetDir = filepath.Join(dir, "net")
-	for name, mac := range map[string]string{"ens3": testDataMAC, "ens4": testMgmtMAC} {
-		stats := filepath.Join(a.sysNetDir, name, "statistics")
-		if err := os.MkdirAll(stats, 0o755); err != nil {
-			t.Fatal(err)
-		}
-		if err := os.WriteFile(filepath.Join(a.sysNetDir, name, "address"), []byte(mac+"\n"), 0o600); err != nil {
-			t.Fatal(err)
-		}
-		for file, v := range map[string]string{"rx_packets": "100", "tx_packets": "200", "rx_dropped": "3", "tx_dropped": "4"} {
-			if err := os.WriteFile(filepath.Join(stats, file), []byte(v+"\n"), 0o600); err != nil {
-				t.Fatal(err)
-			}
-		}
-	}
-
-	a.instanceIDFn = func() (string, error) { return "i-0123456789abcdef0", nil }
-	return a
-}
-
-func TestParseNetcfg_SkipsMgmtNICAndMissingGateway(t *testing.T) {
-	blob := testNetcfg +
-		"NIC2_MAC=02:00:00:aa:00:03\nNIC2_CIDR=10.0.2.5/24\nNIC2_GW=10.0.2.1\nNIC2_DEFAULT=0\n" +
-		"NIC3_MAC=02:00:00:aa:00:04\nNIC3_DHCP=1\nNIC3_DEFAULT=0\n" +
-		"NIC4_CIDR=10.0.4.5/24\nNIC4_GW=10.0.4.1\n" +
-		"NIC5_MAC=02:00:00:aa:00:06\nNIC5_CIDR=not-a-cidr\nNIC5_GW=10.0.5.1\n"
-
-	got := parseNetcfg([]byte(blob))
-	if len(got) != 2 {
-		t.Fatalf("got %d NICs, want the 2 with a gateway: %+v", len(got), got)
-	}
-	if got[0].MAC != testDataMAC || got[0].Gateway != netip.MustParseAddr("10.0.1.1") || got[0].Addr != netip.MustParseAddr("10.0.1.5") {
-		t.Errorf("first NIC = %+v", got[0])
-	}
-	if got[1].MAC != "02:00:00:aa:00:03" || got[1].Gateway != netip.MustParseAddr("10.0.2.1") {
-		t.Errorf("second NIC = %+v", got[1])
-	}
-}
-
-func TestParseNetcfg_OrdersByNICIndexAndLowercasesMAC(t *testing.T) {
-	blob := "NIC10_MAC=02:00:00:AA:00:0A\nNIC10_CIDR=10.0.9.5/24\nNIC10_GW=10.0.9.1\n" +
-		"NIC2_MAC=02:00:00:AA:00:02\nNIC2_CIDR=10.0.2.5/24\nNIC2_GW=10.0.2.1\n"
-	got := parseNetcfg([]byte(blob))
-	if len(got) != 2 || got[0].MAC != "02:00:00:aa:00:02" || got[1].MAC != "02:00:00:aa:00:0a" {
-		t.Fatalf("order or case wrong: %+v", got)
-	}
-}
-
-func TestIfaceByMACAndCounters(t *testing.T) {
-	a := datapathAgent(t, "http://unused", testNetcfg)
-
-	iface, err := ifaceByMAC(a.sysNetDir, "02:00:00:AA:00:01")
-	if err != nil || iface != "ens3" {
-		t.Fatalf("iface = %q, err = %v", iface, err)
-	}
-	if _, err := ifaceByMAC(a.sysNetDir, "02:00:00:ff:ff:ff"); err == nil {
-		t.Error("an unknown MAC must not resolve to an interface")
-	}
-
-	var r NICReport
-	readCounters(a.sysNetDir, iface, &r)
-	if r.RxPackets != 100 || r.TxPackets != 200 || r.RxDropped != 3 || r.TxDropped != 4 {
-		t.Errorf("counters = %+v", r)
-	}
-}
-
-func TestARPFrames_RequestShapeAndReplyMatching(t *testing.T) {
-	mac, _ := net.ParseMAC(testDataMAC)
-	src, gw := netip.MustParseAddr("10.0.1.5"), netip.MustParseAddr("10.0.1.1")
-
-	req := buildARPRequest(mac, src, gw)
-	if len(req) != 42 || req[12] != 0x08 || req[13] != 0x06 || req[21] != 1 {
-		t.Fatalf("request frame malformed: % x", req)
-	}
-	if string(req[38:42]) != string(gw.AsSlice()) {
-		t.Errorf("target IP not the gateway: % x", req[38:42])
-	}
-
-	reply := make([]byte, 42)
-	copy(reply[0:6], mac)
-	reply[12], reply[13] = 0x08, 0x06
-	reply[21] = 2
-	copy(reply[28:32], gw.AsSlice())
-	copy(reply[32:38], mac)
-	if !isARPReplyFrom(reply, mac, gw) {
-		t.Error("a reply from the gateway to our MAC must match")
-	}
-	if isARPReplyFrom(reply, mac, netip.MustParseAddr("10.0.1.99")) {
-		t.Error("a reply from another host must not match")
-	}
-	reply[21] = 1
-	if isARPReplyFrom(reply, mac, gw) {
-		t.Error("a request must not count as a reply")
-	}
-	if isARPReplyFrom(reply[:20], mac, gw) {
-		t.Error("a short frame must not match")
-	}
-}
-
-func TestProbeNICs_ProbesOnlyTheGatewayNIC(t *testing.T) {
-	a := datapathAgent(t, "http://unused", testNetcfg)
-	var mu sync.Mutex
-	var probed []string
-	a.arpProbeFn = func(iface string, _ net.HardwareAddr, src, gw netip.Addr, _ time.Duration) error {
-		mu.Lock()
-		defer mu.Unlock()
-		probed = append(probed, fmt.Sprintf("%s %s>%s", iface, src, gw))
-		return nil
-	}
-
-	reports := a.probeNICs()
-	if len(reports) != 1 || reports[0].MAC != testDataMAC || !reports[0].ProbeOK || reports[0].ConsecutiveFailures != 0 {
-		t.Fatalf("reports = %+v", reports)
-	}
-	if len(probed) != 1 || probed[0] != "ens3 10.0.1.5>10.0.1.1" {
-		t.Errorf("probed = %v, want only the data NIC", probed)
-	}
-	if reports[0].RxPackets != 100 || reports[0].TxDropped != 4 {
-		t.Errorf("counters missing from report: %+v", reports[0])
-	}
-}
-
-func TestProbeNICs_TwoMissesThenSuccessNeverReportsFailing(t *testing.T) {
-	a := datapathAgent(t, "http://unused", testNetcfg)
-	results := []error{errARPNoReply, errARPNoReply, nil, errARPNoReply, errARPNoReply, nil}
-	var n atomic.Int32
-	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error {
-		return results[n.Add(1)-1]
-	}
-
-	for i, want := range []int64{1, 2, 0, 1, 2, 0} {
-		r := a.probeNICs()
-		if len(r) != 1 || !r[0].ProbeOK || r[0].ConsecutiveFailures != want {
-			t.Fatalf("tick %d: report = %+v, want ok with %d failures", i, r, want)
-		}
-	}
-}
-
-func TestProbeNICs_ThreeMissesReportFailingAndRecoveryClears(t *testing.T) {
-	a := datapathAgent(t, "http://unused", testNetcfg)
-	var fail atomic.Bool
-	fail.Store(true)
-	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error {
-		if fail.Load() {
-			return errARPNoReply
-		}
-		return nil
-	}
-
-	for i := 1; i <= 4; i++ {
-		r := a.probeNICs()
-		wantOK := i < 3 // failing from the third consecutive miss
-		if r[0].ProbeOK != wantOK || r[0].ConsecutiveFailures != int64(i) {
-			t.Fatalf("miss %d: report = %+v, want ProbeOK=%v", i, r[0], wantOK)
-		}
-	}
-
-	fail.Store(false)
-	if r := a.probeNICs(); !r[0].ProbeOK || r[0].ConsecutiveFailures != 0 {
-		t.Errorf("a successful probe must clear the failure: %+v", r[0])
-	}
-}
-
-func TestProbeNICs_BlockedProbeIsBoundedAndCountsAsMiss(t *testing.T) {
-	a := datapathAgent(t, "http://unused", testNetcfg)
-	a.probeWait = 50 * time.Millisecond
-	release := make(chan struct{})
-	defer close(release)
-	var calls atomic.Int32
-	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error {
-		calls.Add(1)
-		<-release
-		return nil
-	}
-
-	for i := 1; i <= 3; i++ {
-		start := time.Now()
-		r := a.probeNICs()
-		if d := time.Since(start); d > time.Second {
-			t.Fatalf("tick %d waited %s on a stuck probe", i, d)
-		}
-		if len(r) != 1 || r[0].ConsecutiveFailures != int64(i) {
-			t.Fatalf("tick %d: report = %+v, want %d misses", i, r, i)
-		}
-	}
-	if got := calls.Load(); got != 1 {
-		t.Errorf("a stuck probe was started %d times, want 1 (no goroutine pile-up)", got)
-	}
-}
-
-func TestTick_BlockedProbeDoesNotDelayHeartbeatBeyondOneTick(t *testing.T) {
-	var got atomic.Pointer[url.Values]
-	gw := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		_ = r.ParseForm()
-		if r.FormValue("Action") == "LBAgentHeartbeat" {
-			v := r.Form
-			got.Store(&v)
-		}
-		fmt.Fprint(w, `<LBAgentHeartbeatResponse><LBAgentHeartbeatResult><Status>active</Status><ConfigHash>h1</ConfigHash></LBAgentHeartbeatResult></LBAgentHeartbeatResponse>`)
-	}))
-	defer gw.Close()
-
-	a := datapathAgent(t, gw.URL, testNetcfg)
-	a.localConfigHash = "h1"
-	a.probeWait = 50 * time.Millisecond
-	release := make(chan struct{})
-	defer close(release)
-	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error {
-		<-release
-		return nil
-	}
-
-	start := time.Now()
-	a.tick()
-	if d := time.Since(start); d >= pollInterval {
-		t.Fatalf("tick took %s with a stuck probe, want under one %s tick", d, pollInterval)
-	}
-	if got.Load() == nil {
-		t.Fatal("the heartbeat was never sent")
-	}
-}
-
-func TestHeartbeat_EncodesEvidenceWhenPresent(t *testing.T) {
-	var got atomic.Pointer[url.Values]
-	gw := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		_ = r.ParseForm()
-		v := r.Form
-		got.Store(&v)
-		fmt.Fprint(w, `<LBAgentHeartbeatResponse><LBAgentHeartbeatResult><Status>active</Status><ConfigHash>h1</ConfigHash></LBAgentHeartbeatResult></LBAgentHeartbeatResponse>`)
-	}))
-	defer gw.Close()
-
-	a := datapathAgent(t, gw.URL, testNetcfg)
-	a.localConfigHash = "h1"
-	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error { return errARPNoReply }
-
-	a.tick()
-	v := got.Load()
-	if v == nil {
-		t.Fatal("no heartbeat received")
-	}
-	want := map[string]string{
-		"InstanceID":                        "i-0123456789abcdef0",
-		"NICs.member.1.MAC":                 testDataMAC,
-		"NICs.member.1.ProbeOK":             "true",
-		"NICs.member.1.ConsecutiveFailures": "1",
-		"NICs.member.1.RxPackets":           "100",
-		"NICs.member.1.TxPackets":           "200",
-		"NICs.member.1.RxDropped":           "3",
-		"NICs.member.1.TxDropped":           "4",
-	}
-	for k, w := range want {
-		if g := v.Get(k); g != w {
-			t.Errorf("%s = %q, want %q", k, g, w)
-		}
-	}
-	if v.Get("NICs.member.2.MAC") != "" {
-		t.Error("the mgmt NIC must not be reported")
-	}
-}
-
-func TestHeartbeat_OmitsEvidenceWithoutNetcfg(t *testing.T) {
-	var got atomic.Pointer[url.Values]
-	gw := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		_ = r.ParseForm()
-		v := r.Form
-		got.Store(&v)
-		fmt.Fprint(w, `<LBAgentHeartbeatResponse><LBAgentHeartbeatResult><Status>active</Status><ConfigHash>h1</ConfigHash></LBAgentHeartbeatResult></LBAgentHeartbeatResponse>`)
-	}))
-	defer gw.Close()
-
-	a := datapathAgent(t, gw.URL, testNetcfg)
-	a.localConfigHash = "h1"
-	a.netcfgPath = filepath.Join(t.TempDir(), "absent")
-	a.instanceIDFn = func() (string, error) { return "", errors.New("must not be asked without NICs") }
-
-	a.tick()
-	v := got.Load()
-	if v == nil {
-		t.Fatal("no heartbeat received")
-	}
-	for k := range *v {
-		if k == "InstanceID" || len(k) >= 4 && k[:4] == "NICs" {
-			t.Errorf("unexpected evidence parameter %s", k)
-		}
-	}
-}
-
-func TestDataPathEvidence_InstanceIDRetriedUntilKnown(t *testing.T) {
-	a := datapathAgent(t, "http://unused", testNetcfg)
-	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error { return nil }
-	var calls atomic.Int32
-	a.instanceIDFn = func() (string, error) {
-		if calls.Add(1) < 3 {
-			return "", errors.New("metadata not ready")
-		}
-		return "i-late", nil
-	}
-
-	// A failure backs the lookup off for imdsRetryTicks-1 ticks.
-	ticks := func(n int) string {
-		var id string
-		for range n {
-			id = a.dataPathEvidence().InstanceID
-		}
-		return id
-	}
-	if got := ticks(imdsRetryTicks); got != "" || calls.Load() != 1 {
-		t.Fatalf("after %d ticks: id %q, calls %d; want no id and one lookup", imdsRetryTicks, got, calls.Load())
-	}
-	if got := ticks(imdsRetryTicks); got != "" || calls.Load() != 2 {
-		t.Fatalf("second window: id %q, calls %d; want no id and two lookups", got, calls.Load())
-	}
-	if got := ticks(imdsRetryTicks); got != "i-late" || calls.Load() != 3 {
-		t.Fatalf("third window: id %q, calls %d; want i-late and three lookups", got, calls.Load())
-	}
-	ticks(imdsRetryTicks)
-	if calls.Load() != 3 {
-		t.Errorf("metadata asked %d times, want 3 (stop once known)", calls.Load())
-	}
-}
-
-func TestTick_BlockedInstanceIDLookupDoesNotDelayHeartbeat(t *testing.T) {
-	var got atomic.Pointer[url.Values]
-	gw := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		_ = r.ParseForm()
-		if r.FormValue("Action") == "LBAgentHeartbeat" {
-			v := r.Form
-			got.Store(&v)
-		}
-		fmt.Fprint(w, `<LBAgentHeartbeatResponse><LBAgentHeartbeatResult><Status>active</Status><ConfigHash>h1</ConfigHash></LBAgentHeartbeatResult></LBAgentHeartbeatResponse>`)
-	}))
-	defer gw.Close()
-
-	a := datapathAgent(t, gw.URL, testNetcfg)
-	a.localConfigHash = "h1"
-	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error { return nil }
-	release := make(chan struct{})
-	defer close(release)
-	var calls atomic.Int32
-	a.instanceIDFn = func() (string, error) {
-		calls.Add(1)
-		<-release
-		return "", errors.New("unreachable")
-	}
-
-	for i := range 3 {
-		start := time.Now()
-		a.tick()
-		if d := time.Since(start); d >= pollInterval {
-			t.Fatalf("tick %d took %s with IMDS blocked, want under one %s tick", i, d, pollInterval)
-		}
-	}
-	v := got.Load()
-	if v == nil {
-		t.Fatal("the heartbeat was never sent")
-	}
-	if v.Get("InstanceID") != "" {
-		t.Error("InstanceID must be omitted while the lookup fails")
-	}
-	if v.Get("NICs.member.1.MAC") == "" {
-		t.Error("NIC evidence must still be sent")
-	}
-	if calls.Load() != 1 {
-		t.Errorf("blocked lookup started %d times in 3 ticks, want 1 (backoff)", calls.Load())
-	}
-}
-
-func TestFetchInstanceID_SharesOneDeadline(t *testing.T) {
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		time.Sleep(3 * imdsWait)
-	}))
-	defer srv.Close()
-	old := imdsBase
-	imdsBase = srv.URL + "/latest"
-	defer func() { imdsBase = old }()
-
-	start := time.Now()
-	if _, err := fetchInstanceID(); err == nil {
-		t.Fatal("a hung metadata service must fail the lookup")
-	}
-	if d := time.Since(start); d > imdsWait+imdsWait/2 {
-		t.Fatalf("lookup took %s, want about one %s deadline", d, imdsWait)
-	}
-}
-
-func TestARPProbe_NoResponderFailsClosed(t *testing.T) {
-	mac, _ := net.ParseMAC(testDataMAC)
-	// Nobody answers an ARP request sent on loopback, so a probe that returned
-	// nil here would mask a dead data path. It fails either on the socket (no
-	// CAP_NET_RAW) or on the missing reply.
-	err := arpProbe("lo", mac, netip.MustParseAddr("10.9.9.2"), netip.MustParseAddr("10.9.9.1"), 100*time.Millisecond)
-	if err == nil {
-		t.Fatal("a probe with no responder must fail")
-	}
-	if err := arpProbe("no-such-iface0", mac, netip.MustParseAddr("10.9.9.2"), netip.MustParseAddr("10.9.9.1"), 100*time.Millisecond); err == nil {
-		t.Fatal("a probe on a missing interface must fail")
-	}
-}
-
-func TestFetchInstanceID(t *testing.T) {
-	serve := func(tokenStatus, idStatus int, wantToken string) *httptest.Server {
-		return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			switch {
-			case r.Method == http.MethodPut && r.URL.Path == "/latest/api/token":
-				w.WriteHeader(tokenStatus)
-				fmt.Fprint(w, "tok123")
-			case r.URL.Path == "/latest/meta-data/instance-id":
-				if got := r.Header.Get("X-Aws-Ec2-Metadata-Token"); got != wantToken {
-					t.Errorf("token header = %q, want %q", got, wantToken)
-				}
-				w.WriteHeader(idStatus)
-				fmt.Fprint(w, " i-abc123 \n")
-			default:
-				http.NotFound(w, r)
-			}
-		}))
-	}
-	saved := imdsBase
-	defer func() { imdsBase = saved }()
-
-	for _, tc := range []struct {
-		name                  string
-		tokenStatus, idStatus int
-		wantToken             string
-		wantID                string
-		wantErr               bool
-	}{
-		{"token then id", http.StatusOK, http.StatusOK, "tok123", "i-abc123", false},
-		{"token refused falls back to v1", http.StatusNotFound, http.StatusOK, "", "i-abc123", false},
-		{"id endpoint error", http.StatusOK, http.StatusServiceUnavailable, "tok123", "", true},
-	} {
-		t.Run(tc.name, func(t *testing.T) {
-			srv := serve(tc.tokenStatus, tc.idStatus, tc.wantToken)
-			defer srv.Close()
-			imdsBase = srv.URL + "/latest"
-			id, err := fetchInstanceID()
-			if (err != nil) != tc.wantErr || id != tc.wantID {
-				t.Fatalf("id = %q, err = %v", id, err)
-			}
-		})
-	}
-
-	imdsBase = "http://127.0.0.1:1/latest"
-	if _, err := fetchInstanceID(); err == nil {
-		t.Error("an unreachable metadata service must fail")
-	}
-}

spinifex/agents/elbv2/evidence.go

@@ -1,133 +0,0 @@
-package elbv2
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"strconv"
-	"strings"
-	"time"
-)
-
-const (
-	imdsWait       = time.Second
-	imdsRetryTicks = 6
-)
-
-// imdsBase is the in-VM metadata endpoint (a variable so tests can point it at a fake).
-var imdsBase = "http://169.254.169.254/latest"
-
-// dataPathEvidence is the optional part of a heartbeat: which VM the agent runs
-// in and what each data NIC's gateway probe saw. An empty value adds nothing,
-// which is what an agent without a netcfg blob sends.
-type dataPathEvidence struct {
-	InstanceID string
-	NICs       []NICReport
-}
-
-// addTo encodes the evidence as additional heartbeat query parameters.
-func (e dataPathEvidence) addTo(params url.Values) {
-	if e.InstanceID != "" {
-		params.Set("InstanceID", e.InstanceID)
-	}
-	for i, n := range e.NICs {
-		p := "NICs.member." + strconv.Itoa(i+1) + "."
-		params.Set(p+"MAC", n.MAC)
-		params.Set(p+"ProbeOK", strconv.FormatBool(n.ProbeOK))
-		params.Set(p+"ConsecutiveFailures", strconv.FormatInt(n.ConsecutiveFailures, 10))
-		params.Set(p+"RxPackets", strconv.FormatInt(n.RxPackets, 10))
-		params.Set(p+"TxPackets", strconv.FormatInt(n.TxPackets, 10))
-		params.Set(p+"RxDropped", strconv.FormatInt(n.RxDropped, 10))
-		params.Set(p+"TxDropped", strconv.FormatInt(n.TxDropped, 10))
-	}
-}
-
-// dataPathEvidence gathers the probe results and instance ID for one heartbeat.
-// Neither step can fail the heartbeat: missing evidence is simply omitted.
-func (a *Agent) dataPathEvidence() dataPathEvidence {
-	nics := a.probeNICs()
-	if len(nics) == 0 {
-		return dataPathEvidence{}
-	}
-	if a.instanceID == "" {
-		a.lookupInstanceID()
-	}
-	return dataPathEvidence{InstanceID: a.instanceID, NICs: nics}
-}
-
-// lookupInstanceID asks for the instance ID without ever holding up a tick: the
-// call is bounded by imdsWait and, after a failure, retried only every
-// imdsRetryTicks ticks.
-func (a *Agent) lookupInstanceID() {
-	if a.idSkip > 0 {
-		a.idSkip--
-		return
-	}
-	type result struct {
-		id  string
-		err error
-	}
-	ch := make(chan result, 1)
-	go func() {
-		id, err := a.instanceIDFn()
-		ch <- result{id, err}
-	}()
-	timer := time.NewTimer(imdsWait)
-	defer timer.Stop()
-	select {
-	case r := <-ch:
-		if r.err == nil && r.id != "" {
-			a.instanceID = r.id
-			return
-		}
-	case <-timer.C:
-	}
-	a.idSkip = imdsRetryTicks - 1
-}
-
-// fetchInstanceID reads the instance ID from the in-VM metadata service. The
-// token request and the read share one deadline.
-func fetchInstanceID() (string, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), imdsWait)
-	defer cancel()
-	c := &http.Client{}
-
-	var token string
-	if req, err := http.NewRequestWithContext(ctx, http.MethodPut, imdsBase+"/api/token", nil); err == nil {
-		req.Header.Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "60")
-		if resp, err := c.Do(req); err == nil {
-			b, _ := io.ReadAll(io.LimitReader(resp.Body, 256))
-			resp.Body.Close()
-			if resp.StatusCode == http.StatusOK {
-				token = strings.TrimSpace(string(b))
-			}
-		}
-	}
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, imdsBase+"/meta-data/instance-id", nil)
-	if err != nil {
-		return "", err
-	}
-	if token != "" {
-		req.Header.Set("X-Aws-Ec2-Metadata-Token", token)
-	}
-	resp, err := c.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("instance-id: status %d", resp.StatusCode)
-	}
-	b, err := io.ReadAll(io.LimitReader(resp.Body, 256))
-	if err != nil {
-		return "", err
-	}
-	id := strings.TrimSpace(string(b))
-	if id == "" {
-		return "", fmt.Errorf("instance-id: empty")
-	}
-	return id, nil
-}

spinifex/agents/lambda/hostlink/configdisk.go

@@ -1,71 +0,0 @@
-package hostlink
-
-import (
-	"errors"
-	"fmt"
-	"io"
-	"os"
-
-	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
-	"golang.org/x/sys/unix"
-)
-
-// On a VMM without fw_cfg the executor attaches the same entries as a tar
-// archive on a second disk, which the runtime image's init extracts here in
-// the fw_cfg sysfs layout, so the fw_cfg readers work unchanged.
-const (
-	ConfigDiskRoot   = "/spinifex-cfg"
-	ConfigDiskDir    = ConfigDiskRoot + "/by_name/" + lambdav1.FwCfgPrefix
-	ConfigDiskDevice = "/dev/vdb"
-)
-
-// zeroChunk bounds each write while erasing the configuration disk.
-const zeroChunk = 64 << 10
-
-// unmount is a var so tests without privilege can stand in for the tmpfs.
-var unmount = unix.Unmount
-
-// SealConfigDisk overwrites the whole configuration disk with zeros, then
-// removes the extracted tree. The tree stays if the disk cannot be erased, so
-// SealFwCfg's check on root fails closed.
-func SealConfigDisk(device, root string) error {
-	if err := zeroDevice(device); err != nil {
-		return fmt.Errorf("hostlink: erase configuration disk: %w", err)
-	}
-	if err := unmount(root, 0); err != nil && !errors.Is(err, unix.EINVAL) {
-		return fmt.Errorf("hostlink: unmount %s: %w", root, err)
-	}
-	if err := os.RemoveAll(root); err != nil {
-		return fmt.Errorf("hostlink: remove %s: %w", root, err)
-	}
-	return nil
-}
-
-func zeroDevice(device string) error {
-	f, err := os.OpenFile(device, os.O_WRONLY, 0)
-	if err != nil {
-		return err
-	}
-	size, err := f.Seek(0, io.SeekEnd)
-	if err == nil {
-		_, err = f.Seek(0, io.SeekStart)
-	}
-	if err != nil {
-		_ = f.Close()
-		return err
-	}
-	zeros := make([]byte, zeroChunk)
-	for left := size; left > 0; {
-		n := min(left, int64(len(zeros)))
-		if _, err := f.Write(zeros[:n]); err != nil {
-			_ = f.Close()
-			return err
-		}
-		left -= n
-	}
-	if err := f.Sync(); err != nil {
-		_ = f.Close()
-		return err
-	}
-	return f.Close()
-}

spinifex/agents/lambda/hostlink/configdisk_test.go

@@ -1,63 +0,0 @@
-//test:in-package reaches the unexported configDisk and fwCfgTree readers to seal and read a real archive
-
-package hostlink
-
-import (
-	"bytes"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"golang.org/x/sys/unix"
-)
-
-// configDisk writes a fake disk holding secret and an extracted tree; the
-// stand-in unmount reports the tree is not a mount, as on a plain directory.
-func configDisk(t *testing.T, secret []byte) (device, root string) {
-	t.Helper()
-	dir := t.TempDir()
-	device = filepath.Join(dir, "vdb")
-	require.NoError(t, os.WriteFile(device, append(bytes.Repeat([]byte{0xa5}, 3*zeroChunk+7), secret...), 0o600))
-	root = fwCfgTree(t)
-	require.NoError(t, os.WriteFile(filepath.Join(root, "by_name", "key"), secret, 0o600))
-	old := unmount
-	unmount = func(string, int) error { return unix.EINVAL }
-	t.Cleanup(func() { unmount = old })
-	return device, root
-}
-
-func TestSealConfigDisk(t *testing.T) {
-	secret := []byte("guest-key-material")
-
-	t.Run("erases the disk and removes the tree", func(t *testing.T) {
-		device, root := configDisk(t, secret)
-		before, err := os.Stat(device)
-		require.NoError(t, err)
-		require.NoError(t, SealFwCfg(root, func() error { return SealConfigDisk(device, root) }))
-
-		disk, err := os.ReadFile(device)
-		require.NoError(t, err)
-		assert.Len(t, disk, int(before.Size()))
-		assert.Equal(t, make([]byte, len(disk)), disk, "configuration disk not fully erased")
-		_, err = os.Stat(root)
-		assert.ErrorIs(t, err, os.ErrNotExist)
-	})
-	t.Run("an unwritable disk keeps the tree and fails closed", func(t *testing.T) {
-		device, root := configDisk(t, secret)
-		require.NoError(t, os.Chmod(device, 0o400))
-		err := SealFwCfg(root, func() error { return SealConfigDisk(device, root) })
-		require.Error(t, err)
-		assert.Contains(t, err.Error(), "still readable")
-		_, err = os.Stat(root)
-		assert.NoError(t, err)
-	})
-	t.Run("a failed unmount keeps the tree", func(t *testing.T) {
-		device, root := configDisk(t, secret)
-		unmount = func(string, int) error { return unix.EBUSY }
-		require.Error(t, SealFwCfg(root, func() error { return SealConfigDisk(device, root) }))
-		_, err := os.Stat(root)
-		assert.NoError(t, err)
-	})
-}

spinifex/agents/lambda/hostlink/link.go

@@ -28,12 +28,11 @@ type Request struct {
 	Credential *Credential `json:"credential,omitempty"`
 }
 
-// Credential holds one execution-role session and its function environment.
+// Credential is one execution-role session. It is never logged or stored.
 type Credential struct {
-	AccessKeyID     string            `json:"access_key_id"`
-	SecretAccessKey string            `json:"secret_access_key"`
-	SessionToken    string            `json:"session_token"`
-	Environment     map[string]string `json:"environment,omitempty"`
+	AccessKeyID     string `json:"access_key_id"`
+	SecretAccessKey string `json:"secret_access_key"`
+	SessionToken    string `json:"session_token"`
 }
 
 // LogValue keeps every value out of logs.

spinifex/agents/lambda/hostlink/protocol_testlink.go

@@ -4,5 +4,5 @@ package hostlink
 
 // Protocol is the only ALPN protocol a test-link host and guest offer. It
 // differs from the release protocol because the first frame carries the
-// execution credential and function environment, which an older guest would ignore.
-const Protocol = "spinifex-lambda-testlink/3"
+// execution credential, which an older guest would ignore.
+const Protocol = "spinifex-lambda-testlink/2"

spinifex/agents/lambda/hostlink/test_stream.go

@@ -321,19 +321,6 @@ func (s *TestStream) HostSend(req Request) error {
 	return s.write(frame{Type: typeInvoke, Request: &req})
 }
 
-// CheckInvokeFits encodes req as HostSend would and returns ErrFrameTooLarge
-// when the frame exceeds the bound, so a caller can refuse before launch.
-func CheckInvokeFits(req Request) error {
-	b, err := json.Marshal(frame{Type: typeInvoke, Request: &req})
-	if err != nil {
-		return err
-	}
-	if len(b) > maxFrame {
-		return ErrFrameTooLarge
-	}
-	return nil
-}
-
 // HostShutdown writes a shutdown frame, as the host side of the carrier.
 func (s *TestStream) HostShutdown() error {
 	return s.write(frame{Type: typeShutdown})

spinifex/domains/admission/quota/quota.go

@@ -22,10 +22,10 @@ const ReconcileInterval = 30 * time.Second
 // dimension; the live-counted dimensions need no stored state.
 const KVBucketAccountUsage = "spinifex-account-usage"
 
-// KVBucketQuotaReconcile is the leader-lock bucket for the gateway quota
-// reconcile. It is deliberately distinct from vpcd's network-reconcile lock so
-// the two independent loops never share one mutex and block each other.
-const KVBucketQuotaReconcile = "spinifex-quota-reconcile"
+// KeyReconcile is the sync-bucket leader lease for the gateway quota reconcile.
+// It is deliberately distinct from vpcd's network-reconcile lease so the two
+// independent loops never share one mutex and block each other.
+const KeyReconcile = "quota.reconcile"
 
 // KVBucketAccountQuota holds the per-account limit overrides, keyed by
 // accountID. It is separate from the accounts bucket so a quota write never

spinifex/domains/dns/delegation.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	nsconfig "github.com/mulgadc/northstar/pkg/config"
-	"github.com/nats-io/nats.go"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 
 	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
 )
@@ -48,7 +48,7 @@ func (w *Writer) PlatformDelegation(name string) (servers []string, zone string,
 // Delegate writes, or with remove withdraws, an NS set in the base zone that
 // delegates each name to this cluster's nameservers. It takes the zone-writer
 // election first, so it never races the reconcile's own writes to the zone.
-func (w *Writer) Delegate(ctx context.Context, nc *nats.Conn, holder string, names []string, remove bool) (*ChangeResult, error) {
+func (w *Writer) Delegate(ctx context.Context, lease *kvsync.Lease, names []string, remove bool) (*ChangeResult, error) {
 	if !w.Enabled() {
 		return nil, errors.New("northstar is not configured on this node")
 	}
@@ -77,7 +77,7 @@ func (w *Writer) Delegate(ctx context.Context, nc *nats.Conn, holder string, nam
 	if len(batch.Changes) == 0 {
 		return &ChangeResult{}, nil
 	}
-	release, err := w.awaitWriterElection(ctx, nc, holder)
+	release, err := w.awaitWriterElection(ctx, lease)
 	if err != nil {
 		return nil, err
 	}
@@ -85,19 +85,19 @@ func (w *Writer) Delegate(ctx context.Context, nc *nats.Conn, holder string, nam
 	return w.ApplyBatch(batch)
 }
 
-func (w *Writer) awaitWriterElection(ctx context.Context, nc *nats.Conn, holder string) (func(), error) {
-	if nc == nil {
+func (w *Writer) awaitWriterElection(ctx context.Context, lease *kvsync.Lease) (func(), error) {
+	if lease == nil {
 		return func() {}, nil
 	}
 	tick := time.NewTicker(delegateLockPoll)
 	defer tick.Stop()
 	for {
-		if release, ok := reconcilelock.AcquireLeader(ctx, nc, KVBucketDNSReconcile, holder); ok {
+		if release, ok := reconcilelock.AcquireLeader(ctx, lease); ok {
 			return release, nil
 		}
 		select {
 		case <-ctx.Done():
-			return nil, fmt.Errorf("another node is writing the zone: %w", ctx.Err())
+			return nil, fmt.Errorf("another writer has the zone: %w", ctx.Err())
 		case <-tick.C:
 		}
 	}

spinifex/domains/dns/delegation_test.go

@@ -14,7 +14,7 @@ import (
 
 func TestDelegateWritesAndWithdrawsPlatformDelegations(t *testing.T) {
 	w, _ := newMultiNodeTestWriter(t)
-	_, err := w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net", "B.r53e2e.spx3.net."}, false)
+	_, err := w.Delegate(t.Context(), nil, []string{"a.r53e2e.spx3.net", "B.r53e2e.spx3.net."}, false)
 	require.NoError(t, err)
 
 	for _, name := range []string{"a.r53e2e.spx3.net.", "b.r53e2e.spx3.net"} {
@@ -29,7 +29,7 @@ func TestDelegateWritesAndWithdrawsPlatformDelegations(t *testing.T) {
 	assert.True(t, ok, "a name below the base domain is decided by the base zone")
 	assert.Empty(t, servers)
 
-	_, err = w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net"}, true)
+	_, err = w.Delegate(t.Context(), nil, []string{"a.r53e2e.spx3.net"}, true)
 	require.NoError(t, err)
 	servers, _, _, err = w.PlatformDelegation("a.r53e2e.spx3.net")
 	require.NoError(t, err)
@@ -43,11 +43,11 @@ func TestDelegateWritesAndWithdrawsPlatformDelegations(t *testing.T) {
 // servers written while the cluster published a larger set.
 func TestDelegateRemoveWithdrawsServersFromAnEarlierSet(t *testing.T) {
 	w, _ := newMultiNodeTestWriter(t)
-	_, err := w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net"}, false)
+	_, err := w.Delegate(t.Context(), nil, []string{"a.r53e2e.spx3.net"}, false)
 	require.NoError(t, err)
 
 	w.nameservers = w.nameservers[:1]
-	_, err = w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net"}, true)
+	_, err = w.Delegate(t.Context(), nil, []string{"a.r53e2e.spx3.net"}, true)
 	require.NoError(t, err)
 	servers, _, _, err := w.PlatformDelegation("a.r53e2e.spx3.net")
 	require.NoError(t, err)
@@ -66,7 +66,7 @@ func TestPlatformDelegationIgnoresNamesOutsideTheBaseDomain(t *testing.T) {
 func TestDelegateRefusesPlatformAndForeignNames(t *testing.T) {
 	w, _ := newMultiNodeTestWriter(t)
 	for _, name := range []string{"ns1.spx3.net", "x.elb.spx3.net", "compute.spx3.net", "example.com", "spx3.net"} {
-		_, err := w.Delegate(t.Context(), nil, "node1", []string{"ok.spx3.net", name}, false)
+		_, err := w.Delegate(t.Context(), nil, []string{"ok.spx3.net", name}, false)
 		require.Error(t, err, name)
 	}
 	servers, _, _, err := w.PlatformDelegation("ok.spx3.net")
@@ -76,22 +76,45 @@ func TestDelegateRefusesPlatformAndForeignNames(t *testing.T) {
 
 func TestDelegateWaitsForTheZoneWriterElection(t *testing.T) {
 	w, _ := newMultiNodeTestWriter(t)
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	prev := delegateLockPoll
 	delegateLockPoll = 10 * time.Millisecond
 	t.Cleanup(func() { delegateLockPoll = prev })
 
-	release, ok := reconcilelock.AcquireLeader(t.Context(), nc, KVBucketDNSReconcile, "node2")
+	release, ok := reconcilelock.AcquireLeader(t.Context(), dnsLease(js, "node2"))
 	require.True(t, ok)
 	ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
 	defer cancel()
-	_, err := w.Delegate(ctx, nc, "node1", []string{"a.spx3.net"}, false)
-	require.ErrorContains(t, err, "another node is writing the zone")
+	node1 := dnsLease(js, "node1")
+	_, err := w.Delegate(ctx, node1, []string{"a.spx3.net"}, false)
+	require.ErrorContains(t, err, "another writer has the zone")
 
 	release()
-	_, err = w.Delegate(t.Context(), nc, "node1", []string{"a.spx3.net"}, false)
+	_, err = w.Delegate(t.Context(), node1, []string{"a.spx3.net"}, false)
 	require.NoError(t, err)
 	servers, _, _, err := w.PlatformDelegation("a.spx3.net")
 	require.NoError(t, err)
 	assert.Len(t, servers, 2)
 }
+
+// The reconcile pass and a delegation on one node share the DNS lease, so a
+// delegation waits for this node's own pass as it would for another node's.
+func TestDelegateWaitsForThisNodesOwnReconcilePass(t *testing.T) {
+	w, _ := newMultiNodeTestWriter(t)
+	_, _, js := testutil.StartTestJetStream(t)
+	prev := delegateLockPoll
+	delegateLockPoll = 10 * time.Millisecond
+	t.Cleanup(func() { delegateLockPoll = prev })
+
+	shared := dnsLease(js, "node1")
+	release, ok := reconcilelock.AcquireLeader(t.Context(), shared)
+	require.True(t, ok, "the reconcile pass holds the lease")
+	ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
+	defer cancel()
+	_, err := w.Delegate(ctx, shared, []string{"a.spx3.net"}, false)
+	require.ErrorContains(t, err, "another writer has the zone")
+
+	release()
+	_, err = w.Delegate(t.Context(), shared, []string{"a.spx3.net"}, false)
+	require.NoError(t, err)
+}

spinifex/domains/dns/reconcile.go

@@ -11,7 +11,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/reconciler"
-	"github.com/nats-io/nats.go"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 )
 
 // DefaultReconcileInterval is how often the drift backstop rebuilds managed
@@ -33,6 +33,12 @@ const DefaultReconcileInterval = reconciler.DefaultResync
 // costs a cycle for nothing.
 const DefaultComputeTimeout = 2 * time.Minute
 
+// NewReconcileLease returns the zone writer's election lease, shared by the
+// drift reconcile and Delegate so the two never write the zone together.
+func NewReconcileLease(store kvsync.KVStore, holder string) *kvsync.Lease {
+	return reconcilelock.NewLeaderLease(store, KeyReconcile, holder)
+}
+
 // DesiredFunc returns the full desired managed record set built from the live
 // resource inventory across all tenants. The daemon supplies it by enumerating
 // instances, load balancers, and EKS clusters.
@@ -102,7 +108,8 @@ type Reconciler struct {
 	// lands in. Distinct from both baseDomain and internalDomain, so pruning it
 	// needs its own read the same way internalDomain does for EC2.
 	serviceZone string
-	nc          *nats.Conn
+	// lease elects the one node that writes the zones; nil skips the election.
+	lease *kvsync.Lease
 	// writer applies the converged batch. The reconcile owns the zone object on
 	// the node that won the election, so the batch never leaves this process.
 	writer   *Writer
@@ -111,7 +118,6 @@ type Reconciler struct {
 	// computeTimeout bounds how long one pass spends building the desired set.
 	// A field rather than a constant so a test can shrink it.
 	computeTimeout time.Duration
-	holder         string
 	sources        []reconciler.Source
 }
 
@@ -121,18 +127,15 @@ type Reconciler struct {
 // changes should wake the loop. They are optional: with none supplied the loop
 // falls back to the interval alone, which is the behaviour that predates the
 // watch.
-func NewReconciler(cfg *config.Config, cluster *config.ClusterConfig, nc *nats.Conn, writer *Writer, desired DesiredFunc, sources ...reconciler.Source) *Reconciler {
+func NewReconciler(cfg *config.Config, cluster *config.ClusterConfig, lease *kvsync.Lease, writer *Writer, desired DesiredFunc, sources ...reconciler.Source) *Reconciler {
 	r := &Reconciler{
-		nc:             nc,
+		lease:          lease,
 		writer:         writer,
 		desired:        desired,
 		interval:       DefaultReconcileInterval,
 		computeTimeout: DefaultComputeTimeout,
 		sources:        sources,
 	}
-	if cfg != nil {
-		r.holder = cfg.Node
-	}
 	zoneCfg, ok := zoneS3Config(cfg)
 	if !ok || desired == nil || !writer.Enabled() {
 		return r
@@ -166,25 +169,24 @@ func (r *Reconciler) Run(ctx context.Context) {
 		// No revisit deadline: the desired set is a function of the buckets
 		// being watched, so a change is the only reason to run early.
 		Reconcile: func(ctx context.Context) (time.Duration, error) {
-			r.reconcileOnce(ctx)
-			return 0, nil
+			return r.reconcileOnce(ctx), nil
 		},
 		Resync: r.interval,
 	})
 }
 
 // reconcileOnce computes the converging batch and applies it, on whichever node
-// wins this cycle's election. The read that finds the stale records and the
+// wins this cycle's election, and returns the election's revisit when it lost. The read that finds the stale records and the
 // write that removes them now happen in one process, so a record re-added
 // between them is no longer deleted by a batch that predates it.
-func (r *Reconciler) reconcileOnce(ctx context.Context) {
+func (r *Reconciler) reconcileOnce(ctx context.Context) time.Duration {
 	if !r.enabled {
-		return
+		return 0
 	}
-	if r.nc != nil {
-		release, elected := reconcilelock.AcquireLeader(ctx, r.nc, KVBucketDNSReconcile, r.holder)
+	if r.lease != nil {
+		release, retry, elected := reconcilelock.ElectPass(ctx, r.lease)
 		if !elected {
-			return
+			return retry
 		}
 		defer release()
 	}
@@ -193,18 +195,19 @@ func (r *Reconciler) reconcileOnce(ctx context.Context) {
 		// A corrupt zone is handled in readZone; anything reaching here left the
 		// desired state unapplied, so surface it rather than burying it in WARN.
 		slog.Error("dns reconcile: compute batch failed, retrying next cycle", "error", err)
-		return
+		return 0
 	}
 	if len(batch) == 0 {
-		return
+		return 0
 	}
 	slog.Debug("dns reconcile: converging", "changes", len(batch))
 	res, err := r.writer.ApplyBatch(&ChangeBatch{Changes: batch})
 	if err != nil {
 		slog.Warn("dns reconcile: apply failed, retrying next cycle", "changes", len(batch), "error", err)
-		return
+		return 0
 	}
 	slog.Debug("dns reconcile: converged", "changes", res.Applied, "zones", res.Zones)
+	return 0
 }
 
 // computeBatchBounded runs computeBatch with a deadline, so a pass that never

spinifex/domains/dns/reconcile_election_test.go

@@ -8,41 +8,50 @@ import (
 
 	"github.com/mulgadc/spinifex/internal/testkit"
 	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
-	"github.com/nats-io/nats.go"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-// electingReconcilerFor is reconcilerFor with a live connection, so a pass runs
-// the election instead of skipping it.
-func electingReconcilerFor(w *Writer, nc *nats.Conn, holder string, desired func() DesiredSet) *Reconciler {
+// electingReconcilerFor is reconcilerFor with a lease, so a pass runs the
+// election instead of skipping it.
+func electingReconcilerFor(w *Writer, lease *kvsync.Lease, desired func() DesiredSet) *Reconciler {
 	r := reconcilerFor(w, desired)
-	r.nc = nc
-	r.holder = holder
+	r.lease = lease
 	return r
 }
 
+// dnsLease returns holder's DNS lease on a store of its own, standing in for
+// a separate process.
+func dnsLease(js jetstream.JetStream, holder string) *kvsync.Lease {
+	return NewReconcileLease(kvstore.OpenSyncBucket(js), holder)
+}
+
 // The zone lock is gone, and what replaces it is that a zone has one writer:
 // the node holding the reconcile lease. A node that loses the election must
 // therefore not touch the object at all — if it did, two nodes could be
 // read-modify-writing the same zone with nothing serialising them.
 func TestReconcileOnce_ALoserOfTheElectionDoesNotWriteTheZone(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	w, objects := newTestWriter(t)
 	before := objects[testBase+".toml"]
 	require.NotEmpty(t, before)
 
 	// A peer holds the lease for the whole pass, so this node cannot win it.
-	release, elected := reconcilelock.AcquireLeader(t.Context(), nc, KVBucketDNSReconcile, "peer")
+	release, elected := reconcilelock.AcquireLeader(t.Context(), dnsLease(js, "peer"))
 	require.True(t, elected, "the peer has to hold the lease or this test proves nothing")
 
-	r := electingReconcilerFor(w, nc, "node1", func() DesiredSet {
+	r := electingReconcilerFor(w, dnsLease(js, "node1"), func() DesiredSet {
 		return DesiredSet{Changes: []Change{upsert("lb-1.elb."+testBase, "10.200.1.9")}}
 	})
-	r.reconcileOnce(t.Context())
+	revisit := r.reconcileOnce(t.Context())
 
 	assert.Equal(t, before, objects[testBase+".toml"],
 		"a node that did not win the election must leave the zone object exactly as it found it")
+	assert.Positive(t, revisit, "the loser asks to retry so a dead holder is taken over")
+	assert.LessOrEqual(t, revisit, reconcilelock.LeaderTTL/3, "the retry comes within a renewal interval")
 
 	release()
 }
@@ -50,10 +59,10 @@ func TestReconcileOnce_ALoserOfTheElectionDoesNotWriteTheZone(t *testing.T) {
 // The counterpart, so the test above is not passing because the pass was broken
 // for some other reason: with the lease free, the same reconciler writes.
 func TestReconcileOnce_TheWinnerOfTheElectionWritesTheZone(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	w, objects := newTestWriter(t)
 
-	r := electingReconcilerFor(w, nc, "node1", func() DesiredSet {
+	r := electingReconcilerFor(w, dnsLease(js, "node1"), func() DesiredSet {
 		return DesiredSet{Changes: []Change{upsert("lb-1.elb."+testBase, "10.200.1.9")}}
 	})
 	r.reconcileOnce(t.Context())

spinifex/domains/dns/reconcile_test.go

@@ -224,7 +224,7 @@ func TestReconciler_RunReturnsWhenDisabled(t *testing.T) {
 // path of Run: the startup pass must fire before any watch or resync activity,
 // and cancelling ctx must still let the loop return.
 func TestReconciler_RunPerformsStartupPassThenStopsOnCancel(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	endpoint, _ := fakeS3(t, "northstar")
 
 	var passes atomic.Int64
@@ -235,8 +235,7 @@ func TestReconciler_RunPerformsStartupPassThenStopsOnCancel(t *testing.T) {
 			Endpoint: endpoint, Bucket: "northstar", Region: "us-east-1",
 			AccessKey: "SYSTEM", SecretKey: "SYSTEMSECRET",
 		},
-		nc:     nc,
-		holder: "test-node",
+		lease: dnsLease(js, "test-node"),
 		desired: func() DesiredSet {
 			passes.Add(1)
 			return DesiredSet{}
@@ -268,7 +267,7 @@ func TestReconciler_RunPerformsStartupPassThenStopsOnCancel(t *testing.T) {
 // acquirable afterwards, not merely on the call returning, since it is the
 // leadership and not the pass that the rest of the cluster waits on.
 func TestReconcileOnce_AStuckPassSurrendersLeadership(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	endpoint, _ := fakeS3(t, "northstar")
 
 	blocked := make(chan struct{})
@@ -281,8 +280,7 @@ func TestReconcileOnce_AStuckPassSurrendersLeadership(t *testing.T) {
 			Endpoint: endpoint, Bucket: "northstar", Region: "us-east-1",
 			AccessKey: "SYSTEM", SecretKey: "SYSTEMSECRET",
 		},
-		nc:             nc,
-		holder:         "stuck-node",
+		lease:          dnsLease(js, "stuck-node"),
 		computeTimeout: 100 * time.Millisecond,
 		desired: func() DesiredSet {
 			<-blocked
@@ -302,7 +300,7 @@ func TestReconcileOnce_AStuckPassSurrendersLeadership(t *testing.T) {
 		t.Fatal("reconcileOnce never returned: a stuck pass still holds the lease forever")
 	}
 
-	release, elected := reconcilelock.AcquireLeader(t.Context(), nc, KVBucketDNSReconcile, "next-node")
+	release, elected := reconcilelock.AcquireLeader(t.Context(), dnsLease(js, "next-node"))
 	require.True(t, elected, "another node must be able to take the next cycle")
 	release()
 }

spinifex/domains/dns/types.go

@@ -11,9 +11,9 @@ const (
 	// PUT so every northstar instance reloads just that zone immediately, instead
 	// of waiting for the next S3 sync poll. No queue group: all servers consume.
 	SubjectZoneReload = "dns.zone.reload"
-	// KVBucketDNSReconcile elects the one node that writes the zones for a drift
-	// cycle. Its own bucket, so the election never blocks another reconcile loop.
-	KVBucketDNSReconcile = "spinifex-dns-reconcile"
+	// KeyReconcile elects the one node that writes the zones for a drift cycle.
+	// Its own key, so the election never blocks another reconcile loop.
+	KeyReconcile = "dns.reconcile"
 	// PrivateZone is the fixed AWS-parity private DNS zone (IMDS synthHostname).
 	PrivateZone = "compute.internal"
 	// DefaultTTL is applied when a change omits a TTL.

spinifex/domains/ec2/accountcleanup/remainder.go

@@ -1,155 +0,0 @@
-package accountcleanup
-
-import (
-	"context"
-	"log/slog"
-	"slices"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
-)
-
-// ReasonRecordRemains marks a resource whose record the owner still holds.
-const ReasonRecordRemains = "record-remains"
-
-// ComputeRemainder reads what compute the account still holds, changing nothing.
-type ComputeRemainder interface {
-	ComputeRemaining(ctx context.Context, accountID string) (*Outcome, error)
-}
-
-// StorageRemainder reads what volumes and tag residue the account still holds,
-// changing nothing.
-type StorageRemainder interface {
-	StorageRemaining(ctx context.Context, accountID string) (*Outcome, error)
-}
-
-var (
-	_ ComputeRemainder = (*ComputeCleanup)(nil)
-	_ StorageRemainder = (*StorageCleanup)(nil)
-)
-
-func validRemainderAccount(accountID string) error {
-	if accountID == "" {
-		return awserrors.Errorf(awserrors.ErrorMissingParameter, "The request must specify an account ID.")
-	}
-	if !awsidentifiers.IsAccountID(accountID) {
-		return awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "The account ID must be 12 digits.")
-	}
-	return nil
-}
-
-// ComputeRemaining lists every instance the account still holds and every spot
-// request that can still launch. A read that does not complete is an incomplete
-// reason, never a shorter list. The result is a snapshot, not evidence of absence.
-func (c *ComputeCleanup) ComputeRemaining(ctx context.Context, accountID string) (*Outcome, error) {
-	if err := validRemainderAccount(accountID); err != nil {
-		return nil, err
-	}
-	if c == nil || c.spot == nil || c.instances == nil || c.teardown == nil {
-		return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "Compute cleanup is not fully wired.")
-	}
-	res := &Outcome{AccountID: accountID}
-	defer res.finish()
-
-	scan, err := c.spot.ScanAccount(ctx, accountID)
-	if err != nil {
-		slog.WarnContext(ctx, "ComputeRemaining: spot scan failed", "account_id", accountID, "err", err)
-		res.incomplete(ReasonSpotInventoryIncomplete)
-	} else {
-		for _, r := range slices.Concat(scan.Active, scan.Terminal) {
-			if !terminalSpotState(r.State) {
-				res.Items = append(res.Items, Item{ResourceID: r.RequestID, Kind: KindSpotRequest, State: r.State, Disposition: DispositionResidue, Reason: ReasonRecordRemains})
-			}
-		}
-	}
-
-	inventory, err := c.instances.AccountInstances(ctx, accountID)
-	if err != nil || !inventoryUsable(inventory) {
-		slog.WarnContext(ctx, "ComputeRemaining: instance inventory incomplete", "account_id", accountID, "err", err)
-		res.incomplete(ReasonInstanceInventoryIncomplete)
-	} else {
-		for _, reservation := range inventory.Reservations {
-			for _, in := range reservation.Instances {
-				state := ""
-				if in.State != nil {
-					state = aws.StringValue(in.State.Name)
-				}
-				res.Items = append(res.Items, Item{ResourceID: aws.StringValue(in.InstanceId), Kind: KindInstance, State: state, Disposition: DispositionResidue, Reason: ReasonRecordRemains})
-			}
-		}
-	}
-
-	outstanding, err := c.teardown.AccountTeardownOutstanding(accountID)
-	if err != nil {
-		slog.WarnContext(ctx, "ComputeRemaining: teardown unread", "account_id", accountID, "err", err)
-		res.incomplete(ReasonTeardownUnread)
-	} else {
-		for _, o := range outstanding {
-			if !res.hasInstance(o.InstanceID) {
-				res.Items = append(res.Items, Item{ResourceID: o.InstanceID, Kind: KindInstance, State: o.State, Disposition: DispositionResidue, Reason: ReasonTerminating})
-			}
-		}
-	}
-	return res, nil
-}
-
-// StorageRemaining lists every volume record the account still holds, deleting
-// ones included, and the tag entries of resources whose owner holds no record.
-func (c *StorageCleanup) StorageRemaining(ctx context.Context, accountID string) (*Outcome, error) {
-	if err := validRemainderAccount(accountID); err != nil {
-		return nil, err
-	}
-	if c == nil || c.volumes == nil || c.instances == nil || c.teardown == nil || c.spot == nil || c.tags == nil {
-		return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "Storage cleanup is not fully wired.")
-	}
-	res := &Outcome{AccountID: accountID}
-	defer res.finish()
-
-	volumes, err := c.volumes.VolumeInventory(ctx, accountID)
-	if err != nil {
-		slog.WarnContext(ctx, "StorageRemaining: volume inventory incomplete", "account_id", accountID, "err", err)
-		res.incomplete(ReasonVolumeInventoryIncomplete)
-	}
-	for _, v := range volumes {
-		reason := ReasonRecordRemains
-		if v.State == ec2.VolumeStateDeleting {
-			reason = ReasonVolumeDeleting
-		}
-		res.Items = append(res.Items, Item{ResourceID: v.VolumeID, Kind: KindVolume, State: v.State, Disposition: DispositionResidue, Reason: reason})
-	}
-
-	live, err := c.instances.AccountInstances(ctx, accountID)
-	if err != nil || !inventoryUsable(live) {
-		slog.WarnContext(ctx, "StorageRemaining: no complete instance view", "account_id", accountID, "err", err)
-		res.incomplete(ReasonHolderProofIncomplete)
-		return res, nil
-	}
-	outstanding, err := c.teardown.AccountTeardownOutstanding(accountID)
-	if err != nil {
-		slog.WarnContext(ctx, "StorageRemaining: teardown unread", "account_id", accountID, "err", err)
-		res.incomplete(ReasonHolderProofIncomplete)
-		return res, nil
-	}
-
-	tagged, err := c.tags.ScanAccountResourceIDs(ctx, accountID, storageTagPrefixes...)
-	if err != nil {
-		slog.WarnContext(ctx, "StorageRemaining: tag index read failed", "account_id", accountID, "err", err)
-		res.incomplete(ReasonTagIndexUnread)
-		return res, nil
-	}
-	owned, resolved := c.ownedResourceIDs(ctx, accountID, live, outstanding)
-	for _, id := range tagged {
-		switch {
-		case !resolved[tagPrefix(id)]:
-			res.incomplete(ReasonTagOwnerUnresolved)
-			res.Items = append(res.Items, Item{ResourceID: id, Kind: KindTag, Disposition: DispositionResidue, Reason: ReasonOwnerRecordsUnresolved})
-		case owned[id]:
-			// The resource itself is reported; its tags are not residue.
-		default:
-			res.Items = append(res.Items, Item{ResourceID: id, Kind: KindTag, Disposition: DispositionResidue, Reason: ReasonRecordRemains})
-		}
-	}
-	return res, nil
-}

spinifex/domains/ec2/accountcleanup/remainder_test.go

@@ -1,97 +0,0 @@
-package accountcleanup
-
-import (
-	"errors"
-	"testing"
-
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountinventory"
-	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
-	ec2spotinstance "github.com/mulgadc/spinifex/spinifex/domains/ec2/spotinstance"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestComputeRemainingReportsEveryHeldInstanceAndOpenSpot(t *testing.T) {
-	r := newComputeRig()
-	r.spot.scans = []*ec2spotinstance.AccountScan{{
-		Active:   []ec2spotinstance.AccountSpotRequest{{RequestID: "sir-0aaaaaaaaaaaaaaa1", State: ec2.SpotInstanceStateOpen}},
-		Terminal: []ec2spotinstance.AccountSpotRequest{{RequestID: "sir-0aaaaaaaaaaaaaaa2", State: ec2.SpotInstanceStateClosed}},
-	}}
-	r.teardown.outstanding = []ec2instance.OutstandingInstance{{InstanceID: "i-0aaaaaaaaaaaaaa99", State: ec2.InstanceStateNameTerminated}}
-
-	res, err := r.cleanup().ComputeRemaining(t.Context(), testAccount)
-	require.NoError(t, err)
-
-	assert.Equal(t, StatusIncomplete, res.Status)
-	assert.NotNil(t, itemFor(res, runningID))
-	assert.NotNil(t, itemFor(res, stoppedID))
-	assert.NotNil(t, itemFor(res, "i-0aaaaaaaaaaaaaa99"))
-	assert.NotNil(t, itemFor(res, "sir-0aaaaaaaaaaaaaaa1"))
-	assert.Nil(t, itemFor(res, "sir-0aaaaaaaaaaaaaaa2"), "a closed request launches nothing")
-	assert.Empty(t, r.terminator.terminated, "a re-read changes nothing")
-}
-
-func TestComputeRemainingCleanWhenNothingHeld(t *testing.T) {
-	r := newComputeRig()
-	r.inventory.out = listing(nil)
-	res, err := r.cleanup().ComputeRemaining(t.Context(), testAccount)
-	require.NoError(t, err)
-	assert.Equal(t, StatusClean, res.Status)
-}
-
-func TestComputeRemainingUnreadStoreIsIncompleteNeverClean(t *testing.T) {
-	for name, mutate := range map[string]func(*computeRig){
-		"instances": func(r *computeRig) { r.inventory.out, r.inventory.err = nil, errors.New("node silent") },
-		"spot":      func(r *computeRig) { r.spot.scanErr = errors.New("spot unreadable") },
-		"teardown":  func(r *computeRig) { r.teardown.err = errors.New("no responder") },
-	} {
-		t.Run(name, func(t *testing.T) {
-			r := newComputeRig()
-			r.inventory.out = listing(nil)
-			mutate(r)
-			res, err := r.cleanup().ComputeRemaining(t.Context(), testAccount)
-			require.NoError(t, err)
-			assert.Equal(t, StatusIncomplete, res.Status)
-			assert.NotEmpty(t, res.IncompleteReasons)
-		})
-	}
-}
-
-func TestStorageRemainingReportsVolumeRecordsAndTagResidue(t *testing.T) {
-	r := newStorageRig()
-	r.volumes.vols = []accountinventory.VolumeItem{
-		volume(freeVol, ec2.VolumeStateAvailable, ""),
-		volume("vol-0aaaaaaaaaaaaaaa2", ec2.VolumeStateDeleting, ""),
-	}
-	r.tags.ids = []string{"i-0aaaaaaaaaaaaaaaf", freeVol}
-
-	res, err := r.cleanup().StorageRemaining(t.Context(), testAccount)
-	require.NoError(t, err)
-
-	assert.Equal(t, StatusIncomplete, res.Status)
-	assert.Equal(t, ReasonRecordRemains, itemFor(res, freeVol).Reason)
-	assert.Equal(t, ReasonVolumeDeleting, itemFor(res, "vol-0aaaaaaaaaaaaaaa2").Reason)
-	assert.Equal(t, KindTag, itemFor(res, "i-0aaaaaaaaaaaaaaaf").Kind, "tags of a resource with no record are residue")
-	assert.Empty(t, r.deleter.deleted, "a re-read deletes nothing")
-	assert.Empty(t, r.tags.deleted, "a re-read clears no tag")
-}
-
-func TestStorageRemainingUnreadStoreIsIncompleteNeverClean(t *testing.T) {
-	for name, mutate := range map[string]func(*storageRig){
-		"volumes":   func(r *storageRig) { r.volumes.failAt = -1 },
-		"instances": func(r *storageRig) { r.instances.out, r.instances.err = nil, errors.New("node silent") },
-		"teardown":  func(r *storageRig) { r.teardown.err = errors.New("no responder") },
-		"tags":      func(r *storageRig) { r.tags.scanErr = errors.New("tag index unreadable") },
-	} {
-		t.Run(name, func(t *testing.T) {
-			r := newStorageRig()
-			r.volumes.vols = nil
-			mutate(r)
-			res, err := r.cleanup().StorageRemaining(t.Context(), testAccount)
-			require.NoError(t, err)
-			assert.Equal(t, StatusIncomplete, res.Status)
-			assert.NotEmpty(t, res.IncompleteReasons)
-		})
-	}
-}

spinifex/domains/ec2/awsapi/instance/AccountTeardownOutstanding.go

@@ -1,28 +0,0 @@
-package instance
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
-	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
-	"github.com/nats-io/nats.go"
-)
-
-// AccountTeardownOutstandingSubject serves the instances of an account whose
-// teardown is not complete, from the shared record store.
-const AccountTeardownOutstandingSubject = "ec2.AccountTeardownOutstanding"
-
-var accountTeardownOutstandingTimeout = 15 * time.Second
-
-// AccountTeardownOutstanding reads the account's unfinished instance teardowns.
-// No responder, a timeout or an error reply is an error, never an empty list.
-func AccountTeardownOutstanding(ctx context.Context, natsConn *nats.Conn, accountID string) ([]ec2instance.OutstandingInstance, error) {
-	out, err := natsmsg.NATSRequest[ec2instance.TeardownOutstandingOutput](ctx, natsConn, AccountTeardownOutstandingSubject,
-		&ec2instance.TeardownOutstandingInput{}, accountTeardownOutstandingTimeout, accountID)
-	if err != nil {
-		return nil, fmt.Errorf("read outstanding teardown: %w", err)
-	}
-	return out.Instances, nil
-}

spinifex/domains/ec2/awsapi/instance/AccountTeardownOutstanding_test.go

@@ -1,88 +0,0 @@
-package instance
-
-import (
-	"context"
-	"encoding/json"
-	"testing"
-	"time"
-
-	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
-	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/nats-io/nats.go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func serveTeardownOutstanding(t *testing.T, nc *nats.Conn, reply []byte) {
-	t.Helper()
-	_, err := nc.Subscribe(AccountTeardownOutstandingSubject, func(msg *nats.Msg) { _ = msg.Respond(reply) })
-	require.NoError(t, err)
-	require.NoError(t, nc.Flush())
-}
-
-func TestAccountTeardownOutstanding_ReturnsTheServedInstances(t *testing.T) {
-	t.Parallel()
-	_, nc := startTestNATSServer(t)
-	reply, err := json.Marshal(ec2instance.TeardownOutstandingOutput{Instances: []ec2instance.OutstandingInstance{{InstanceID: "i-1"}}})
-	require.NoError(t, err)
-	serveTeardownOutstanding(t, nc, reply)
-
-	got, err := AccountTeardownOutstanding(context.Background(), nc, "123456789012")
-
-	require.NoError(t, err)
-	require.Len(t, got, 1)
-	assert.Equal(t, "i-1", got[0].InstanceID)
-}
-
-// A read that cannot be answered must never look like an empty list, because an
-// empty list lets the storage cleanup believe no QEMU still holds a volume.
-func TestAccountTeardownOutstanding_UnansweredReadIsAnError(t *testing.T) {
-	t.Parallel()
-
-	t.Run("no responder", func(t *testing.T) {
-		t.Parallel()
-		_, nc := startTestNATSServer(t)
-
-		got, err := AccountTeardownOutstanding(context.Background(), nc, "123456789012")
-
-		require.Error(t, err)
-		assert.Nil(t, got)
-		assert.ErrorIs(t, err, nats.ErrNoResponders)
-	})
-	t.Run("error reply", func(t *testing.T) {
-		t.Parallel()
-		_, nc := startTestNATSServer(t)
-		serveTeardownOutstanding(t, nc, awserrors.GenerateErrorPayload(awserrors.ErrorServerInternal))
-
-		got, err := AccountTeardownOutstanding(context.Background(), nc, "123456789012")
-
-		require.Error(t, err)
-		assert.Nil(t, got)
-	})
-	t.Run("unreadable reply", func(t *testing.T) {
-		t.Parallel()
-		_, nc := startTestNATSServer(t)
-		serveTeardownOutstanding(t, nc, []byte("not json"))
-
-		got, err := AccountTeardownOutstanding(context.Background(), nc, "123456789012")
-
-		require.Error(t, err)
-		assert.Nil(t, got)
-	})
-}
-
-// Not parallel: it shortens the package timeout.
-func TestAccountTeardownOutstanding_TimeoutIsAnError(t *testing.T) {
-	_, nc := startTestNATSServer(t)
-	_, err := nc.Subscribe(AccountTeardownOutstandingSubject, func(*nats.Msg) {})
-	require.NoError(t, err)
-	require.NoError(t, nc.Flush())
-	previous := accountTeardownOutstandingTimeout
-	accountTeardownOutstandingTimeout = 200 * time.Millisecond
-	t.Cleanup(func() { accountTeardownOutstandingTimeout = previous })
-
-	got, err := AccountTeardownOutstanding(context.Background(), nc, "123456789012")
-
-	require.Error(t, err)
-	assert.Nil(t, got)
-}

spinifex/domains/ec2/awsapi/instance/ModifyInstanceAttribute.go

@@ -11,7 +11,6 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/nats-io/nats.go"
 )
@@ -66,20 +65,6 @@ func ModifyInstanceAttribute(ctx context.Context, input *ec2.ModifyInstanceAttri
 
 	slog.InfoContext(ctx, "ModifyInstanceAttribute: Processing request", "instance_id", *input.InstanceId)
 
-	// Termination protection on a running instance lives on its owner, so it
-	// goes there first. A nil value is not a value and keeps the queue path.
-	if input.DisableApiTermination != nil && input.DisableApiTermination.Value != nil {
-		cmd := ec2v1.EC2InstanceCommand{
-			ID:                        *input.InstanceId,
-			Attributes:                ec2v1.EC2CommandAttributes{SetDisableApiTermination: true},
-			DisableApiTerminationData: &ec2v1.DisableApiTerminationData{Value: *input.DisableApiTermination.Value},
-		}
-		if _, err := routeInstanceMutation[ec2.ModifyInstanceAttributeOutput](ctx, natsConn, *input.InstanceId, accountID, cmd, "ec2.ModifyInstanceAttribute", input); err != nil {
-			return ec2.ModifyInstanceAttributeOutput{}, err
-		}
-		return ec2.ModifyInstanceAttributeOutput{}, nil
-	}
-
 	jsonData, err := json.Marshal(input)
 	if err != nil {
 		slog.ErrorContext(ctx, "ModifyInstanceAttribute: Failed to marshal request", "instance_id", *input.InstanceId, "err", err)

spinifex/domains/ec2/awsapi/instance/ModifyInstanceAttribute_test.go

@@ -3,13 +3,10 @@ package instance
 import (
 	"context"
 	"encoding/json"
-	"sync"
-	"sync/atomic"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
@@ -229,53 +226,3 @@ func TestModifyInstanceAttribute_ValidationFailure(t *testing.T) {
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorInvalidParameterValue, err.Error())
 }
-
-func TestModifyInstanceAttribute_DisableApiTerminationRouting(t *testing.T) {
-	const id = "i-0123456789abcdef0"
-	_, nc := startTestNATSServer(t)
-
-	var gotValues []bool
-	var mu sync.Mutex
-	_, err := nc.Subscribe(ec2v1.InstanceCommandSubject(id), func(msg *nats.Msg) {
-		var cmd ec2v1.EC2InstanceCommand
-		if json.Unmarshal(msg.Data, &cmd) == nil && cmd.Attributes.SetDisableApiTermination && cmd.DisableApiTerminationData != nil {
-			mu.Lock()
-			gotValues = append(gotValues, cmd.DisableApiTerminationData.Value)
-			mu.Unlock()
-		}
-		reply := nats.NewMsg(msg.Reply)
-		reply.Data = []byte(`{}`)
-		reply.Header.Set(ec2v1.HandledHeader, "1")
-		_ = msg.RespondMsg(reply)
-	})
-	require.NoError(t, err)
-	var queue atomic.Int32
-	_, err = nc.Subscribe("ec2.ModifyInstanceAttribute", func(msg *nats.Msg) {
-		queue.Add(1)
-		_ = msg.Respond([]byte(`{}`))
-	})
-	require.NoError(t, err)
-
-	for _, v := range []bool{true, false} {
-		_, err := ModifyInstanceAttribute(context.Background(), &ec2.ModifyInstanceAttributeInput{
-			InstanceId:            aws.String(id),
-			DisableApiTermination: &ec2.AttributeBooleanValue{Value: aws.Bool(v)},
-		}, nc, "acct")
-		require.NoError(t, err)
-	}
-	mu.Lock()
-	assert.Equal(t, []bool{true, false}, gotValues, "explicit true and false both reach the owner")
-	mu.Unlock()
-	assert.Zero(t, queue.Load())
-
-	// A nil value is not a value: queue path, no owner command.
-	_, err = ModifyInstanceAttribute(context.Background(), &ec2.ModifyInstanceAttributeInput{
-		InstanceId:            aws.String(id),
-		DisableApiTermination: &ec2.AttributeBooleanValue{},
-	}, nc, "acct")
-	require.NoError(t, err)
-	assert.Equal(t, int32(1), queue.Load())
-	mu.Lock()
-	assert.Len(t, gotValues, 2, "no owner command for a nil value")
-	mu.Unlock()
-}

spinifex/domains/ec2/awsapi/instance/ModifyInstanceMetadataOptions.go

@@ -3,11 +3,11 @@ package instance
 import (
 	"context"
 	"errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"strings"
+	"time"
 
-	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/nats-io/nats.go"
 )
@@ -22,16 +22,5 @@ func ModifyInstanceMetadataOptions(ctx context.Context, input *ec2.ModifyInstanc
 	if !strings.HasPrefix(*input.InstanceId, "i-") {
 		return nil, errors.New(awserrors.ErrorInvalidInstanceIDMalformed)
 	}
-	cmd := ec2v1.EC2InstanceCommand{
-		ID:         *input.InstanceId,
-		Attributes: ec2v1.EC2CommandAttributes{ModifyInstanceMetadataOptions: true},
-		InstanceMetadataOptionsData: &ec2v1.InstanceMetadataOptionsData{
-			HttpTokens:              aws.StringValue(input.HttpTokens),
-			HttpEndpoint:            aws.StringValue(input.HttpEndpoint),
-			HttpProtocolIpv6:        aws.StringValue(input.HttpProtocolIpv6),
-			InstanceMetadataTags:    aws.StringValue(input.InstanceMetadataTags),
-			HttpPutResponseHopLimit: input.HttpPutResponseHopLimit,
-		},
-	}
-	return routeInstanceMutation[ec2.ModifyInstanceMetadataOptionsOutput](ctx, natsConn, *input.InstanceId, accountID, cmd, "ec2.ModifyInstanceMetadataOptions", input)
+	return natsmsg.NATSRequest[ec2.ModifyInstanceMetadataOptionsOutput](ctx, natsConn, "ec2.ModifyInstanceMetadataOptions", input, 30*time.Second, accountID)
 }

spinifex/domains/ec2/awsapi/instance/ModifyInstanceMetadataOptions_test.go

@@ -2,16 +2,11 @@ package instance
 
 import (
 	"context"
-	"encoding/json"
-	"sync/atomic"
 	"testing"
-	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -35,179 +30,3 @@ func TestModifyInstanceMetadataOptions_RejectsMalformedID(t *testing.T) {
 		})
 	}
 }
-
-const metaTestID = "i-0123456789abcdef0"
-
-func metaInput(hop int64) *ec2.ModifyInstanceMetadataOptionsInput {
-	return &ec2.ModifyInstanceMetadataOptionsInput{
-		InstanceId:              aws.String(metaTestID),
-		HttpPutResponseHopLimit: aws.Int64(hop),
-	}
-}
-
-// ownerReply answers one ec2.cmd request, optionally carrying the handled marker.
-func ownerReply(t *testing.T, nc *nats.Conn, handled bool, body []byte, calls *atomic.Int32) {
-	t.Helper()
-	_, err := nc.Subscribe(ec2v1.InstanceCommandSubject(metaTestID), func(msg *nats.Msg) {
-		calls.Add(1)
-		reply := nats.NewMsg(msg.Reply)
-		reply.Data = body
-		if handled {
-			reply.Header.Set(ec2v1.HandledHeader, "1")
-		}
-		_ = msg.RespondMsg(reply)
-	})
-	require.NoError(t, err)
-}
-
-func queueReply(t *testing.T, nc *nats.Conn, body []byte, calls *atomic.Int32) {
-	t.Helper()
-	_, err := nc.QueueSubscribe("ec2.ModifyInstanceMetadataOptions", "spinifex-workers", func(msg *nats.Msg) {
-		calls.Add(1)
-		_ = msg.Respond(body)
-	})
-	require.NoError(t, err)
-}
-
-func metaOutputJSON(t *testing.T, hop int64) []byte {
-	t.Helper()
-	b, err := json.Marshal(&ec2.ModifyInstanceMetadataOptionsOutput{
-		InstanceId:              aws.String(metaTestID),
-		InstanceMetadataOptions: &ec2.InstanceMetadataOptionsResponse{HttpPutResponseHopLimit: aws.Int64(hop)},
-	})
-	require.NoError(t, err)
-	return b
-}
-
-func TestModifyInstanceMetadataOptions_OwnerAnswerReturnedAsIs(t *testing.T) {
-	_, nc := startTestNATSServer(t)
-	var owner, queue atomic.Int32
-	ownerReply(t, nc, true, metaOutputJSON(t, 3), &owner)
-	queueReply(t, nc, []byte(`{}`), &queue)
-
-	out, err := ModifyInstanceMetadataOptions(context.Background(), metaInput(3), nc, "acct")
-	require.NoError(t, err)
-	assert.Equal(t, int64(3), aws.Int64Value(out.InstanceMetadataOptions.HttpPutResponseHopLimit))
-	assert.Equal(t, int32(1), owner.Load())
-	assert.Zero(t, queue.Load())
-}
-
-func TestModifyInstanceMetadataOptions_OwnerErrorWithHeaderIsNotRetried(t *testing.T) {
-	_, nc := startTestNATSServer(t)
-	var owner, queue atomic.Int32
-	ownerReply(t, nc, true, awserrors.GenerateErrorPayload(awserrors.ErrorServerInternal), &owner)
-	queueReply(t, nc, metaOutputJSON(t, 3), &queue)
-
-	_, err := ModifyInstanceMetadataOptions(context.Background(), metaInput(3), nc, "acct")
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-	assert.Zero(t, queue.Load())
-}
-
-func TestModifyInstanceMetadataOptions_OwnerValidationErrorRelayed(t *testing.T) {
-	_, nc := startTestNATSServer(t)
-	var owner, queue atomic.Int32
-	ownerReply(t, nc, true, awserrors.GenerateErrorPayload(awserrors.ErrorUnsupportedOperation), &owner)
-	queueReply(t, nc, metaOutputJSON(t, 3), &queue)
-
-	_, err := ModifyInstanceMetadataOptions(context.Background(), metaInput(3), nc, "acct")
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorUnsupportedOperation, err.Error())
-	assert.Zero(t, queue.Load())
-}
-
-func TestModifyInstanceMetadataOptions_HeaderlessOtherErrorReturnedAsIs(t *testing.T) {
-	_, nc := startTestNATSServer(t)
-	var owner, queue atomic.Int32
-	ownerReply(t, nc, false, awserrors.GenerateErrorPayload(awserrors.ErrorInvalidInstanceIDNotFound), &owner)
-	queueReply(t, nc, metaOutputJSON(t, 3), &queue)
-
-	_, err := ModifyInstanceMetadataOptions(context.Background(), metaInput(3), nc, "acct")
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
-	assert.Zero(t, queue.Load())
-}
-
-func TestS9_N1OwnerRejectionFallsBackOnceToQueueGroup(t *testing.T) {
-	_, nc := startTestNATSServer(t)
-	var owner, queue atomic.Int32
-	ownerReply(t, nc, false, awserrors.GenerateErrorPayload(awserrors.ErrorServerInternal), &owner)
-	queueReply(t, nc, metaOutputJSON(t, 3), &queue)
-
-	out, err := ModifyInstanceMetadataOptions(context.Background(), metaInput(3), nc, "acct")
-	require.NoError(t, err)
-	assert.Equal(t, int64(3), aws.Int64Value(out.InstanceMetadataOptions.HttpPutResponseHopLimit))
-	assert.Equal(t, int32(1), owner.Load(), "exactly one owner request")
-	assert.Equal(t, int32(1), queue.Load(), "exactly one queue-group request")
-}
-
-func TestModifyInstanceMetadataOptions_NoOwnerUsesQueueGroupAfterThreeAttempts(t *testing.T) {
-	_, nc := startTestNATSServer(t)
-	var sleeps atomic.Int32
-	prev := cmdRetrySleep
-	cmdRetrySleep = func(time.Duration) { sleeps.Add(1) }
-	t.Cleanup(func() { cmdRetrySleep = prev })
-
-	var queue atomic.Int32
-	queueReply(t, nc, metaOutputJSON(t, 2), &queue)
-
-	out, err := ModifyInstanceMetadataOptions(context.Background(), metaInput(2), nc, "acct")
-	require.NoError(t, err)
-	assert.Equal(t, int64(2), aws.Int64Value(out.InstanceMetadataOptions.HttpPutResponseHopLimit))
-	assert.Equal(t, int32(2), sleeps.Load(), "three attempts means two backoffs")
-	assert.Equal(t, int32(1), queue.Load())
-}
-
-func TestModifyInstanceMetadataOptions_AbsentIsNotFound(t *testing.T) {
-	_, nc := startTestNATSServer(t)
-	var queue atomic.Int32
-	queueReply(t, nc, awserrors.GenerateErrorPayload(awserrors.ErrorInvalidInstanceIDNotFound), &queue)
-
-	_, err := ModifyInstanceMetadataOptions(context.Background(), metaInput(2), nc, "acct")
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
-}
-
-func TestModifyInstanceMetadataOptions_OwnerTimeoutIsServerInternalWithoutFallback(t *testing.T) {
-	_, nc := startTestNATSServer(t)
-	prev := instanceCmdTimeout
-	instanceCmdTimeout = 50 * time.Millisecond
-	t.Cleanup(func() { instanceCmdTimeout = prev })
-
-	// The owner applies the change, then withholds its reply.
-	var applied, queue atomic.Int32
-	_, err := nc.Subscribe(ec2v1.InstanceCommandSubject(metaTestID), func(*nats.Msg) { applied.Add(1) })
-	require.NoError(t, err)
-	queueReply(t, nc, metaOutputJSON(t, 3), &queue)
-
-	_, err = ModifyInstanceMetadataOptions(context.Background(), metaInput(3), nc, "acct")
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-	assert.Equal(t, int32(1), applied.Load(), "the owner saw the command, so the outcome is unknown")
-	assert.Zero(t, queue.Load(), "a timeout never falls back to the queue group")
-}
-
-func TestModifyInstanceMetadataOptions_TimeoutAfterApplyIsUnknownNotRolledBack(t *testing.T) {
-	_, nc := startTestNATSServer(t)
-	prev := instanceCmdTimeout
-	instanceCmdTimeout = 50 * time.Millisecond
-	t.Cleanup(func() { instanceCmdTimeout = prev })
-
-	// The owner persists the change, then the reply is lost.
-	var persisted atomic.Int64
-	_, err := nc.Subscribe(ec2v1.InstanceCommandSubject(metaTestID), func(msg *nats.Msg) {
-		var cmd ec2v1.EC2InstanceCommand
-		if json.Unmarshal(msg.Data, &cmd) == nil && cmd.InstanceMetadataOptionsData != nil {
-			persisted.Store(aws.Int64Value(cmd.InstanceMetadataOptionsData.HttpPutResponseHopLimit))
-		}
-	})
-	require.NoError(t, err)
-	var queue atomic.Int32
-	queueReply(t, nc, metaOutputJSON(t, 1), &queue)
-
-	_, err = ModifyInstanceMetadataOptions(context.Background(), metaInput(3), nc, "acct")
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-	assert.Zero(t, queue.Load(), "no fallback that could race a live owner")
-	assert.Equal(t, int64(3), persisted.Load(), "the gateway error does not undo what the owner stored")
-}

spinifex/domains/ec2/awsapi/instance/instanceid.go

@@ -99,46 +99,3 @@ func classifyUnownedInstance(ctx context.Context, natsConn *nats.Conn, instanceI
 	}
 	return bucketStateAbsent, allOK
 }
-
-// routeInstanceMutation sends cmd to the instance's owner first and falls back
-// to the queue-group subject only when that is safe: no daemon runs the
-// instance (ErrNoResponders after every retry), or the owner is N-1 and
-// rejected the command without writing (ServerInternal with no HandledHeader).
-// A timeout or other transport error is an unknown outcome and never falls back.
-func routeInstanceMutation[Out any](ctx context.Context, natsConn *nats.Conn, instanceID, accountID string, cmd ec2v1.EC2InstanceCommand, queueSubject string, queueInput any) (*Out, error) {
-	queue := func() (*Out, error) {
-		return natsmsg.NATSRequest[Out](ctx, natsConn, queueSubject, queueInput, 30*time.Second, accountID)
-	}
-
-	data, err := json.Marshal(cmd)
-	if err != nil {
-		return nil, errors.New(awserrors.ErrorServerInternal)
-	}
-	msg, err := requestInstanceCmd(ctx, natsConn, instanceID, accountID, data)
-	if err != nil {
-		if errors.Is(err, nats.ErrNoResponders) {
-			return queue()
-		}
-		slog.ErrorContext(ctx, "Owner request failed, outcome unknown", "instance_id", instanceID, "subject", queueSubject, "err", err)
-		return nil, errors.New(awserrors.ErrorServerInternal)
-	}
-
-	responseError, parseErr := awserrors.ValidateErrorPayload(msg.Data)
-	if parseErr != nil {
-		handled := msg.Header.Get(ec2v1.HandledHeader) == "1"
-		if !handled && *responseError.Code == awserrors.ErrorServerInternal {
-			slog.WarnContext(ctx, "Owner did not recognise the command, using the queue group", "instance_id", instanceID, "subject", queueSubject)
-			return queue()
-		}
-		if responseError.Message != nil && *responseError.Message != "" {
-			return nil, awserrors.Errorf(*responseError.Code, "%s", *responseError.Message)
-		}
-		return nil, errors.New(*responseError.Code)
-	}
-
-	var out Out
-	if err := json.Unmarshal(msg.Data, &out); err != nil {
-		return nil, errors.New(awserrors.ErrorServerInternal)
-	}
-	return &out, nil
-}

spinifex/domains/ec2/instance/service_impl.go

@@ -1265,6 +1265,10 @@ func (s *InstanceServiceImpl) StartInstance(ctx context.Context, instance *vm.VM
 				slog.WarnContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
 				return refusal
 			}
+			if errors.Is(err, vm.ErrUEFIClaimHeld) {
+				slog.WarnContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
+				return errInsufficientCapacity
+			}
 			slog.ErrorContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
 			return errors.New(awserrors.ErrorServerInternal)
 		}
@@ -2760,7 +2764,7 @@ func (s *InstanceServiceImpl) capacityRefusal(ctx context.Context, instance *vm.
 func (s *InstanceServiceImpl) launchClaimedInstance(ctx context.Context, instance *vm.VM, undo func()) error {
 	// Before anything is stamped or claimed, so the record handed back is the
 	// one that was claimed.
-	if err := vm.CheckUEFIVarStore(instance); err != nil {
+	if err := s.vmMgr.CheckUEFIVarStore(ctx, instance); err != nil {
 		slog.ErrorContext(ctx, "launchClaimedInstance: refusing to launch a UEFI instance without exactly one variable store",
 			"instanceId", instance.ID, "err", err)
 		undo()
@@ -2893,6 +2897,10 @@ func (s *InstanceServiceImpl) launchClaimedInstance(ctx context.Context, instanc
 		if refusal := volumeHeldElsewhereError(err); refusal != nil {
 			return refusal
 		}
+		// A live node holds the variable store, so another node may take it.
+		if errors.Is(err, vm.ErrUEFIClaimHeld) {
+			return s.capacityRefusal(ctx, instance)
+		}
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 

spinifex/domains/ec2/instance/teardown_outstanding.go

@@ -1,33 +1,12 @@
 package instance
 
-import "context"
-
 // OutstandingInstance is an instance of an account that is not yet fully torn
 // down, with the state its record carries.
 type OutstandingInstance struct {
-	InstanceID string `json:"instance_id"`
+	InstanceID string
 	// State is the record's lifecycle state, such as running or terminated.
-	State string `json:"state"`
+	State string
 	// Teardown is the per-dependent progress (done, pending or failed) the
 	// record carries. It is empty when no dependent has been recorded.
-	Teardown map[string]string `json:"teardown,omitempty"`
-}
-
-// TeardownOutstandingInput is the empty request of the outstanding-teardown
-// read; the account comes from the message header.
-type TeardownOutstandingInput struct{}
-
-// TeardownOutstandingOutput carries every instance not yet fully torn down.
-type TeardownOutstandingOutput struct {
-	Instances []OutstandingInstance `json:"instances"`
-}
-
-// AccountTeardownOutstandingRPC serves AccountTeardownOutstanding over the
-// message bus, so a process without the instance record store can read it.
-func (s *InstanceServiceImpl) AccountTeardownOutstandingRPC(_ context.Context, _ *TeardownOutstandingInput, accountID string) (*TeardownOutstandingOutput, error) {
-	outstanding, err := s.AccountTeardownOutstanding(accountID)
-	if err != nil {
-		return nil, err
-	}
-	return &TeardownOutstandingOutput{Instances: outstanding}, nil
+	Teardown map[string]string
 }

spinifex/domains/ec2/instance/uefistore/cleanup.go

@@ -25,9 +25,9 @@ func (s *Store) Exists(ctx context.Context, instanceID string) (bool, error) {
 	return s.kv.Exists(ctx, instanceID)
 }
 
-// MarkDeleting records that instanceID's store is to be deleted, and drops any
-// write-back still owed. It reports false with no error when the instance has
-// no store record. Only a terminate or an operator's forget may call it.
+// MarkDeleting records that instanceID's store is to be deleted. It reports
+// false with no error when the instance has no store record. Only a terminate
+// or an operator's forget may call it.
 func (s *Store) MarkDeleting(ctx context.Context, instanceID string) (bool, error) {
 	for range markAttempts {
 		rec, rev, err := s.Get(ctx, instanceID)
@@ -41,7 +41,6 @@ func (s *Store) MarkDeleting(ctx context.Context, instanceID string) (bool, erro
 			return true, nil
 		}
 		rec.State = StateDeleting
-		rec.WriteBack = nil
 		_, err = s.CompareAndSet(ctx, rec, rev)
 		if err == nil {
 			return true, nil

spinifex/domains/ec2/instance/uefistore/cleanup_test.go

@@ -37,7 +37,7 @@ func keysUnder(t *testing.T, objects objectstore.ObjectStore, prefix string) []s
 }
 
 // Deletion is final: a writer holding the revision of a deleting record, such
-// as a write-back retry, must not bring the store back to live.
+// as a late pointer move, must not bring the store back to live.
 func TestCompareAndSet_RefusesDeletingToLive(t *testing.T) {
 	s, _ := openStore(t)
 	rev, err := s.Create(t.Context(), liveRecord())
@@ -47,8 +47,7 @@ func TestCompareAndSet_RefusesDeletingToLive(t *testing.T) {
 	rev, err = s.CompareAndSet(t.Context(), deleting, rev)
 	require.NoError(t, err)
 
-	revived := liveRecord()
-	revived.WriteBack = writeBack()
+	revived := pointedRecord()
 	_, err = s.CompareAndSet(t.Context(), revived, rev)
 	require.ErrorIs(t, err, uefistore.ErrDeleting)
 
@@ -58,24 +57,9 @@ func TestCompareAndSet_RefusesDeletingToLive(t *testing.T) {
 	assert.Equal(t, uefistore.StateDeleting, got.State)
 }
 
-// A write-back owed on a deleting store would put an object behind the delete.
-func TestCompareAndSet_RefusesDeletingWithAWriteBack(t *testing.T) {
+func TestMarkDeleting_MovesLiveToDeletingAndKeepsWhatItNames(t *testing.T) {
 	s, _ := openStore(t)
-	rev, err := s.Create(t.Context(), liveRecord())
-	require.NoError(t, err)
-	rec := liveRecord()
-	rec.State = uefistore.StateDeleting
-	rec.WriteBack = writeBack()
-
-	_, err = s.CompareAndSet(t.Context(), rec, rev)
-	require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
-}
-
-func TestMarkDeleting_MovesLiveToDeletingAndDropsTheWriteBack(t *testing.T) {
-	s, _ := openStore(t)
-	rec := liveRecord()
-	rec.WriteBack = writeBack()
-	_, err := s.Create(t.Context(), rec)
+	_, err := s.Create(t.Context(), pointedRecord())
 	require.NoError(t, err)
 
 	has, err := s.MarkDeleting(t.Context(), instanceID)
@@ -85,8 +69,8 @@ func TestMarkDeleting_MovesLiveToDeletingAndDropsTheWriteBack(t *testing.T) {
 	got, rev, err := s.Get(t.Context(), instanceID)
 	require.NoError(t, err)
 	assert.Equal(t, uefistore.StateDeleting, got.State)
-	assert.Nil(t, got.WriteBack)
 	assert.Equal(t, "111122223333", got.AccountID)
+	assert.Equal(t, pointer("b"), got.Pointer, "the pointer stays readable until the prefix is gone")
 
 	has, err = s.MarkDeleting(t.Context(), instanceID)
 	require.NoError(t, err)

spinifex/domains/ec2/instance/uefistore/fence.go

@@ -0,0 +1,93 @@
+package uefistore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+)
+
+// ClaimPrefix is the kvsync prefix of every store's claim. An instance's claim
+// is ClaimPrefix+instanceID, and names the node that may hold variable changes
+// no object has yet.
+const ClaimPrefix = "uefi."
+
+// NewClaims returns node's group of store claims over store. The daemon builds
+// it once per process, as kvsync requires of every prefix.
+func NewClaims(store kvsync.KVStore, node string) *kvsync.ClaimGroup {
+	return kvsync.NewClaimGroup(store, ClaimPrefix, node)
+}
+
+// ErrFenced is a stamp or pointer move refused because the record carries a
+// claim generation other than the caller's: a later acquisition has stamped
+// it, so the caller no longer owns the store and must change nothing.
+var ErrFenced = errors.New("uefistore: store record is fenced to another claim generation")
+
+// fenceAttempts bounds the read-then-write loops of Stamp and MovePointer. A
+// lost race is re-read, so a concurrent stamp is seen and refused, not retried.
+const fenceAttempts = 5
+
+// Stamp raises instanceID's record to generation, the claim token the caller
+// has just acquired or adopted, and returns the record it stamped, whose
+// pointer the caller boots from. One above generation is ErrFenced, and one
+// deleting is ErrDeleting. A record already at generation is still written by
+// CAS, because a follower can answer the read with an older pointer.
+func (s *Store) Stamp(ctx context.Context, instanceID string, generation uint64) (*UEFIStoreRecord, error) {
+	if generation == 0 {
+		return nil, fmt.Errorf("uefistore: stamp %s: generation 0 is no claim", instanceID)
+	}
+	for range fenceAttempts {
+		rec, rev, err := s.Get(ctx, instanceID)
+		if err != nil {
+			return nil, err
+		}
+		switch {
+		case rec.State == StateDeleting:
+			return nil, fmt.Errorf("%w: %s", ErrDeleting, instanceID)
+		case rec.Generation > generation:
+			return nil, fmt.Errorf("%w: %s is at generation %d, above the claim's %d", ErrFenced, instanceID, rec.Generation, generation)
+		}
+		rec.Generation = generation
+		_, err = s.CompareAndSet(ctx, rec, rev)
+		if err == nil {
+			return rec, nil
+		}
+		if !errors.Is(err, kvstore.ErrConflict) {
+			return nil, err
+		}
+	}
+	return nil, fmt.Errorf("uefistore: stamp %s: %w after %d attempts", instanceID, kvstore.ErrConflict, fenceAttempts)
+}
+
+// MovePointer points instanceID's record at ref only while the record is at
+// generation. A record at any other generation is ErrFenced and is left
+// unchanged. Every move is a CAS at the revision read, even to the current
+// key, so a read a follower answered stale cannot report success.
+func (s *Store) MovePointer(ctx context.Context, instanceID string, generation uint64, ref UEFIVarStoreRef) error {
+	if err := ref.validate(instanceID); err != nil {
+		return fmt.Errorf("%w: move %s: %w", ErrInvalidRecord, instanceID, err)
+	}
+	for range fenceAttempts {
+		rec, rev, err := s.Get(ctx, instanceID)
+		if err != nil {
+			return err
+		}
+		switch {
+		case rec.State == StateDeleting:
+			return fmt.Errorf("%w: %s", ErrDeleting, instanceID)
+		case rec.Generation != generation:
+			return fmt.Errorf("%w: %s is at generation %d, and the writer holds %d", ErrFenced, instanceID, rec.Generation, generation)
+		}
+		rec.Pointer = &ref
+		_, err = s.CompareAndSet(ctx, rec, rev)
+		if err == nil {
+			return nil
+		}
+		if !errors.Is(err, kvstore.ErrConflict) {
+			return err
+		}
+	}
+	return fmt.Errorf("uefistore: move %s: %w after %d attempts", instanceID, kvstore.ErrConflict, fenceAttempts)
+}

spinifex/domains/ec2/instance/uefistore/fence_test.go

@@ -0,0 +1,213 @@
+package uefistore_test
+
+import (
+	"testing"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func revisionOf(t *testing.T, s *uefistore.Store) uint64 {
+	t.Helper()
+	_, rev, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	return rev
+}
+
+func TestStamp_RaisesTheGenerationAndReturnsThePointerToBootFrom(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), pointedRecord())
+	require.NoError(t, err)
+
+	rec, err := s.Stamp(t.Context(), instanceID, 9)
+	require.NoError(t, err)
+	assert.Equal(t, uint64(9), rec.Generation)
+	assert.Equal(t, pointer("b"), rec.Pointer)
+
+	got, _, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, rec, got, "the returned record is the one stored")
+}
+
+// A daemon restart adopts its claim at the same generation. The stamp still
+// writes by CAS, so a pointer read from a lagging follower fails the write and
+// is read again rather than booted from.
+func TestStamp_AtTheRecordsGenerationConfirmsTheReadByCAS(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), pointedRecord())
+	require.NoError(t, err)
+	before := revisionOf(t, s)
+
+	rec, err := s.Stamp(t.Context(), instanceID, 7)
+	require.NoError(t, err)
+	assert.Equal(t, pointer("b"), rec.Pointer)
+	assert.Equal(t, uint64(7), rec.Generation)
+	assert.Greater(t, revisionOf(t, s), before, "the read must be confirmed by a revision-checked write")
+}
+
+func TestStamp_RefusesAndChangesNothing(t *testing.T) {
+	tests := []struct {
+		name       string
+		deleting   bool
+		generation uint64
+		want       error
+	}{
+		{name: "a later acquisition stamped it", generation: 6, want: uefistore.ErrFenced},
+		{name: "the store is being deleted", deleting: true, generation: 8, want: uefistore.ErrDeleting},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s, _ := openStore(t)
+			rev, err := s.Create(t.Context(), pointedRecord())
+			require.NoError(t, err)
+			if tt.deleting {
+				_, err = s.MarkDeleting(t.Context(), instanceID)
+				require.NoError(t, err)
+				rev = revisionOf(t, s)
+			}
+
+			_, err = s.Stamp(t.Context(), instanceID, tt.generation)
+			require.ErrorIs(t, err, tt.want)
+			assert.Equal(t, rev, revisionOf(t, s))
+		})
+	}
+}
+
+func TestStamp_MissingRecordIsNotFound(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Stamp(t.Context(), instanceID, 1)
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+}
+
+// The record keeps only the current version, so each move replaces the
+// pointer and names nothing it replaced.
+func TestMovePointer_ReplacesThePointer(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), pointedRecord())
+	require.NoError(t, err)
+
+	for _, name := range []string{"c", "d"} {
+		require.NoError(t, s.MovePointer(t.Context(), instanceID, 7, *pointer(name)))
+	}
+
+	got, _, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, pointer("d"), got.Pointer)
+	assert.Equal(t, uint64(7), got.Generation)
+}
+
+// A move to the key the record already names is still a CAS at the revision
+// read, so a fenced writer that read a lagging follower cannot see success.
+func TestMovePointer_ToTheCurrentKeyConfirmsTheReadByCAS(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), pointedRecord())
+	require.NoError(t, err)
+	before := revisionOf(t, s)
+
+	require.NoError(t, s.MovePointer(t.Context(), instanceID, 7, *pointer("b")))
+	assert.Greater(t, revisionOf(t, s), before)
+}
+
+func TestMovePointer_RefusesAndChangesNothing(t *testing.T) {
+	tests := []struct {
+		name       string
+		deleting   bool
+		generation uint64
+		ref        *uefistore.UEFIVarStoreRef
+		want       error
+	}{
+		{name: "an earlier generation", generation: 6, ref: pointer("c"), want: uefistore.ErrFenced},
+		{name: "a later generation than the stamp", generation: 8, ref: pointer("c"), want: uefistore.ErrFenced},
+		{name: "a deleting store", deleting: true, generation: 7, ref: pointer("c"), want: uefistore.ErrDeleting},
+		{name: "a key under another instance", generation: 7,
+			ref:  &uefistore.UEFIVarStoreRef{Key: uefistore.KeyPrefix("i-0fedcba9876543210") + "c", Firmware: pointer("c").Firmware, WriterNode: "n"},
+			want: uefistore.ErrInvalidRecord},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s, _ := openStore(t)
+			_, err := s.Create(t.Context(), pointedRecord())
+			require.NoError(t, err)
+			if tt.deleting {
+				_, err = s.MarkDeleting(t.Context(), instanceID)
+				require.NoError(t, err)
+			}
+			rev := revisionOf(t, s)
+
+			err = s.MovePointer(t.Context(), instanceID, tt.generation, *tt.ref)
+			require.ErrorIs(t, err, tt.want)
+			got, gotRev, err := s.Get(t.Context(), instanceID)
+			require.NoError(t, err)
+			assert.Equal(t, rev, gotRev)
+			assert.Equal(t, pointer("b"), got.Pointer)
+		})
+	}
+}
+
+// claimedStore is a store record and a claim group per node over one
+// JetStream, as two daemons see them.
+type claimedStore struct {
+	records *uefistore.Store
+	a, b    *kvsync.Claim
+}
+
+func newClaimedStore(t *testing.T) *claimedStore {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	records := uefistore.NewStore(js)
+	require.NoError(t, records.Open(t.Context()))
+	_, err := records.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+	return &claimedStore{
+		records: records,
+		a:       uefistore.NewClaims(kvstore.OpenSyncBucket(js), "node-a").Get(instanceID),
+		b:       uefistore.NewClaims(kvstore.OpenSyncBucket(js), "node-b").Get(instanceID),
+	}
+}
+
+// A node partitioned long enough to be taken over is fenced by the new
+// owner's stamp: its pointer move fails and the record keeps what it had.
+func TestFence_TheOldHoldersMoveFailsAfterTheNewOwnersStamp(t *testing.T) {
+	c := newClaimedStore(t)
+	first, err := c.a.TryLock(t.Context())
+	require.NoError(t, err)
+	_, err = c.records.Stamp(t.Context(), instanceID, first.Generation)
+	require.NoError(t, err)
+	require.NoError(t, c.records.MovePointer(t.Context(), instanceID, first.Generation, *pointer("b")))
+
+	second, err := c.b.TakeOver(t.Context(), first)
+	require.NoError(t, err)
+	_, err = c.records.Stamp(t.Context(), instanceID, second.Generation)
+	require.NoError(t, err)
+	rev := revisionOf(t, c.records)
+
+	err = c.records.MovePointer(t.Context(), instanceID, first.Generation, *pointer("late"))
+	require.ErrorIs(t, err, uefistore.ErrFenced)
+	got, gotRev, err := c.records.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, rev, gotRev)
+	assert.Equal(t, pointer("b"), got.Pointer)
+	assert.Equal(t, second.Generation, got.Generation)
+}
+
+// The new owner reads the pointer after it stamps, so a move the old holder
+// landed between the takeover and the stamp is the one it boots from.
+func TestFence_AMoveBetweenTakeoverAndStampIsThePointerTheNewOwnerBoots(t *testing.T) {
+	c := newClaimedStore(t)
+	first, err := c.a.TryLock(t.Context())
+	require.NoError(t, err)
+	_, err = c.records.Stamp(t.Context(), instanceID, first.Generation)
+	require.NoError(t, err)
+
+	second, err := c.b.TakeOver(t.Context(), first)
+	require.NoError(t, err)
+	require.NoError(t, c.records.MovePointer(t.Context(), instanceID, first.Generation, *pointer("landed")))
+
+	rec, err := c.records.Stamp(t.Context(), instanceID, second.Generation)
+	require.NoError(t, err)
+	assert.Equal(t, keyOf("landed"), rec.Pointer.Key)
+}

spinifex/domains/ec2/instance/uefistore/uefistore.go

@@ -1,6 +1,6 @@
 // Package uefistore holds the types that describe an instance's UEFI variable
-// store and the bucket of store records that carry its deletion and write-back
-// obligations past the life of the instance record.
+// store, the bucket of store records that carry its pointer and deletion
+// obligation past the life of the instance record, and the claim that fences it.
 package uefistore
 
 import (
@@ -53,22 +53,27 @@ type FirmwareIdentity struct {
 // UEFIStoreRecord is the durable owner of one instance's variable store
 // objects. It outlives the instance record, so an obligation recorded here is
 // not lost when a terminated record expires.
+//
+// Pointer names the only version of the store the record keeps, the object the
+// instance boots from, and only a writer whose claim token has generation
+// Generation may move it.
 type UEFIStoreRecord struct {
-	SchemaVersion uint16         `json:"schema_version"`
-	InstanceID    string         `json:"instance_id"`
-	AccountID     string         `json:"account_id"`
-	State         string         `json:"state"`
-	WriteBack     *UEFIWriteBack `json:"write_back,omitempty"`
+	SchemaVersion uint16           `json:"schema_version"`
+	InstanceID    string           `json:"instance_id"`
+	AccountID     string           `json:"account_id"`
+	State         string           `json:"state"`
+	Generation    uint64           `json:"generation"`
+	Pointer       *UEFIVarStoreRef `json:"pointer,omitempty"`
 }
 
-// UEFIWriteBack is a write-back that failed and is still owed. Key is the
-// object every retry puts to, so retries leave no extra objects; SeededFrom is
-// the pointer Node seeded from, which a retry must still find on the record.
-type UEFIWriteBack struct {
-	Node       string    `json:"node"`
-	Key        string    `json:"key"`
-	SeededFrom string    `json:"seeded_from"`
-	Since      time.Time `json:"since"`
+// UEFIVarStoreRef names the store object an instance launches from, with the
+// firmware it was written under, so a launch can check it without reading the
+// object's metadata.
+type UEFIVarStoreRef struct {
+	Key        string           `json:"key"`
+	Firmware   FirmwareIdentity `json:"firmware"`
+	WriterNode string           `json:"writer_node"`
+	WrittenAt  time.Time        `json:"written_at"`
 }
 
 // KeyPrefix is the object-store prefix every store object of ownerID (an
@@ -83,8 +88,8 @@ func ownsKey(ownerID, key string) bool {
 	return ok && name != "" && !strings.Contains(name, "/")
 }
 
-// validate checks rec against key. A write-back key outside the instance's own
-// prefix is refused because a superseded obligation deletes that key.
+// validate checks rec against key. The pointer must name one object directly
+// under the instance's own prefix, because a deletion acts on it.
 func (rec *UEFIStoreRecord) validate(key string) error {
 	switch {
 	case rec.SchemaVersion > RecordSchemaVersion:
@@ -98,29 +103,32 @@ func (rec *UEFIStoreRecord) validate(key string) error {
 		return fmt.Errorf("%w: instance_id %s stored at key %s", ErrInvalidRecord, rec.InstanceID, key)
 	case rec.State != StateLive && rec.State != StateDeleting:
 		return fmt.Errorf("%w: %s has state %q", ErrInvalidRecord, key, rec.State)
-	case rec.State == StateDeleting && rec.WriteBack != nil:
-		// A write-back retry puts an object, which would land behind the delete.
-		return fmt.Errorf("%w: %s is deleting with a write-back still owed", ErrInvalidRecord, key)
 	}
-	if wb := rec.WriteBack; wb != nil {
-		switch {
-		case wb.Node == "":
-			return fmt.Errorf("%w: %s write-back names no node", ErrInvalidRecord, key)
-		case wb.Since.IsZero():
-			return fmt.Errorf("%w: %s write-back has no since", ErrInvalidRecord, key)
-		case !ownsKey(rec.InstanceID, wb.Key):
-			return fmt.Errorf("%w: %s write-back key %q is not under %s", ErrInvalidRecord, key, wb.Key, KeyPrefix(rec.InstanceID))
-		case !ownsKey(rec.InstanceID, wb.SeededFrom):
-			return fmt.Errorf("%w: %s write-back seeded_from %q is not under %s", ErrInvalidRecord, key, wb.SeededFrom, KeyPrefix(rec.InstanceID))
+	if p := rec.Pointer; p != nil {
+		if err := p.validate(rec.InstanceID); err != nil {
+			return fmt.Errorf("%w: %s pointer: %w", ErrInvalidRecord, key, err)
 		}
 	}
 	return nil
 }
 
+// validate checks ref as a pointer of instanceID's store.
+func (ref *UEFIVarStoreRef) validate(instanceID string) error {
+	switch {
+	case !ownsKey(instanceID, ref.Key):
+		return fmt.Errorf("key %q is not under %s", ref.Key, KeyPrefix(instanceID))
+	case ref.Firmware.Arch == "" || ref.Firmware.VarsSize <= 0:
+		return fmt.Errorf("key %q names no firmware arch and VARS size", ref.Key)
+	case ref.WriterNode == "":
+		return fmt.Errorf("key %q names no writer node", ref.Key)
+	}
+	return nil
+}
+
 func bucketConfig() kvstore.Config {
 	return kvstore.Config{
 		Name:        BucketName,
-		Description: "Spinifex UEFI variable store records (deletion and write-back obligations)",
+		Description: "Spinifex UEFI variable store records (pointers and deletion obligations)",
 		History:     1,
 		// Left false: a lost bucket has lost obligations, and an empty
 		// recreated one would report every store as having none.

spinifex/domains/ec2/instance/uefistore/uefistore_test.go

@@ -30,15 +30,25 @@ func liveRecord() *uefistore.UEFIStoreRecord {
 	return &uefistore.UEFIStoreRecord{InstanceID: instanceID, AccountID: "111122223333", State: uefistore.StateLive}
 }
 
-func writeBack() *uefistore.UEFIWriteBack {
-	return &uefistore.UEFIWriteBack{
-		Node:       "node-2",
-		Key:        uefistore.KeyPrefix(instanceID) + "b",
-		SeededFrom: uefistore.KeyPrefix(instanceID) + "a",
-		Since:      time.Date(2026, 11, 3, 1, 2, 3, 0, time.UTC),
+func pointer(name string) *uefistore.UEFIVarStoreRef {
+	return &uefistore.UEFIVarStoreRef{
+		Key:        keyOf(name),
+		Firmware:   uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: 540672, CodeSHA256: "c0de"},
+		WriterNode: "node-2",
+		WrittenAt:  time.Date(2026, 11, 3, 1, 2, 3, 0, time.UTC),
 	}
 }
 
+func keyOf(name string) string { return uefistore.KeyPrefix(instanceID) + name }
+
+// pointedRecord is a live record stamped at generation 7 that points at "b".
+func pointedRecord() *uefistore.UEFIStoreRecord {
+	rec := liveRecord()
+	rec.Generation = 7
+	rec.Pointer = pointer("b")
+	return rec
+}
+
 func rawPut(t *testing.T, js jetstream.JetStream, key, body string) {
 	t.Helper()
 	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
@@ -66,8 +76,7 @@ func TestOpen_CreatesBucketWithNoTTLAndStampsVersion(t *testing.T) {
 
 func TestCreate_RoundTripsAndStampsSchemaVersion(t *testing.T) {
 	s, _ := openStore(t)
-	rec := liveRecord()
-	rec.WriteBack = writeBack()
+	rec := pointedRecord()
 
 	rev, err := s.Create(t.Context(), rec)
 	require.NoError(t, err)
@@ -82,9 +91,7 @@ func TestCreate_RoundTripsAndStampsSchemaVersion(t *testing.T) {
 // The wire names are what every release reads, so they are pinned.
 func TestRecord_WireShape(t *testing.T) {
 	s, js := openStore(t)
-	rec := liveRecord()
-	rec.WriteBack = writeBack()
-	_, err := s.Create(t.Context(), rec)
+	_, err := s.Create(t.Context(), pointedRecord())
 	require.NoError(t, err)
 
 	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
@@ -96,11 +103,12 @@ func TestRecord_WireShape(t *testing.T) {
 		"instance_id": "i-0123456789abcdef0",
 		"account_id": "111122223333",
 		"state": "live",
-		"write_back": {
-			"node": "node-2",
+		"generation": 7,
+		"pointer": {
 			"key": "uefi/i-0123456789abcdef0/b",
-			"seeded_from": "uefi/i-0123456789abcdef0/a",
-			"since": "2026-11-03T01:02:03Z"
+			"firmware": {"arch": "x86_64", "vars_size": 540672, "code_sha256": "c0de"},
+			"writer_node": "node-2",
+			"written_at": "2026-11-03T01:02:03Z"
 		}
 	}`, string(entry.Value()))
 }
@@ -144,8 +152,7 @@ func TestCompareAndSet_StaleRevisionConflictsAndWritesNothing(t *testing.T) {
 	first, err := s.Create(t.Context(), liveRecord())
 	require.NoError(t, err)
 
-	moved := liveRecord()
-	moved.WriteBack = writeBack()
+	moved := pointedRecord()
 	second, err := s.CompareAndSet(t.Context(), moved, first)
 	require.NoError(t, err)
 	assert.Greater(t, second, first)
@@ -159,7 +166,7 @@ func TestCompareAndSet_StaleRevisionConflictsAndWritesNothing(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, second, rev)
 	assert.Equal(t, uefistore.StateLive, got.State)
-	assert.Equal(t, writeBack(), got.WriteBack)
+	assert.Equal(t, pointer("b"), got.Pointer)
 }
 
 // Two actors that read the same revision race to change it; exactly one may
@@ -179,12 +186,11 @@ func TestCompareAndSet_ConcurrentWritersFromOneReadHaveOneWinner(t *testing.T) {
 	)
 	for i := range writers {
 		wg.Go(func() {
-			rec := liveRecord()
-			rec.WriteBack = writeBack()
-			rec.WriteBack.Node = fmt.Sprintf("node-%d", i)
+			rec := pointedRecord()
+			rec.Pointer.WriterNode = fmt.Sprintf("node-%d", i)
 			if _, err := s.CompareAndSet(context.Background(), rec, rev); err == nil {
 				mu.Lock()
-				wins = append(wins, rec.WriteBack.Node)
+				wins = append(wins, rec.Pointer.WriterNode)
 				mu.Unlock()
 			} else {
 				assert.ErrorIs(t, err, kvstore.ErrConflict)
@@ -196,7 +202,7 @@ func TestCompareAndSet_ConcurrentWritersFromOneReadHaveOneWinner(t *testing.T) {
 	require.Len(t, wins, 1)
 	got, _, err := s.Get(t.Context(), instanceID)
 	require.NoError(t, err)
-	assert.Equal(t, wins[0], got.WriteBack.Node)
+	assert.Equal(t, wins[0], got.Pointer.WriterNode)
 }
 
 func TestCompareAndDelete_StaleRevisionKeepsRecord(t *testing.T) {
@@ -217,29 +223,20 @@ func TestCompareAndDelete_StaleRevisionKeepsRecord(t *testing.T) {
 	require.ErrorIs(t, err, kvstore.ErrNotFound)
 }
 
-// A superseded obligation deletes its recorded key, so a key outside the
+// A deletion acts on the key the record points at, so a key outside the
 // instance's own prefix must never reach the bucket.
 func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
+	other := uefistore.KeyPrefix("i-0fedcba9876543210") + "b"
 	cases := map[string]func(*uefistore.UEFIStoreRecord){
-		"unknown state":            func(r *uefistore.UEFIStoreRecord) { r.State = "gone" },
-		"not an instance ID":       func(r *uefistore.UEFIStoreRecord) { r.InstanceID = "ami-0123456789abcdef0" },
-		"write-back with no node":  func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Node = "" },
-		"write-back with no since": func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Since = time.Time{} },
-		"write-back key under another instance": func(r *uefistore.UEFIStoreRecord) {
-			r.WriteBack.Key = uefistore.KeyPrefix("i-0fedcba9876543210") + "b"
-		},
-		"write-back key under an image": func(r *uefistore.UEFIStoreRecord) {
-			r.WriteBack.Key = uefistore.KeyPrefix("ami-0123456789abcdef0") + "b"
-		},
-		"write-back key is the prefix itself": func(r *uefistore.UEFIStoreRecord) {
-			r.WriteBack.Key = uefistore.KeyPrefix(instanceID)
-		},
-		"write-back key nested below the prefix": func(r *uefistore.UEFIStoreRecord) {
-			r.WriteBack.Key = uefistore.KeyPrefix(instanceID) + "x/y"
-		},
-		"write-back seeded from another instance": func(r *uefistore.UEFIStoreRecord) {
-			r.WriteBack.SeededFrom = uefistore.KeyPrefix("i-0fedcba9876543210") + "a"
-		},
+		"unknown state":                   func(r *uefistore.UEFIStoreRecord) { r.State = "gone" },
+		"not an instance ID":              func(r *uefistore.UEFIStoreRecord) { r.InstanceID = "ami-0123456789abcdef0" },
+		"pointer under another instance":  func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = other },
+		"pointer under an image":          func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = uefistore.KeyPrefix("ami-0123456789abcdef0") + "b" },
+		"pointer is the prefix itself":    func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = uefistore.KeyPrefix(instanceID) },
+		"pointer nested below the prefix": func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = keyOf("x/y") },
+		"pointer with no firmware arch":   func(r *uefistore.UEFIStoreRecord) { r.Pointer.Firmware.Arch = "" },
+		"pointer with no VARS size":       func(r *uefistore.UEFIStoreRecord) { r.Pointer.Firmware.VarsSize = 0 },
+		"pointer with no writer node":     func(r *uefistore.UEFIStoreRecord) { r.Pointer.WriterNode = "" },
 	}
 	for name, mutate := range cases {
 		t.Run(name, func(t *testing.T) {
@@ -247,8 +244,7 @@ func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
 			rev, err := s.Create(t.Context(), liveRecord())
 			require.NoError(t, err)
 
-			rec := liveRecord()
-			rec.WriteBack = writeBack()
+			rec := pointedRecord()
 			mutate(rec)
 			_, err = s.CompareAndSet(t.Context(), rec, rev)
 			require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
@@ -256,7 +252,7 @@ func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
 			got, gotRev, err := s.Get(t.Context(), instanceID)
 			require.NoError(t, err)
 			assert.Equal(t, rev, gotRev, "an invalid record must not be written")
-			assert.Nil(t, got.WriteBack)
+			assert.Nil(t, got.Pointer)
 		})
 	}
 }
@@ -265,11 +261,11 @@ func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
 // is reported rather than decoded into something a reaper would act on.
 func TestGet_RefusesRecordsItCannotTrust(t *testing.T) {
 	cases := map[string]string{
-		"newer schema":       `{"schema_version":2,"instance_id":"i-0123456789abcdef0","state":"live"}`,
-		"no schema":          `{"instance_id":"i-0123456789abcdef0","state":"live"}`,
-		"another instance":   `{"schema_version":1,"instance_id":"i-0fedcba9876543210","state":"deleting"}`,
-		"unknown state":      `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"purged"}`,
-		"foreign write-back": `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"live","write_back":{"node":"n","key":"uefi/i-0fedcba9876543210/b","seeded_from":"uefi/i-0123456789abcdef0/a","since":"2026-11-03T00:00:00Z"}}`,
+		"newer schema":     `{"schema_version":2,"instance_id":"i-0123456789abcdef0","state":"live"}`,
+		"no schema":        `{"instance_id":"i-0123456789abcdef0","state":"live"}`,
+		"another instance": `{"schema_version":1,"instance_id":"i-0fedcba9876543210","state":"deleting"}`,
+		"unknown state":    `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"purged"}`,
+		"foreign pointer":  `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"live","generation":3,"pointer":{"key":"uefi/i-0fedcba9876543210/b","firmware":{"arch":"x86_64","vars_size":540672},"writer_node":"n"}}`,
 	}
 	for name, body := range cases {
 		t.Run(name, func(t *testing.T) {

spinifex/domains/ec2/instance/volume_held_elsewhere_test.go

@@ -102,3 +102,38 @@ func TestStartPaths_ReportAVolumeHeldElsewhereByCode(t *testing.T) {
 		assert.Equal(t, awserrors.ErrorIncorrectState, awserrors.ValidErrorCodeFromError(err))
 	})
 }
+
+// uefiClaimHeldMounter fails the launch the way a start does while a live
+// node holds the instance's UEFI variable store.
+type uefiClaimHeldMounter struct{ raceVolumeMounter }
+
+func (uefiClaimHeldMounter) Mount(context.Context, *vm.VM) error {
+	return fmt.Errorf("%w: node-a holds i-uefi: %w", vm.ErrUEFIClaimHeld, vm.ErrMountRetryable)
+}
+
+// A held UEFI store is a capacity refusal on both start paths, so the caller
+// tries another node rather than reporting a fault.
+func TestStartPaths_ReportAHeldUEFIStoreAsCapacity(t *testing.T) {
+	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
+	types, _ := defaultPrepareInstanceTypes()
+
+	t.Run("claimed relaunch", func(t *testing.T) {
+		mgr := vm.NewManager()
+		mgr.SetDeps(vm.Deps{NodeID: "node-b", VolumeMounter: uefiClaimHeldMounter{}})
+		svc := &InstanceServiceImpl{resourceMgr: &fakeResourceCapacityProvider{instanceTypes: types}, vmMgr: mgr}
+
+		err := svc.launchClaimedInstance(t.Context(), &vm.VM{ID: "i-uefi-claim", AccountID: "acc", InstanceType: "t3.micro", Status: vm.StatePending}, func() {})
+		assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, awserrors.ValidErrorCodeFromError(err))
+	})
+
+	t.Run("live owner", func(t *testing.T) {
+		const id = "i-uefi-live"
+		mgr := mgrWith(map[string]*vm.VM{id: {ID: id, Status: vm.StateStopped, InstanceType: "t3.micro"}})
+		mgr.SetDeps(vm.Deps{NodeID: "node-b", VolumeMounter: uefiClaimHeldMounter{}})
+		v, _ := mgr.Get(id)
+		svc := &InstanceServiceImpl{resourceMgr: &fakeResourceCapacityProvider{instanceTypes: types}, vmMgr: mgr}
+
+		err := svc.StartInstance(t.Context(), v, ec2v1.EC2InstanceCommand{ID: id})
+		assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, awserrors.ValidErrorCodeFromError(err))
+	})
+}

spinifex/domains/ec2/inventory/inventory.go

@@ -1,67 +0,0 @@
-// Package inventory is the EC2-owned read capability for an account's complete
-// instance list, for callers outside the EC2 domain.
-package inventory
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-
-	"github.com/aws/aws-sdk-go/service/ec2"
-	ec2instanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/instance"
-	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/nats-io/nats.go"
-	"github.com/nats-io/nats.go/jetstream"
-)
-
-// Instances lists every non-terminated instance an account owns, or returns an
-// error. It never returns a shorter list than the truth.
-type Instances interface {
-	ListAccountInstances(ctx context.Context, accountID string) (*ec2.DescribeInstancesOutput, error)
-}
-
-// NATSInstances reads the complete list across nodeIDs over NATS.
-type NATSInstances struct {
-	nc      *nats.Conn
-	nodeIDs []string
-}
-
-var _ Instances = (*NATSInstances)(nil)
-
-// NewNATSInstances returns a reader over the configured node set.
-func NewNATSInstances(nc *nats.Conn, nodeIDs []string) *NATSInstances {
-	return &NATSInstances{nc: nc, nodeIDs: nodeIDs}
-}
-
-// ListAccountInstances passes through the strict listing unchanged: any node
-// that did not answer, or any record that could not be read, is an error.
-func (n *NATSInstances) ListAccountInstances(ctx context.Context, accountID string) (*ec2.DescribeInstancesOutput, error) {
-	return ec2instanceapi.DescribeAccountInstancesStrict(ctx, n.nc, n.nodeIDs, accountID)
-}
-
-// NewENITagReader returns a reader of one ENI's tags straight from its stored
-// record, so an ENI that is absent and one that cannot be read are told apart:
-// a missing key is found=false, a failed read or an undecodable record is an
-// error. accountID is the account the ENI is stored under.
-func NewENITagReader(js jetstream.JetStream, accountID string) func(ctx context.Context, eniID string) (map[string]string, bool, error) {
-	return func(ctx context.Context, eniID string) (map[string]string, bool, error) {
-		kv, err := js.KeyValue(ctx, ec2vpc.KVBucketENIs)
-		if err != nil {
-			return nil, false, fmt.Errorf("open ENI bucket: %w", err)
-		}
-		entry, err := kv.Get(ctx, kvstore.AccountKey(accountID, eniID))
-		if errors.Is(err, jetstream.ErrKeyNotFound) {
-			return nil, false, nil
-		}
-		if err != nil {
-			return nil, false, fmt.Errorf("read ENI record %s: %w", eniID, err)
-		}
-		var record ec2vpc.ENIRecord
-		if err := json.Unmarshal(entry.Value(), &record); err != nil {
-			return nil, false, fmt.Errorf("decode ENI record %s: %w", eniID, err)
-		}
-		return record.Tags, true, nil
-	}
-}

spinifex/domains/ec2/inventory/inventory_test.go

@@ -1,49 +0,0 @@
-package inventory
-
-import (
-	"encoding/json"
-	"testing"
-
-	testutil "github.com/mulgadc/spinifex/internal/testkit"
-	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestListAccountInstancesRefusesAnUnknownNodeSet(t *testing.T) {
-	out, err := NewNATSInstances(nil, nil).ListAccountInstances(t.Context(), "000000000000")
-	require.Error(t, err)
-	assert.Nil(t, out, "an unknown node set must not yield a list")
-}
-
-func TestENITagReaderSeparatesGoneFromUnreadable(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	js := testutil.NewJetStream(t, nc)
-	read := NewENITagReader(js, "000000000000")
-
-	_, _, err := read(t.Context(), "eni-1")
-	require.Error(t, err, "a missing ENI bucket is a read failure, not an absent ENI")
-
-	enis, err := kvstore.GetOrCreateBucket(t.Context(), js, ec2vpc.KVBucketENIs, 10)
-	require.NoError(t, err)
-	good, err := json.Marshal(ec2vpc.ENIRecord{NetworkInterfaceId: "eni-good", Tags: map[string]string{"k": "v"}})
-	require.NoError(t, err)
-	_, err = enis.Put(t.Context(), kvstore.AccountKey("000000000000", "eni-good"), good)
-	require.NoError(t, err)
-	_, err = enis.Put(t.Context(), kvstore.AccountKey("000000000000", "eni-corrupt"), []byte("{not json"))
-	require.NoError(t, err)
-
-	tags, found, err := read(t.Context(), "eni-good")
-	require.NoError(t, err)
-	assert.True(t, found)
-	assert.Equal(t, map[string]string{"k": "v"}, tags)
-
-	_, found, err = read(t.Context(), "eni-absent")
-	require.NoError(t, err)
-	assert.False(t, found, "a missing key is the ENI gone")
-
-	_, found, err = read(t.Context(), "eni-corrupt")
-	require.Error(t, err, "an undecodable record is a read failure")
-	assert.False(t, found)
-}

spinifex/domains/ec2/systeminstance/launcher.go

@@ -57,7 +57,7 @@ const (
 	BootAMI BootMode = "ami"
 )
 
-// SystemInstanceLauncher launches, reboots and terminates system-managed VMs.
+// SystemInstanceLauncher launches and terminates system-managed VMs.
 // Defined here to avoid a circular import between elbv2/eks and daemon.
 type SystemInstanceLauncher interface {
 	// LaunchSystemInstance creates and starts a system-managed VM, returning its ID and private IP.
@@ -65,10 +65,6 @@ type SystemInstanceLauncher interface {
 
 	// TerminateSystemInstance stops and cleans up a system-managed VM.
 	TerminateSystemInstance(instanceID string) error
-
-	// RebootSystemInstance restarts a system-managed load balancer VM into a
-	// fresh QEMU on its owning node, keeping its ID, ENIs and addresses.
-	RebootSystemInstance(instanceID string) error
 }
 
 // SystemInstanceInput describes a system VM to launch.

spinifex/domains/ecs/eni_test.go

@@ -322,7 +322,7 @@ func TestReaper_Awsvpc_ReleasesENI(t *testing.T) {
 	rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
 	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
 
-	sc := NewScheduler(nc, svc, "test-holder")
+	sc := testScheduler(t, nc, svc, "test-holder")
 	sc.reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
 
 	assert.Equal(t, 1, eni.releaseCalls)

spinifex/domains/ecs/enumeration_failure_test.go

@@ -27,7 +27,7 @@ func dropRevisit(pass func(context.Context) (time.Duration, error)) func(context
 // drained, services never relaunched, stopped tasks never pruned.
 func TestSchedulerPasses_EnumerationFailureIsReported(t *testing.T) {
 	svc, nc := newTestService(t)
-	sc := NewScheduler(nc, svc, "test-holder")
+	sc := testScheduler(t, nc, svc, "test-holder")
 	_, err := svc.CreateCluster(context.Background(), &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
 	require.NoError(t, err)
 

spinifex/domains/ecs/instance_recovery_test.go

@@ -49,7 +49,7 @@ func TestRegister_RestoresReapedInstanceToActive(t *testing.T) {
 	rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
 	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
 
-	NewScheduler(nc, svc, "test-holder").reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
+	testScheduler(t, nc, svc, "test-holder").reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
 
 	reaped := instanceStatus(t, svc, "web", "i-1")
 	require.Equal(t, InstanceStatusDraining, reaped.Status)
@@ -89,7 +89,7 @@ func TestReaper_ReleasesCapacityAndRecoversClean(t *testing.T) {
 	seeded.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
 	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &seeded))
 
-	NewScheduler(nc, svc, "test-holder").reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
+	testScheduler(t, nc, svc, "test-holder").reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
 
 	reaped := instanceStatus(t, svc, "web", "i-1")
 	require.Equal(t, InstanceStatusDraining, reaped.Status)

spinifex/domains/ecs/revisit_test.go

@@ -50,7 +50,7 @@ func schedulerRig(t *testing.T) (*Scheduler, *Service, jetstream.KeyValue) {
 	registerInstance(t, svc, "web", "i-1", 4096, 8192)
 	kv, err := svc.bucket(t.Context(), testAccountID)
 	require.NoError(t, err)
-	return NewScheduler(nc, svc, "test-holder"), svc, kv
+	return testScheduler(t, nc, svc, "test-holder"), svc, kv
 }
 
 // runOneTask places a task and returns its ID, read back from the record rather

spinifex/domains/ecs/scheduler.go

@@ -13,17 +13,19 @@ import (
 
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/reconciler"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
-// Scheduler timing. The leader lease lives in spinifex-ecs-leader (60s TTL); the
-// holder refreshes well inside that. The reaper marks an instance DRAINING after
-// three missed 30s heartbeats.
+// KeyScheduler is the ECS scheduler's leader lease in the sync bucket.
+const KeyScheduler = "ecs.scheduler"
+
+// Scheduler timing. A follower asks again at the lease's renewal rate. The
+// reaper marks an instance DRAINING after three missed 30s heartbeats.
 const (
-	schedulerLeaderKey  = "scheduler"
-	leaseRefresh        = 20 * time.Second
+	schedulerLeaseTTL   = 60 * time.Second
+	leaseRefresh        = schedulerLeaseTTL / 3
 	reaperInterval      = 30 * time.Second
 	heartbeatTimeout    = 90 * time.Second
 	stoppedReasonReaped = "ContainerInstance disconnected"
@@ -59,17 +61,14 @@ const (
 	reconcileResync = 5 * time.Minute
 )
 
-// Scheduler is the per-daemon ECS control loop. A single leader (elected via the
-// shared leader bucket) owns the Layer-2 bus subscriptions and the heartbeat
+// Scheduler is the per-daemon ECS control loop. A single leader (elected on its
+// lease) owns the Layer-2 bus subscriptions and the heartbeat
 // reaper; losers idle and retry. Placement itself happens synchronously in the
 // RunTask handler, so a brief leaderless gap never blocks RunTask.
 type Scheduler struct {
-	nc     *nats.Conn
-	svc    *Service
-	holder string
-
-	lease    *kvlease.Lease
-	leaseErr error
+	nc    *nats.Conn
+	svc   *Service
+	lease *kvsync.Lease
 
 	// wake carries a bus message that changed something the service reconcile
 	// reads. Fed by the task-state subscription and not by the heartbeat one:
@@ -89,47 +88,28 @@ type Scheduler struct {
 	subs []*nats.Subscription
 }
 
-// NewScheduler constructs a Scheduler. holder identifies this daemon in the lease.
-func NewScheduler(nc *nats.Conn, svc *Service, holder string) *Scheduler {
-	// One slot: it means "something the reconcile reads has changed", and a
-	// second arriving before the pass runs says nothing the first did not.
-	sc := &Scheduler{nc: nc, svc: svc, holder: holder, wake: make(chan struct{}, 1)}
-	sc.lease, sc.leaseErr = kvlease.New(kvlease.Config{
-		Name:   "ecs/scheduler",
-		Bucket: sc.leaderBucket,
-		Key:    schedulerLeaderKey,
-		Holder: holder,
-		TTL:    KVBucketECSLeaderTTL,
-		Renew:  leaseRefresh,
-		Retry:  leaseRefresh,
-		// The leader owns the Layer-2 bus subscriptions and the instance-role
-		// converge pass. A node that can do neither stands down rather than hold
-		// the lease and process nothing.
-		OnGained: sc.onGained,
-		OnLost:   sc.unsubscribeBus,
-	})
-	return sc
+// NewSchedulerLease returns the ECS scheduler's leader lease for holder.
+func NewSchedulerLease(store kvsync.KVStore, holder string) *kvsync.Lease {
+	return kvsync.NewLease(store, KeyScheduler, holder, schedulerLeaseTTL)
 }
 
-func (sc *Scheduler) leaderBucket(ctx context.Context) (jetstream.KeyValue, error) {
-	js, err := jetstream.New(sc.nc)
-	if err != nil {
-		return nil, err
-	}
-	return InitLeaderBucket(ctx, js)
+// NewScheduler constructs a Scheduler that leads while it holds lease.
+func NewScheduler(nc *nats.Conn, svc *Service, lease *kvsync.Lease) *Scheduler {
+	// One slot: it means "something the reconcile reads has changed", and a
+	// second arriving before the pass runs says nothing the first did not.
+	return &Scheduler{nc: nc, svc: svc, lease: lease, wake: make(chan struct{}, 1)}
 }
 
 // Run drives the leadership + reaper loop until ctx is cancelled. It is intended
 // to run as a daemon-boot goroutine; panics are the caller's recover concern.
 func (sc *Scheduler) Run(ctx context.Context) {
-	if sc.leaseErr != nil {
-		slog.ErrorContext(ctx, "ECS scheduler: lease config invalid", "holder", sc.holder, "err", sc.leaseErr)
-		return
-	}
 	leaseDone := make(chan struct{})
 	go func() {
 		defer close(leaseDone)
-		sc.lease.Run(ctx)
+		// The leader owns the Layer-2 bus subscriptions and the instance-role
+		// converge pass. A node that can do neither stands down rather than hold
+		// the lease and process nothing.
+		kvsync.RunElection(ctx, sc.lease, sc.onGained, sc.unsubscribeBus)
 	}()
 
 	// Two loops rather than four tickers, split by what wakes them. Services are

spinifex/domains/ecs/scheduler_test.go

@@ -6,7 +6,9 @@ package ecs
 
 import (
 	"context"
+	"sync"
 	"testing"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ecs"
@@ -14,26 +16,67 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/nats-io/nats.go"
 )
 
+// testScheduler is holder's scheduler on a sync bucket of its own, as a
+// separate daemon's would be.
+func testScheduler(t *testing.T, nc *nats.Conn, svc *Service, holder string) *Scheduler {
+	t.Helper()
+	return NewScheduler(nc, svc, NewSchedulerLease(kvstore.OpenSyncBucket(testutil.NewJetStream(t, nc)), holder))
+}
+
+// electOnce runs sc's election with the scheduler's own callbacks and returns
+// what its first onGained returned. stop ends the election and waits for it.
+func electOnce(t *testing.T, sc *Scheduler) (stop func(), gainedErr error) {
+	t.Helper()
+	ctx, cancel := context.WithCancel(t.Context())
+	gained := make(chan error, 1)
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		kvsync.RunElection(ctx, sc.lease, func(ctx context.Context) error {
+			err := sc.onGained(ctx)
+			gained <- err
+			return err
+		}, sc.unsubscribeBus)
+	}()
+	stop = func() { cancel(); <-done }
+	t.Cleanup(stop)
+	select {
+	case gainedErr = <-gained:
+	case <-time.After(10 * time.Second):
+		t.Fatal("the election never gained the lease")
+	}
+	return sync.OnceFunc(stop), gainedErr
+}
+
+// requireLeaseKeyAbsent fails unless no node holds the scheduler's key.
+func requireLeaseKeyAbsent(t *testing.T, nc *nats.Conn, msg string) {
+	t.Helper()
+	_, _, err := kvstore.OpenSyncBucket(testutil.NewJetStream(t, nc)).Load(t.Context(), KeyScheduler)
+	require.ErrorIs(t, err, kvsync.ErrNotFound, msg)
+}
+
 // TestScheduler_StandsDownWithoutIAM is the fleet outage this guards: a leader
 // that cannot converge instance roles holds the lease, writes no sts:AssumeRole
 // grant, and every agent's credentials lapse an hour later.
 func TestScheduler_StandsDownWithoutIAM(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
+	_, nc, _ := testutil.StartTestJetStream(t)
 	svc := NewService(nc, testRegion, "")
-	sc := NewScheduler(nc, svc, "holder-a")
-	require.NoError(t, sc.leaseErr)
+	sc := testScheduler(t, nc, svc, "holder-a")
 
-	assert.False(t, sc.lease.TryAcquire(t.Context()), "a node without IAM must refuse the lease")
+	stop, err := electOnce(t, sc)
+	require.Error(t, err, "a node without IAM must refuse the lease")
+	stop()
 	assert.False(t, sc.lease.Held())
 	assert.Zero(t, sc.subCount())
 
 	// The key must be free, or a healthy node waits out the full TTL.
-	kv, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
-	_, err = kv.Get(t.Context(), schedulerLeaderKey)
-	require.Error(t, err)
+	requireLeaseKeyAbsent(t, nc, "a node that stood down left its key")
 }
 
 // TestConvergeInstanceRoles_RewritesProvisionedAccounts pins the pass that
@@ -49,7 +92,7 @@ func TestConvergeInstanceRoles_RewritesProvisionedAccounts(t *testing.T) {
 	require.NoError(t, err)
 
 	iam.RolePolicies[ecsInstanceRoleName] = `{"Version":"2012-10-17","Statement":[]}`
-	sc := NewScheduler(nc, svc, "holder-a")
+	sc := testScheduler(t, nc, svc, "holder-a")
 	require.NoError(t, sc.convergeInstanceRoles(t.Context()))
 
 	assert.Contains(t, iam.RolePolicies[ecsInstanceRoleName], "sts:AssumeRole")
@@ -65,7 +108,7 @@ func TestConvergeInstanceRoles_SkipsAccountsWithoutTheRole(t *testing.T) {
 	_, err := svc.CreateCluster(context.Background(), &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
 	require.NoError(t, err)
 
-	sc := NewScheduler(nc, svc, "holder-a")
+	sc := testScheduler(t, nc, svc, "holder-a")
 	require.NoError(t, sc.convergeInstanceRoles(t.Context()))
 
 	assert.Zero(t, iam.CreateRoleCalls)
@@ -77,7 +120,7 @@ func TestConvergeInstanceRoles_SkipsAccountsWithoutTheRole(t *testing.T) {
 // caller that reaches the pass without going through leader election.
 func TestConvergeInstanceRoles_NilIAMIsAnError(t *testing.T) {
 	_, nc, _ := testutil.StartTestJetStream(t)
-	sc := NewScheduler(nc, NewService(nc, testRegion, ""), "holder-a")
+	sc := testScheduler(t, nc, NewService(nc, testRegion, ""), "holder-a")
 
 	err := sc.convergeInstanceRoles(t.Context())
 	require.Error(t, err)
@@ -94,3 +137,59 @@ func TestECSInstanceRolePolicyDoc_GrantsAssumeRole(t *testing.T) {
 	assert.Contains(t, doc, `"ecs:*"`)
 	assert.Contains(t, doc, `"ecr:GetAuthorizationToken"`)
 }
+
+// containerInstanceStatus is i-1's status in the web cluster.
+func containerInstanceStatus(t *testing.T, svc *Service) string {
+	t.Helper()
+	di, err := svc.DescribeContainerInstances(context.Background(), &ecs.DescribeContainerInstancesInput{
+		Cluster: aws.String("web"), ContainerInstances: []*string{aws.String("i-1")},
+	}, testAccountID)
+	require.NoError(t, err)
+	return aws.StringValue(di.ContainerInstances[0].Status)
+}
+
+// A leader that loses the lease must drop the bus and stop reaping, or two
+// nodes would mark instances DRAINING and stop their tasks at once. Re-elected,
+// it takes both back up.
+func TestScheduler_LosingTheLeaseDropsTheBusAndTheReaperUntilReElection(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	svc := NewService(nc, testRegion, "").WithDeps(Deps{IAM: newStubIAM()})
+	_, err := svc.CreateCluster(context.Background(), &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
+	require.NoError(t, err)
+	registerInstance(t, svc, "web", "i-1", 1024, 2048)
+	kv, err := svc.bucket(t.Context(), testAccountID)
+	require.NoError(t, err)
+	var rec InstanceRecord
+	_, err = taskdefinition.GetJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec)
+	require.NoError(t, err)
+	rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
+	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
+
+	store := kvstore.OpenSyncBucket(testutil.NewJetStream(t, nc))
+	sc := NewScheduler(nc, svc, kvsync.NewLease(store, KeyScheduler, "holder-a", 300*time.Millisecond))
+	ctx, cancel := context.WithCancel(t.Context())
+	done := make(chan struct{})
+	go func() { defer close(done); kvsync.RunElection(ctx, sc.lease, sc.onGained, sc.unsubscribeBus) }()
+	t.Cleanup(func() { cancel(); <-done })
+
+	held := func() bool { return sc.lease.Held() && sc.subCount() == 4 }
+	require.Eventually(t, held, 5*time.Second, 10*time.Millisecond, "the leader subscribes to the four bus subjects")
+
+	syncKV, err := store.KV(t.Context())
+	require.NoError(t, err)
+	_, err = syncKV.Put(t.Context(), KeyScheduler, []byte(`{"holder":"holder-b","generation":0}`))
+	require.NoError(t, err)
+	require.Eventually(t, func() bool { return !sc.lease.Held() && sc.subCount() == 0 }, 5*time.Second, 10*time.Millisecond,
+		"a lost lease must drop every bus subscription")
+
+	revisit, err := sc.timerPass(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, leaseRefresh, revisit)
+	assert.Equal(t, "ACTIVE", containerInstanceStatus(t, svc), "a node that lost the lease must not reap")
+
+	require.NoError(t, syncKV.Delete(t.Context(), KeyScheduler))
+	require.Eventually(t, held, 5*time.Second, 10*time.Millisecond, "re-elected, the bus comes back")
+	_, err = sc.timerPass(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, "DRAINING", containerInstanceStatus(t, svc), "the re-elected leader reaps again")
+}

spinifex/domains/ecs/service_test.go

@@ -14,6 +14,7 @@ import (
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -571,7 +572,7 @@ func TestScheduler_ReapBucket_StopsStaleInstanceTasks(t *testing.T) {
 	rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
 	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
 
-	sc := NewScheduler(nc, svc, "test-holder")
+	sc := testScheduler(t, nc, svc, "test-holder")
 	sc.reapBucket(context.Background(), kv, testAccountID, time.Now().UTC())
 
 	di, err := svc.DescribeContainerInstances(context.Background(), &ecs.DescribeContainerInstancesInput{
@@ -586,32 +587,38 @@ func TestScheduler_ReapBucket_StopsStaleInstanceTasks(t *testing.T) {
 	assert.Equal(t, stoppedReasonReaped, aws.StringValue(dt.Tasks[0].StoppedReason))
 }
 
-// Leader election: first holder wins Create; a second holder is rejected.
+// Leader election: the first holder wins; a second node, and a second attempt
+// in the same process, are both refused while it leads.
 func TestScheduler_AcquireLease_SingleLeader(t *testing.T) {
 	_, nc, _ := testutil.StartTestJetStream(t)
 	svc := NewService(nc, testRegion, "").WithDeps(Deps{IAM: newStubIAM()})
-	a := NewScheduler(nc, svc, "holder-a")
-	b := NewScheduler(nc, svc, "holder-b")
-	require.True(t, a.lease.TryAcquire(t.Context()))
-	assert.False(t, b.lease.TryAcquire(t.Context()))
-	assert.True(t, a.lease.TryAcquire(t.Context()), "a holder re-attempting keeps leadership")
+	a := testScheduler(t, nc, svc, "holder-a")
+	b := testScheduler(t, nc, svc, "holder-b")
+	tok, err := a.lease.TryLock(t.Context())
+	require.NoError(t, err)
+	t.Cleanup(func() { a.lease.Unlock(context.Background(), tok) })
+
+	_, err = b.lease.TryLock(t.Context())
+	assert.ErrorAs(t, err, new(*kvsync.HeldError))
+	_, err = a.lease.TryLock(t.Context())
+	assert.ErrorAs(t, err, new(*kvsync.HeldError), "a second attempt in the leading process is refused")
+	assert.True(t, a.lease.Held(), "a refused second attempt keeps leadership")
 }
 
-// Leadership owns the Layer-2 bus subscriptions. This pins the wiring — gaining
-// the lease subscribes, losing it unsubscribes — which is the regression this
-// conversion could introduce and the one thing kvlease cannot check itself.
+// Leadership owns the Layer-2 bus subscriptions. This pins the wiring: gaining
+// the lease subscribes, losing it unsubscribes.
 func TestScheduler_LeadershipWiresBusSubscriptions(t *testing.T) {
 	_, nc, _ := testutil.StartTestJetStream(t)
 	svc := NewService(nc, testRegion, "").WithDeps(Deps{IAM: newStubIAM()})
-	sc := NewScheduler(nc, svc, "holder-a")
-	require.NoError(t, sc.leaseErr)
+	sc := testScheduler(t, nc, svc, "holder-a")
 
 	require.Zero(t, sc.subCount(), "subscriptions must not exist before election")
 
-	require.True(t, sc.lease.TryAcquire(t.Context()))
+	stop, err := electOnce(t, sc)
+	require.NoError(t, err)
 	assert.Equal(t, 4, sc.subCount(), "leader must own the register, heartbeat, task-state and reconcile-wake subscriptions")
 
-	sc.lease.Release(t.Context())
+	stop()
 	assert.Zero(t, sc.subCount(), "a released leader must drop its subscriptions")
 }
 
@@ -620,20 +627,21 @@ func TestScheduler_LeadershipWiresBusSubscriptions(t *testing.T) {
 func TestScheduler_LoserHoldsNoSubscriptions(t *testing.T) {
 	_, nc, _ := testutil.StartTestJetStream(t)
 	svc := NewService(nc, testRegion, "").WithDeps(Deps{IAM: newStubIAM()})
-	a := NewScheduler(nc, svc, "holder-a")
-	b := NewScheduler(nc, svc, "holder-b")
+	a := testScheduler(t, nc, svc, "holder-a")
+	b := testScheduler(t, nc, svc, "holder-b")
 
-	require.True(t, a.lease.TryAcquire(t.Context()))
-	require.False(t, b.lease.TryAcquire(t.Context()))
+	_, err := electOnce(t, a)
+	require.NoError(t, err)
+	_, err = b.lease.TryLock(t.Context())
+	require.ErrorAs(t, err, new(*kvsync.HeldError))
 	assert.Zero(t, b.subCount())
 	assert.Equal(t, 4, a.subCount())
 }
 
 func TestScheduler_RunReleasesLeaseOnShutdown(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
+	_, nc, _ := testutil.StartTestJetStream(t)
 	svc := NewService(nc, testRegion, "").WithDeps(Deps{IAM: newStubIAM()})
-	sc := NewScheduler(nc, svc, "holder-a")
-	require.NoError(t, sc.leaseErr)
+	sc := testScheduler(t, nc, svc, "holder-a")
 
 	ctx, cancel := context.WithCancel(t.Context())
 	done := make(chan struct{})
@@ -643,10 +651,7 @@ func TestScheduler_RunReleasesLeaseOnShutdown(t *testing.T) {
 	cancel()
 	<-done
 
-	kv, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
-	_, err = kv.Get(t.Context(), schedulerLeaderKey)
-	require.Error(t, err, "Run returned with the lease key still present")
+	requireLeaseKeyAbsent(t, nc, "Run returned with the lease key still present")
 }
 
 // --- GPU placement dimension (Epic C2) ---

spinifex/domains/ecs/store.go

@@ -4,9 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"time"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
@@ -19,18 +17,11 @@ import (
 // cluster, task-definition, task, service, and container-instance state. It is
 // created lazily on first cluster touch (no daemon-boot pre-creation) so accounts
 // without any ECS clusters never grow a bucket (ecs-v1.md Q2).
-//
-// Shared leader bucket "spinifex-ecs-leader" holds 60s-TTL CAS locks keyed by
-// "{accountID}/{clusterName}" for the per-cluster scheduler leader election
-// (Q3). Created once at daemon boot.
+
 const (
 	KVBucketECSAccountPrefix  = "ecs-account-"
 	KVBucketECSAccountVersion = 1
 	KVBucketECSAccountHistory = 1
-
-	KVBucketECSLeader        = "spinifex-ecs-leader"
-	KVBucketECSLeaderVersion = 1
-	KVBucketECSLeaderTTL     = 60 * time.Second
 )
 
 // Key-path helpers for the per-account bucket. The layout matches ecs-v1.md Q2.
@@ -119,15 +110,15 @@ func CapacityProviderKey(name string) string {
 	return CapacityProvidersPrefix() + name
 }
 
-// Store is the per-daemon ECS KV handle. Per-account and leader buckets are
-// accessed via the package-level factories below.
+// Store is the per-daemon ECS KV handle. Per-account buckets are accessed via
+// the package-level factories below.
 type Store struct {
 	nc *nats.Conn
 }
 
 // NewStore constructs a Store bound to the supplied NATS connection. It does not
 // touch JetStream — per-account buckets are created lazily by
-// GetOrCreateAccountBucket and the leader bucket by InitLeaderBucket.
+// GetOrCreateAccountBucket.
 func NewStore(nc *nats.Conn) (*Store, error) {
 	if nc == nil {
 		return nil, errors.New("ecs store: nats connection is nil")
@@ -185,13 +176,3 @@ func GetOrCreateAccountBucket(ctx context.Context, js jetstream.JetStream, accou
 	}
 	return kv, nil
 }
-
-// InitLeaderBucket creates (or attaches to) the shared spinifex-ecs-leader bucket
-// used for per-cluster scheduler leader-lease CAS locks.
-func InitLeaderBucket(ctx context.Context, js jetstream.JetStream) (jetstream.KeyValue, error) {
-	return kvlease.OpenBucket(ctx, js, kvlease.BucketConfig{
-		Name:    KVBucketECSLeader,
-		TTL:     KVBucketECSLeaderTTL,
-		Version: KVBucketECSLeaderVersion,
-	})
-}

spinifex/domains/ecs/store_test.go

@@ -1,13 +1,9 @@
 package ecs
 
 import (
-	"encoding/json"
-	"strings"
 	"testing"
-	"time"
 
 	"github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -46,71 +42,6 @@ func TestGetOrCreateAccountBucket_Idempotent(t *testing.T) {
 	assert.Equal(t, kv1.Bucket(), kv2.Bucket())
 }
 
-func TestInitLeaderBucket_Idempotent(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	js := testutil.NewJetStream(t, nc)
-
-	kv1, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
-	require.NotNil(t, kv1)
-	assert.Equal(t, KVBucketECSLeader, kv1.Bucket())
-
-	kv2, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
-	require.NotNil(t, kv2)
-	assert.Equal(t, KVBucketECSLeader, kv2.Bucket())
-}
-
-// The scheduler calls InitLeaderBucket on every tick. Creating first and falling
-// back on already-exists reaches the same handle, so idempotency alone cannot
-// tell the two orders apart — only the API traffic can.
-//
-// On a multi-node cluster the stored bucket is replicated while this call asks
-// for the default, and JetStream answers the mismatch with an error rather than
-// a no-op: 10058, once per tick, forever. That is invisible from the caller and
-// showed up only as a 16% JetStream API error rate on the meta leader.
-func TestInitLeaderBucket_AttachesWithoutCreating(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
-
-	// Watch from before the first call. The server queues the API advisory after
-	// the reply the caller waits on, so subscribing once the bucket exists can
-	// still catch the create that first call legitimately made.
-	creates := make(chan string, 8)
-	sub, err := nc.Subscribe("$JS.EVENT.ADVISORY.API", func(m *nats.Msg) {
-		var adv struct {
-			Subject string `json:"subject"`
-		}
-		if json.Unmarshal(m.Data, &adv) == nil &&
-			strings.HasPrefix(adv.Subject, "$JS.API.STREAM.CREATE.") {
-			creates <- adv.Subject
-		}
-	})
-	require.NoError(t, err)
-	defer func() { _ = sub.Unsubscribe() }()
-	require.NoError(t, nc.Flush())
-
-	_, err = InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
-
-	// Take the first, legitimate create off the channel before the call under
-	// test, so anything left there afterwards can only have come from it.
-	select {
-	case <-creates:
-	case <-time.After(5 * time.Second):
-		t.Fatal("the first InitLeaderBucket must create the bucket")
-	}
-
-	_, err = InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
-	require.NoError(t, nc.Flush())
-
-	select {
-	case subject := <-creates:
-		t.Fatalf("InitLeaderBucket issued %s on an existing bucket; attach before create", subject)
-	case <-time.After(250 * time.Millisecond):
-	}
-}
-
 // Key-path helpers must produce the ecs-v1.md Q2 layout exactly: prefixes are
 // what the List* enumerations watch, so a drift here silently breaks listing.
 func TestKeyPaths(t *testing.T) {

spinifex/domains/eks/awsapi/gateway_wire_test.go

@@ -14,6 +14,7 @@ import (
 	ekscluster "github.com/mulgadc/spinifex/spinifex/domains/eks/cluster"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -55,7 +56,7 @@ func TestEKSRequest_InternalAddonsWireBody(t *testing.T) {
 		ControlPlaneNodes: []ekscluster.ControlPlaneNode{{InstanceID: cpInstanceID}},
 	}))
 
-	svc, err := ekscluster.NewEKSServiceImpl(ekscluster.EKSServiceDeps{NATSConn: nc, Region: authzRegion})
+	svc, err := ekscluster.NewEKSServiceImpl(ekscluster.EKSServiceDeps{NATSConn: nc, Region: authzRegion, Leases: ekscluster.NewClusterLeases(kvstore.OpenSyncBucket(js), "test")})
 	require.NoError(t, err)
 	sub, err := nc.Subscribe("eks.ListStagedAddonManifests", func(m *nats.Msg) {
 		acct := natsmsg.AccountIDFromMsg(m)
@@ -112,7 +113,7 @@ func TestEKSRequest_InternalWorkersWireBody(t *testing.T) {
 		require.NoError(t, ekscluster.PutNodegroupRecord(t.Context(), kv, rec))
 	}
 
-	svc, err := ekscluster.NewEKSServiceImpl(ekscluster.EKSServiceDeps{NATSConn: nc, Region: authzRegion})
+	svc, err := ekscluster.NewEKSServiceImpl(ekscluster.EKSServiceDeps{NATSConn: nc, Region: authzRegion, Leases: ekscluster.NewClusterLeases(kvstore.OpenSyncBucket(js), "test")})
 	require.NoError(t, err)
 	sub, err := nc.Subscribe("eks.ListManagedWorkers", func(m *nats.Msg) {
 		acct := natsmsg.AccountIDFromMsg(m)

spinifex/domains/eks/cluster/cluster_reconciler.go

@@ -16,8 +16,8 @@ import (
 	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/reconciler"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
 
 	"github.com/nats-io/nats.go"
@@ -25,8 +25,7 @@ import (
 )
 
 const (
-	defaultReconcileLeaseRefresh = 20 * time.Second
-	defaultReconcileInterval     = 30 * time.Second
+	defaultReconcileInterval = 30 * time.Second
 	// reconcileResync is the drift backstop for a loop whose signals now arrive as
 	// watch updates and state reports. It also bounds every deadline below it, so
 	// the 15-minute create timeout is served by the resync rather than by a timer.
@@ -123,6 +122,13 @@ type natsSubscriber interface {
 // ErrReconcilerLeaseLost is returned by Run when the leader lease is lost mid-loop.
 var ErrReconcilerLeaseLost = errors.New("eks: reconciler lease lost")
 
+// closedLost stands in for the Lost channel of a reconciler that never acquired.
+var closedLost = func() <-chan struct{} {
+	ch := make(chan struct{})
+	close(ch)
+	return ch
+}()
+
 // ErrReconcilerClusterDeleting is returned by Run when meta.Status is DELETING;
 // DeleteCluster owns teardown so the reconciler exits.
 var ErrReconcilerClusterDeleting = errors.New("eks: cluster deleting")
@@ -138,15 +144,12 @@ type HTTPDoer interface {
 // ClusterReconciler drives one cluster's CREATING → ACTIVE → DELETING lifecycle
 // under a leader lease. One instance per (accountID, clusterName).
 type ClusterReconciler struct {
-	leaderKV    jetstream.KeyValue
 	acctKV      jetstream.KeyValue
 	accountID   string
 	clusterName string
-	holderID    string
 	healthURL   string
 
-	lease         *kvlease.Lease
-	leaseRefresh  time.Duration
+	lease         *kvsync.Lease
 	interval      time.Duration
 	healthTimeout time.Duration
 	createTimeout time.Duration
@@ -166,6 +169,9 @@ type ClusterReconciler struct {
 	// terminal records why the loop is ending — a terminal cluster status or a
 	// lost lease — and stopRun cancels it. reconciler.Run has no exit condition
 	// of its own, so ending it is cancelling it, and this is what Run reports.
+	// lost is the Lost channel of the acquisition AcquireLease took.
+	lost <-chan struct{}
+
 	terminalMu sync.Mutex
 	terminal   error
 	stopRun    context.CancelFunc
@@ -226,11 +232,6 @@ func WithReconcileInterval(d time.Duration) ReconcilerOption {
 	return func(r *ClusterReconciler) { r.interval = d }
 }
 
-// WithLeaseRefresh overrides the leader-lease refresh period (should be < KVBucketEKSLeaderTTL).
-func WithLeaseRefresh(d time.Duration) ReconcilerOption {
-	return func(r *ClusterReconciler) { r.leaseRefresh = d }
-}
-
 // WithHealthzTimeout overrides the per-probe HTTP timeout.
 func WithHealthzTimeout(d time.Duration) ReconcilerOption {
 	return func(r *ClusterReconciler) { r.healthTimeout = d }
@@ -335,12 +336,12 @@ func WithEtcdResetRecovery(grace, backoff time.Duration, maxAttempts int) Reconc
 	}
 }
 
-// NewClusterReconciler constructs a ClusterReconciler. healthURL is the
-// fully-qualified probe target (e.g. https://nlb-dns:443/healthz) or empty
-// to skip /healthz probing (CREATING transitions on bootstrap state alone).
-func NewClusterReconciler(leaderKV, acctKV jetstream.KeyValue, accountID, clusterName, holderID, healthURL string, opts ...ReconcilerOption) (*ClusterReconciler, error) {
-	if leaderKV == nil {
-		return nil, errors.New("eks: NewClusterReconciler nil leaderKV")
+// NewClusterReconciler constructs a ClusterReconciler that leads while it holds
+// lease, the cluster's lease from ClusterLeases. healthURL is the probe target
+// (e.g. https://nlb-dns:443/healthz) or empty to skip /healthz probing.
+func NewClusterReconciler(lease *kvsync.Lease, acctKV jetstream.KeyValue, accountID, clusterName, healthURL string, opts ...ReconcilerOption) (*ClusterReconciler, error) {
+	if lease == nil {
+		return nil, errors.New("eks: NewClusterReconciler nil lease")
 	}
 	if acctKV == nil {
 		return nil, errors.New("eks: NewClusterReconciler nil acctKV")
@@ -351,17 +352,12 @@ func NewClusterReconciler(leaderKV, acctKV jetstream.KeyValue, accountID, cluste
 	if clusterName == "" {
 		return nil, errors.New("eks: NewClusterReconciler empty clusterName")
 	}
-	if holderID == "" {
-		return nil, errors.New("eks: NewClusterReconciler empty holderID")
-	}
 	r := &ClusterReconciler{
-		leaderKV:        leaderKV,
 		acctKV:          acctKV,
 		accountID:       accountID,
 		clusterName:     clusterName,
-		holderID:        holderID,
 		healthURL:       healthURL,
-		leaseRefresh:    defaultReconcileLeaseRefresh,
+		lease:           lease,
 		interval:        defaultReconcileInterval,
 		healthTimeout:   defaultHealthzTimeout,
 		createTimeout:   defaultCreateTimeout,
@@ -390,42 +386,23 @@ func NewClusterReconciler(leaderKV, acctKV jetstream.KeyValue, accountID, cluste
 	for _, o := range opts {
 		o(r)
 	}
-	lease, err := kvlease.New(kvlease.Config{
-		Name:   "eks/cluster-reconciler",
-		Bucket: kvlease.StaticBucket(leaderKV),
-		Key:    reconcilerLeaderKey(accountID, clusterName),
-		Holder: holderID,
-		Attrs:  []any{"account", accountID, "cluster", clusterName},
-		TTL:    KVBucketEKSLeaderTTL,
-		Renew:  r.leaseRefresh,
-	})
-	if err != nil {
-		return nil, fmt.Errorf("cluster reconciler lease: %w", err)
-	}
-	r.lease = lease
 	return r, nil
 }
 
-func reconcilerLeaderKey(accountID, clusterName string) string {
-	return accountID + "/" + clusterName
-}
-
-// AcquireLease attempts a CAS Create on the leader bucket. Returns
-// (release, true) on success; (nil, false) otherwise. The bucket TTL reaps
-// stale leases when release does not run.
+// AcquireLease takes the cluster's lease. Returns (release, true) on success;
+// (nil, false) when another holder, or another run in this process, has it.
 func (r *ClusterReconciler) AcquireLease(ctx context.Context) (func(), bool) {
-	if !r.lease.TryAcquire(ctx) {
-		return nil, false
+	release, lost, ok := tryLease(ctx, r.lease)
+	if ok {
+		r.lost = lost
 	}
-	return func() { r.lease.Release(ctx) }, true
+	return release, ok
 }
 
 // Run drives reconcile until ctx is cancelled, the lease is lost, or the cluster
-// reaches a terminal state. Caller must AcquireLease first.
+// reaches a terminal state. Caller must AcquireLease first; without it Run
+// returns ErrReconcilerLeaseLost at once.
 func (r *ClusterReconciler) Run(ctx context.Context) error {
-	if r.lease == nil {
-		return errors.New("ClusterReconciler: Run called without AcquireLease")
-	}
 	runCtx, stop := context.WithCancel(ctx)
 	defer stop()
 	r.stopRun = stop
@@ -463,11 +440,16 @@ func (r *ClusterReconciler) Run(ctx context.Context) error {
 	}
 
 	// The lease turning over is not a KV write this loop watches, so losing it has
-	// to end the loop from outside it.
+	// to end the loop from outside it. The channel is the acquisition's own, taken
+	// with its token, so a later acquisition's cannot stand in for it.
+	lost := r.lost
+	if lost == nil {
+		lost = closedLost
+	}
 	go func() {
 		select {
 		case <-runCtx.Done():
-		case <-r.lease.Lost():
+		case <-lost:
 			r.endRun(ErrReconcilerLeaseLost)
 		}
 	}()

spinifex/domains/eks/cluster/cluster_reconciler_state_test.go

@@ -21,14 +21,12 @@ func newStateReconcilerHarness(t *testing.T, opts ...ReconcilerOption) (*Cluster
 	t.Helper()
 	_, nc, _ := testutil.StartTestJetStream(t)
 	js := testutil.NewJetStream(t, nc)
-	leaderKV, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
 	acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
 	require.NoError(t, err)
 	require.NoError(t, PutClusterMeta(t.Context(), acctKV, sampleClusterMeta("alpha")))
 
 	base := []ReconcilerOption{WithStateSource(nc, StateSubject(testAccountID, "alpha"))}
-	r, err := NewClusterReconciler(leaderKV, acctKV, testAccountID, "alpha", "holder-1", "", append(base, opts...)...)
+	r, err := NewClusterReconciler(otherNodeLease(js, "holder-1"), acctKV, testAccountID, "alpha", "", append(base, opts...)...)
 	require.NoError(t, err)
 	return r, nc, acctKV
 }
@@ -146,7 +144,6 @@ func TestClusterReconciler_StateReportSubscriptionDrivesActive(t *testing.T) {
 	t.Parallel()
 	r, nc, acctKV := newStateReconcilerHarness(t,
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 	freshenClusterCreatedAt(t, acctKV)
 	seedBootstrapState(t, acctKV)

spinifex/domains/eks/cluster/cluster_reconciler_test.go

@@ -12,6 +12,8 @@ import (
 	"time"
 
 	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -54,39 +56,47 @@ func onePass(t *testing.T, r *ClusterReconciler) error {
 	return err
 }
 
-func newReconcilerHarness(t *testing.T, healthURL string, opts ...ReconcilerOption) (*ClusterReconciler, jetstream.KeyValue, jetstream.KeyValue) {
+func newReconcilerHarness(t *testing.T, healthURL string, opts ...ReconcilerOption) (*ClusterReconciler, jetstream.JetStream, jetstream.KeyValue) {
+	t.Helper()
+	return newReconcilerHarnessTTL(t, clusterLeaseTTL, healthURL, opts...)
+}
+
+// newReconcilerHarnessTTL is newReconcilerHarness with a lease of the given ttl,
+// for a test that has to see a renewal.
+func newReconcilerHarnessTTL(t *testing.T, ttl time.Duration, healthURL string, opts ...ReconcilerOption) (*ClusterReconciler, jetstream.JetStream, jetstream.KeyValue) {
 	t.Helper()
 	_, nc, _ := testutil.StartTestJetStream(t)
 	js := testutil.NewJetStream(t, nc)
-	leaderKV, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
 	acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
 	require.NoError(t, err)
 	require.NoError(t, PutClusterMeta(t.Context(), acctKV, sampleClusterMeta("alpha")))
 
-	r, err := NewClusterReconciler(leaderKV, acctKV, testAccountID, "alpha", "holder-1", healthURL, opts...)
+	leases := kvsync.NewLeaseGroup(kvstore.OpenSyncBucket(js), KeyPrefixCluster, "holder-1", ttl)
+	r, err := NewClusterReconciler(leases.Get(clusterLeaseName(testAccountID, "alpha")), acctKV, testAccountID, "alpha", healthURL, opts...)
 	require.NoError(t, err)
-	return r, leaderKV, acctKV
+	return r, js, acctKV
+}
+
+// otherNodeLease is the cluster's reconciler lease as another node sees it.
+func otherNodeLease(js jetstream.JetStream, holder string) *kvsync.Lease {
+	return NewClusterLeases(kvstore.OpenSyncBucket(js), holder).Get(clusterLeaseName(testAccountID, "alpha"))
 }
 
 func TestNewClusterReconciler_EmptyInputsRejected(t *testing.T) {
 	t.Parallel()
 	_, nc, _ := testutil.StartTestJetStream(t)
 	js := testutil.NewJetStream(t, nc)
-	leaderKV, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
 	acctKV, err := GetOrCreateAccountBucket(t.Context(), js, testAccountID)
 	require.NoError(t, err)
+	lease := otherNodeLease(js, "h")
 
-	_, err = NewClusterReconciler(nil, acctKV, testAccountID, "alpha", "h", "")
+	_, err = NewClusterReconciler(nil, acctKV, testAccountID, "alpha", "")
 	require.Error(t, err)
-	_, err = NewClusterReconciler(leaderKV, nil, testAccountID, "alpha", "h", "")
+	_, err = NewClusterReconciler(lease, nil, testAccountID, "alpha", "")
 	require.Error(t, err)
-	_, err = NewClusterReconciler(leaderKV, acctKV, "", "alpha", "h", "")
+	_, err = NewClusterReconciler(lease, acctKV, "", "alpha", "")
 	require.Error(t, err)
-	_, err = NewClusterReconciler(leaderKV, acctKV, testAccountID, "", "h", "")
-	require.Error(t, err)
-	_, err = NewClusterReconciler(leaderKV, acctKV, testAccountID, "alpha", "", "")
+	_, err = NewClusterReconciler(lease, acctKV, testAccountID, "", "")
 	require.Error(t, err)
 }
 
@@ -99,21 +109,21 @@ func TestClusterReconciler_AcquireLeaseFirstHolderWins(t *testing.T) {
 	require.NotNil(t, release)
 	defer release()
 
-	// Re-acquiring from the same holder succeeds: the lease is already held.
+	// A second run in the same process is refused while the first holds it.
 	release2, ok2 := r.AcquireLease(t.Context())
-	assert.True(t, ok2)
-	assert.NotNil(t, release2)
+	assert.False(t, ok2)
+	assert.Nil(t, release2)
 }
 
 func TestClusterReconciler_AcquireLeaseSecondHolderLoses(t *testing.T) {
 	t.Parallel()
-	r1, leaderKV, acctKV := newReconcilerHarness(t, "")
+	r1, js, acctKV := newReconcilerHarness(t, "")
 
 	release, ok := r1.AcquireLease(t.Context())
 	require.True(t, ok)
 	defer release()
 
-	r2, err := NewClusterReconciler(leaderKV, acctKV, testAccountID, "alpha", "holder-2", "")
+	r2, err := NewClusterReconciler(otherNodeLease(js, "holder-2"), acctKV, testAccountID, "alpha", "")
 	require.NoError(t, err)
 	release2, ok2 := r2.AcquireLease(t.Context())
 	assert.False(t, ok2)
@@ -154,7 +164,6 @@ func TestClusterReconciler_CreatingTransitionsToActiveOnReadyAndHealthz(t *testi
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 	release, ok := r.AcquireLease(t.Context())
 	require.True(t, ok)
@@ -193,7 +202,6 @@ func TestClusterReconciler_CreatingStaysWhenBootstrapMissing(t *testing.T) {
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 	release, ok := r.AcquireLease(t.Context())
 	require.True(t, ok)
@@ -228,7 +236,6 @@ func TestClusterReconciler_CreatingStaysWhenJWKSUnverified(t *testing.T) {
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 	release, ok := r.AcquireLease(t.Context())
 	require.True(t, ok)
@@ -262,7 +269,6 @@ func TestClusterReconciler_CreatingStaysWhenHealthzFails(t *testing.T) {
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 	release, ok := r.AcquireLease(t.Context())
 	require.True(t, ok)
@@ -287,7 +293,6 @@ func TestClusterReconciler_CreatingTimesOutToFailed(t *testing.T) {
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 		WithCreateTimeout(1*time.Millisecond),
 	)
 	release, ok := r.AcquireLease(t.Context())
@@ -320,7 +325,6 @@ func TestClusterReconciler_CreatingWithinDeadlineStaysCreating(t *testing.T) {
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 		WithCreateTimeout(time.Hour),
 	)
 	release, ok := r.AcquireLease(t.Context())
@@ -345,7 +349,6 @@ func TestClusterReconciler_DeletingExitsLoop(t *testing.T) {
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 	release, ok := r.AcquireLease(t.Context())
 	require.True(t, ok)
@@ -363,11 +366,10 @@ func TestClusterReconciler_DeletingExitsLoop(t *testing.T) {
 func TestClusterReconciler_LostLeaseExitsLoop(t *testing.T) {
 	t.Parallel()
 	stub := &stubHTTPDoer{status: http.StatusOK}
-	r, leaderKV, acctKV := newReconcilerHarness(t,
+	r, js, acctKV := newReconcilerHarnessTTL(t, 300*time.Millisecond,
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Second),
-		WithLeaseRefresh(20*time.Millisecond),
 	)
 	release, ok := r.AcquireLease(t.Context())
 	require.True(t, ok)
@@ -381,16 +383,17 @@ func TestClusterReconciler_LostLeaseExitsLoop(t *testing.T) {
 	runErr := make(chan error, 1)
 	go func() { runErr <- r.Run(ctx) }()
 
-	// Steal the lease in another holder's name. The renewal loses its CAS at the
-	// stale revision, which closes Lost and exits Run.
-	time.Sleep(40 * time.Millisecond)
-	_, err := leaderKV.Put(t.Context(), reconcilerLeaderKey(testAccountID, "alpha"), []byte("holder-2"))
+	// Overwrite the key in another holder's name. The next renewal loses its
+	// CAS at the stale revision, which closes Lost and exits Run.
+	syncKV, err := kvstore.OpenSyncBucket(js).KV(t.Context())
+	require.NoError(t, err)
+	_, err = syncKV.Put(t.Context(), KeyPrefixCluster+clusterLeaseName(testAccountID, "alpha"), []byte(`{"holder":"holder-2","generation":0}`))
 	require.NoError(t, err)
 
 	select {
 	case err := <-runErr:
 		assert.ErrorIs(t, err, ErrReconcilerLeaseLost)
-	case <-time.After(2 * time.Second):
+	case <-time.After(5 * time.Second):
 		t.Fatal("Run did not exit after lease loss")
 	}
 }
@@ -402,7 +405,6 @@ func TestClusterReconciler_ActiveProbesAndWarnsOnHealthzFail(t *testing.T) {
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 	release, ok := r.AcquireLease(t.Context())
 	require.True(t, ok)
@@ -438,7 +440,6 @@ func TestClusterReconciler_ActiveHealthRecoversClearsIssue(t *testing.T) {
 		"https://nlb.example/healthz",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 	release, ok := r.AcquireLease(t.Context())
 	require.True(t, ok)
@@ -468,9 +469,111 @@ func TestClusterReconciler_ProbeHealthzEmptyURLNoop(t *testing.T) {
 		"",
 		WithHTTPClient(stub),
 		WithReconcileInterval(10*time.Millisecond),
-		WithLeaseRefresh(10*time.Second),
 	)
 
 	require.NoError(t, r.probeHealthz(context.Background()))
 	assert.EqualValues(t, 0, stub.calls.Load(), "no HTTP call when URL empty")
 }
+
+// putForeignHolder writes the cluster's lease key in another holder's name and
+// never renews it, standing in for a node that died holding it.
+func putForeignHolder(t *testing.T, js jetstream.JetStream) {
+	t.Helper()
+	syncKV, err := kvstore.OpenSyncBucket(js).KV(t.Context())
+	require.NoError(t, err)
+	_, err = syncKV.Put(t.Context(), KeyPrefixCluster+clusterLeaseName(testAccountID, "alpha"), []byte(`{"holder":"holder-2","generation":0}`))
+	require.NoError(t, err)
+}
+
+func requireStillRunning(t *testing.T, finished <-chan struct{}) {
+	t.Helper()
+	select {
+	case <-finished:
+		t.Fatal("the reconciler goroutine exited while its cluster still wants one")
+	default:
+	}
+}
+
+// A dead node's key is not cleared by the store, so the reconciler has to keep
+// asking until it has sat a TTL; one attempt would leave the cluster unreconciled.
+func TestRunClusterReconciler_TakesOverADeadHoldersKey(t *testing.T) {
+	t.Parallel()
+	r, js, acctKV := newReconcilerHarnessTTL(t, 300*time.Millisecond, "https://nlb.example/healthz",
+		WithHTTPClient(&stubHTTPDoer{status: http.StatusOK}), WithReconcileInterval(10*time.Second))
+	freshenClusterCreatedAt(t, acctKV)
+	seedBootstrapState(t, acctKV)
+	putForeignHolder(t, js)
+
+	release, finished, err := RunClusterReconciler(t.Context(), r.lease, acctKV, testAccountID, "alpha", "https://nlb.example/healthz",
+		WithHTTPClient(&stubHTTPDoer{status: http.StatusOK}), WithReconcileInterval(10*time.Second))
+	require.NoError(t, err, "a held lease is contended for, not an error")
+	defer release()
+
+	require.Eventually(t, r.lease.Held, 5*time.Second, 10*time.Millisecond, "the dead holder's key must be taken over")
+	requireStillRunning(t, finished)
+}
+
+// Losing the lease ends a Run, not the reconciler: it contends again and
+// resumes once it holds the lease once more.
+func TestRunClusterReconciler_ResumesAfterALostLease(t *testing.T) {
+	t.Parallel()
+	r, js, acctKV := newReconcilerHarnessTTL(t, 300*time.Millisecond, "https://nlb.example/healthz")
+	freshenClusterCreatedAt(t, acctKV)
+	seedBootstrapState(t, acctKV)
+	release, finished, err := RunClusterReconciler(t.Context(), r.lease, acctKV, testAccountID, "alpha", "https://nlb.example/healthz",
+		WithHTTPClient(&stubHTTPDoer{status: http.StatusOK}), WithReconcileInterval(10*time.Second))
+	require.NoError(t, err)
+	defer release()
+	require.Eventually(t, r.lease.Held, 5*time.Second, 10*time.Millisecond)
+
+	putForeignHolder(t, js)
+	require.Eventually(t, func() bool { return !r.lease.Held() }, 5*time.Second, 10*time.Millisecond, "the overwrite must cost the lease")
+	require.Eventually(t, r.lease.Held, 5*time.Second, 10*time.Millisecond, "the reconciler must win the lease back")
+	requireStillRunning(t, finished)
+}
+
+// A cluster that has left CREATING and ACTIVE wants no reconciler, so one that
+// cannot get the lease stops asking.
+func TestRunClusterReconciler_StopsContendingForADeletingCluster(t *testing.T) {
+	t.Parallel()
+	r, js, acctKV := newReconcilerHarnessTTL(t, 300*time.Millisecond, "https://nlb.example/healthz")
+	meta, err := GetClusterMeta(t.Context(), acctKV, "alpha")
+	require.NoError(t, err)
+	meta.Status = ClusterStatusDeleting
+	require.NoError(t, PutClusterMeta(t.Context(), acctKV, meta))
+	putForeignHolder(t, js)
+
+	release, finished, err := RunClusterReconciler(t.Context(), r.lease, acctKV, testAccountID, "alpha", "")
+	require.NoError(t, err)
+	defer release()
+	select {
+	case <-finished:
+	case <-time.After(5 * time.Second):
+		t.Fatal("the reconciler kept contending for a DELETING cluster")
+	}
+	assert.False(t, r.lease.Held())
+}
+
+// Run watches the Lost channel of the acquisition AcquireLease took. A later
+// acquisition of the same lease must not keep a Run alive whose own was lost.
+func TestClusterReconciler_RunWatchesItsOwnAcquisition(t *testing.T) {
+	t.Parallel()
+	r, js, acctKV := newReconcilerHarnessTTL(t, 300*time.Millisecond, "https://nlb.example/healthz",
+		WithHTTPClient(&stubHTTPDoer{status: http.StatusOK}), WithReconcileInterval(10*time.Second))
+	_, ok := r.AcquireLease(t.Context())
+	require.True(t, ok)
+
+	putForeignHolder(t, js)
+	require.Eventually(t, func() bool { return !r.lease.Held() }, 5*time.Second, 10*time.Millisecond)
+	syncKV, err := kvstore.OpenSyncBucket(js).KV(t.Context())
+	require.NoError(t, err)
+	require.NoError(t, syncKV.Delete(t.Context(), KeyPrefixCluster+clusterLeaseName(testAccountID, "alpha")))
+	tok, err := r.lease.TryLock(t.Context())
+	require.NoError(t, err, "a later acquisition of the same lease")
+	defer r.lease.Unlock(t.Context(), tok)
+
+	seedBootstrapState(t, acctKV)
+	ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+	defer cancel()
+	assert.ErrorIs(t, r.Run(ctx), ErrReconcilerLeaseLost, "the acquisition this Run was given is gone")
+}

spinifex/domains/eks/cluster/cluster_teardown.go

@@ -7,7 +7,6 @@ import (
 	"log/slog"
 	"time"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	telemetry "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
 	"github.com/nats-io/nats.go/jetstream"
@@ -22,12 +21,6 @@ var teardownConcurrency = 4
 // writer added refs to the record while it was tearing down.
 const maxTeardownPasses = 5
 
-// Teardown lease timing. TTL must match the leader bucket's TTL; vars for tests.
-var (
-	teardownLeaseTTL   = KVBucketEKSLeaderTTL
-	teardownLeaseRenew = defaultReconcileLeaseRefresh
-)
-
 var (
 	errTeardownNotRecorded = errors.New("eks: cluster is not DELETING, teardown refused")
 	errTeardownLeaseLost   = errors.New("eks: teardown lease lost")
@@ -35,45 +28,17 @@ var (
 	errLaunchInFlight      = errors.New("eks: cluster create launch still running")
 )
 
-// teardownLeaderKey is the per-cluster cross-node gate for a DELETING
-// teardown. It is distinct from reconcilerLeaderKey so a delete never contends
-// with the still-live reconciler, which holds that lease until it reads DELETING.
-func teardownLeaderKey(accountID, clusterName string) string {
-	return reconcilerLeaderKey(accountID, clusterName) + "/teardown"
-}
-
 // teardownLease is a held teardown lease: lost closes if renewal loses the key.
 type teardownLease struct {
 	release func()
 	lost    <-chan struct{}
 }
 
-// acquireTeardownLease takes the per-cluster teardown lease, renewing it on
-// ctx. The lease re-adopts a key its own holder already has, so it serialises
-// nodes, not goroutines; startTeardown's in-process gate covers the latter. A
-// nil leaderKV (single-node/test) skips gating.
+// acquireTeardownLease takes the per-cluster teardown lease, renewing it until
+// released. It refuses when another node holds it or another goroutine here does.
 func (s *EKSServiceImpl) acquireTeardownLease(ctx context.Context, accountID, clusterName string) (teardownLease, bool) {
-	if s.leaderKV == nil {
-		return teardownLease{release: func() {}}, true
-	}
-	key := teardownLeaderKey(accountID, clusterName)
-	lease, err := kvlease.New(kvlease.Config{
-		Name:   "eks/teardown",
-		Bucket: kvlease.StaticBucket(s.leaderKV),
-		Key:    key,
-		Holder: s.deps.HolderID,
-		Attrs:  []any{"account", accountID, "cluster", clusterName},
-		TTL:    teardownLeaseTTL,
-		Renew:  teardownLeaseRenew,
-	})
-	if err != nil {
-		slog.ErrorContext(ctx, "eks: teardown lease config invalid", "key", key, "err", err)
-		return teardownLease{}, false
-	}
-	if !lease.TryAcquire(ctx) {
-		return teardownLease{}, false
-	}
-	return teardownLease{release: func() { lease.Release(ctx) }, lost: lease.Lost()}, true
+	release, lost, ok := tryLease(ctx, s.leases.Get(teardownLeaseName(accountID, clusterName)))
+	return teardownLease{release: release, lost: lost}, ok
 }
 
 // teardownPassTimeout bounds one converger run, so a step that hangs cannot
@@ -81,11 +46,11 @@ func (s *EKSServiceImpl) acquireTeardownLease(ctx context.Context, accountID, cl
 var teardownPassTimeout = 15 * time.Minute
 
 // holdTeardown claims the right to tear a cluster down on this node: the
-// in-process gate, which the lease cannot provide because it re-adopts a key
-// its own node holds, then the cross-node lease. It refuses, without waiting,
-// when either is held, or, if capped, when the node is at teardownConcurrency.
-func (s *EKSServiceImpl) holdTeardown(accountID, name string, capped bool) (release func(), lost <-chan struct{}, ok bool) {
-	key := teardownLeaderKey(accountID, name)
+// in-process gate, which also bounds this node's teardowns, then the cross-node
+// lease. It refuses, without waiting, when either is held, or, if capped, when
+// the node is at teardownConcurrency.
+func (s *EKSServiceImpl) holdTeardown(accountID, name string, capped bool) (release func(retire bool), lost <-chan struct{}, ok bool) {
+	key := teardownLeaseName(accountID, name)
 	s.teardownMu.Lock()
 	if s.teardownRunning == nil {
 		s.teardownRunning = map[string]struct{}{}
@@ -110,7 +75,15 @@ func (s *EKSServiceImpl) holdTeardown(accountID, name string, capped bool) (rele
 		leave()
 		return nil, nil, false
 	}
-	return func() { lease.release(); leave() }, lease.lost, true
+	// Retiring drops the teardown lease before the gate opens: every Get of it
+	// is under the gate, so no later teardown of a recreated name can hold it.
+	return func(retire bool) {
+		lease.release()
+		if retire {
+			s.leases.Forget(key)
+		}
+		leave()
+	}, lease.lost, true
 }
 
 // startTeardown hands one DELETING cluster to a background converger and
@@ -122,7 +95,8 @@ func (s *EKSServiceImpl) startTeardown(accountID, name string) bool {
 		return false
 	}
 	s.teardownWG.Go(func() {
-		defer release()
+		deleted := false
+		defer func() { release(deleted) }()
 		ctx, cancel := context.WithTimeout(s.bgCtx, teardownPassTimeout)
 		defer cancel()
 		started := time.Now()
@@ -131,6 +105,7 @@ func (s *EKSServiceImpl) startTeardown(accountID, name string) bool {
 				"cluster", name, "account", accountID, "err", err)
 			return
 		}
+		deleted = true
 		slog.Info("eks: cluster teardown complete", "cluster", name, "account", accountID,
 			"duration_ms", telemetry.Millis(time.Since(started)))
 	})
@@ -211,6 +186,9 @@ func (s *EKSServiceImpl) convergeClusterDeletion(ctx context.Context, accountID,
 			return terr
 		}
 
+		// Forgotten while the record still blocks a same-name create, and after
+		// teardownClusterInfra stopped this node's reconciler.
+		s.leases.Forget(clusterLeaseName(accountID, name))
 		done, err := s.removeClusterRecord(ctx, acctKV, name, meta)
 		if err != nil {
 			if !errors.Is(err, errNodegroupsAttached) && !errors.Is(err, errTeardownNotRecorded) {

spinifex/domains/eks/cluster/cp_orphans.go

@@ -1,158 +0,0 @@
-package cluster
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"sort"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/nats-io/nats.go/jetstream"
-)
-
-// ClusterRecordState is what a read of a cluster's record established.
-type ClusterRecordState int
-
-const (
-	ClusterRecordPresent ClusterRecordState = iota
-	ClusterRecordMissing
-	ClusterRecordUnreadable
-)
-
-// ClusterRecordVerdict is the outcome of looking a cluster record up without
-// creating anything. Reason names a missing or unreadable record; Err is set
-// only when the record could not be read.
-type ClusterRecordVerdict struct {
-	State  ClusterRecordState
-	Reason string
-	Err    error
-}
-
-// JudgeClusterRecord reads the cluster record read-only. A missing bucket or a
-// missing key is Missing, any other failure is Unreadable, and neither is ever
-// repaired. The billable reaper and the orphan report both use it, so they
-// judge a record the same way.
-func JudgeClusterRecord(ctx context.Context, js jetstream.JetStream, account, cluster string) ClusterRecordVerdict {
-	kv, err := OpenAccountBucket(ctx, js, account)
-	switch {
-	case errors.Is(err, jetstream.ErrBucketNotFound):
-		return ClusterRecordVerdict{State: ClusterRecordMissing, Reason: "account bucket missing"}
-	case err != nil:
-		return ClusterRecordVerdict{State: ClusterRecordUnreadable, Reason: "account bucket lookup failed", Err: err}
-	}
-	_, err = GetClusterMeta(ctx, kv, cluster)
-	switch {
-	case err == nil:
-		return ClusterRecordVerdict{State: ClusterRecordPresent}
-	case errors.Is(err, ErrClusterNotFound):
-		return ClusterRecordVerdict{State: ClusterRecordMissing, Reason: "cluster record missing"}
-	}
-	return ClusterRecordVerdict{State: ClusterRecordUnreadable, Reason: "cluster record read failed", Err: err}
-}
-
-// ENITagReader returns the tags on a control-plane ENI. found is false when
-// the ENI is gone.
-type ENITagReader func(ctx context.Context, eniID string) (tagSet map[string]string, found bool, err error)
-
-// ControlPlaneRow is one running EKS control-plane VM the report could not tie
-// to a live cluster record. Unreadable rows carry Err and are never omitted.
-type ControlPlaneRow struct {
-	InstanceID       string
-	Cluster          string
-	Account          string
-	Reason           string
-	Age              time.Duration
-	AvailabilityZone string
-	Unreadable       bool
-	Err              error
-}
-
-// FindControlPlaneOrphans classifies the instances in reservations. An instance
-// with no ENI, or whose ENI is gone or not tagged as an EKS control plane, is
-// skipped. A failed ENI read is an unreadable row, never a silent drop. A
-// control plane whose cluster record is present is omitted; one whose bucket or
-// record is missing is an orphan row; any other read failure is unreadable. It
-// only reads.
-func FindControlPlaneOrphans(ctx context.Context, js jetstream.JetStream, reservations []*ec2.Reservation, eni ENITagReader, now time.Time) []ControlPlaneRow {
-	var rows []ControlPlaneRow
-	for _, res := range reservations {
-		if res == nil {
-			continue
-		}
-		for _, inst := range res.Instances {
-			if inst == nil || !instanceRunning(inst) || len(inst.NetworkInterfaces) == 0 || inst.NetworkInterfaces[0] == nil {
-				continue
-			}
-			eniID := aws.StringValue(inst.NetworkInterfaces[0].NetworkInterfaceId)
-			if eniID == "" {
-				continue
-			}
-			row := ControlPlaneRow{InstanceID: aws.StringValue(inst.InstanceId)}
-			if inst.Placement != nil {
-				row.AvailabilityZone = aws.StringValue(inst.Placement.AvailabilityZone)
-			}
-			if inst.LaunchTime != nil {
-				row.Age = now.Sub(*inst.LaunchTime).Truncate(time.Second)
-			}
-
-			tagSet, found, err := eni(ctx, eniID)
-			if err != nil {
-				row.Unreadable, row.Err, row.Reason = true, err, "control-plane ENI read failed"
-				rows = append(rows, row)
-				continue
-			}
-			if !found || tagSet[tags.ManagedByKey] != tags.ManagedByEKS || tagSet[clusterEKSRoleTagKey] != clusterEKSRoleControlPlane {
-				continue
-			}
-			row.Cluster, row.Account = tagSet[clusterEKSClusterTagKey], tagSet[clusterEKSAccountTagKey]
-			if row.Cluster == "" || row.Account == "" {
-				continue
-			}
-
-			verdict := JudgeClusterRecord(ctx, js, row.Account, row.Cluster)
-			switch verdict.State {
-			case ClusterRecordPresent:
-				continue
-			case ClusterRecordUnreadable:
-				row.Unreadable, row.Err = true, verdict.Err
-			}
-			row.Reason = verdict.Reason
-			rows = append(rows, row)
-		}
-	}
-	sort.Slice(rows, func(i, j int) bool { return rows[i].InstanceID < rows[j].InstanceID })
-	return rows
-}
-
-// ReportControlPlaneOrphans writes the orphan report. list must return every
-// non-terminated system instance or an error: a failed or partial listing is
-// returned as an error and nothing is printed, so a short list is never taken
-// for a complete one. Unreadable rows are printed and counted separately.
-func ReportControlPlaneOrphans(ctx context.Context, w io.Writer, js jetstream.JetStream, list func(context.Context) (*ec2.DescribeInstancesOutput, error), eni ENITagReader, now time.Time) (orphans, unreadable int, err error) {
-	out, err := list(ctx)
-	if err != nil {
-		return 0, 0, fmt.Errorf("listing incomplete: %w", err)
-	}
-	if out == nil {
-		return 0, 0, errors.New("listing incomplete: no result")
-	}
-	rows := FindControlPlaneOrphans(ctx, js, out.Reservations, eni, now)
-	for _, r := range rows {
-		if r.Unreadable {
-			unreadable++
-			fmt.Fprintf(w, "UNREADABLE  %s  cluster=%s account=%s az=%s age=%s  %s: %v\n", r.InstanceID, r.Cluster, r.Account, r.AvailabilityZone, r.Age, r.Reason, r.Err)
-			continue
-		}
-		orphans++
-		fmt.Fprintf(w, "ORPHAN      %s  cluster=%s account=%s az=%s age=%s  %s\n", r.InstanceID, r.Cluster, r.Account, r.AvailabilityZone, r.Age, r.Reason)
-	}
-	return orphans, unreadable, nil
-}
-
-func instanceRunning(inst *ec2.Instance) bool {
-	return inst.State != nil && aws.StringValue(inst.State.Name) == ec2.InstanceStateNameRunning
-}

spinifex/domains/eks/cluster/cp_orphans_test.go

@@ -1,198 +0,0 @@
-package cluster
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"errors"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	testutil "github.com/mulgadc/spinifex/internal/testkit"
-	ec2inventory "github.com/mulgadc/spinifex/spinifex/domains/ec2/inventory"
-	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func orphanInstance(id, eni string) *ec2.Instance {
-	return &ec2.Instance{
-		InstanceId:        aws.String(id),
-		State:             &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)},
-		LaunchTime:        aws.Time(time.Unix(0, 0)),
-		Placement:         &ec2.Placement{AvailabilityZone: aws.String("ap-southeast-2a")},
-		NetworkInterfaces: []*ec2.InstanceNetworkInterface{{NetworkInterfaceId: aws.String(eni)}},
-	}
-}
-
-func cpTags(cluster, account string) map[string]string {
-	return map[string]string{
-		tags.ManagedByKey:       tags.ManagedByEKS,
-		clusterEKSClusterTagKey: cluster,
-		clusterEKSAccountTagKey: account,
-		clusterEKSRoleTagKey:    clusterEKSRoleControlPlane,
-	}
-}
-
-func workerTags(cluster, account string) map[string]string {
-	m := cpTags(cluster, account)
-	m[clusterEKSRoleTagKey] = "worker"
-	return m
-}
-
-func TestFindControlPlaneOrphansUnreadableRecordIsNotAnOrphan(t *testing.T) {
-	f := newEKSServiceFixture(t)
-	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
-	ctx, cancel := context.WithCancel(t.Context())
-	cancel()
-	eni := func(context.Context, string) (map[string]string, bool, error) {
-		return cpTags("gone", testAccountID), true, nil
-	}
-	res := []*ec2.Reservation{{Instances: []*ec2.Instance{orphanInstance("i-1", "eni-1")}}}
-
-	rows := FindControlPlaneOrphans(ctx, js, res, eni, time.Unix(60, 0))
-
-	require.Len(t, rows, 1)
-	assert.True(t, rows[0].Unreadable, "a read that failed is not a missing record")
-	assert.Error(t, rows[0].Err)
-}
-
-func TestFindControlPlaneOrphans(t *testing.T) {
-	const lost = "999999999999"
-	f := newEKSServiceFixture(t)
-	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
-	require.NoError(t, PutClusterMeta(t.Context(), f.kv, sampleClusterMeta("alive")))
-
-	eniTags := map[string]map[string]string{
-		"eni-orphan":   cpTags("gone", testAccountID),
-		"eni-live":     cpTags("alive", testAccountID),
-		"eni-nobucket": cpTags("lost", lost),
-		"eni-worker":   workerTags("gone", testAccountID),
-	}
-	eni := func(_ context.Context, id string) (map[string]string, bool, error) {
-		if id == "eni-broken" {
-			return nil, false, errors.New("vpc service down")
-		}
-		m, ok := eniTags[id]
-		return m, ok, nil
-	}
-	res := []*ec2.Reservation{{Instances: []*ec2.Instance{
-		orphanInstance("i-orphan", "eni-orphan"),
-		orphanInstance("i-live", "eni-live"),
-		orphanInstance("i-nobucket", "eni-nobucket"),
-		orphanInstance("i-worker", "eni-worker"),
-		orphanInstance("i-ghost", "eni-ghost"),
-		orphanInstance("i-broken", "eni-broken"),
-		{InstanceId: aws.String("i-noeni")},
-	}}}
-
-	rows := FindControlPlaneOrphans(t.Context(), js, res, eni, time.Unix(60, 0))
-
-	got := map[string]ControlPlaneRow{}
-	for _, r := range rows {
-		got[r.InstanceID] = r
-	}
-	assert.Len(t, got, 3, "only orphan, nobucket and the unreadable ENI are reported")
-	assert.Equal(t, "cluster record missing", got["i-orphan"].Reason)
-	assert.False(t, got["i-orphan"].Unreadable)
-	assert.Equal(t, "account bucket missing", got["i-nobucket"].Reason)
-	assert.True(t, got["i-broken"].Unreadable, "a failed ENI read is never dropped")
-	assert.Error(t, got["i-broken"].Err)
-	assert.NotContains(t, got, "i-live", "a present record is omitted")
-	assert.NotContains(t, got, "i-worker")
-	assert.NotContains(t, got, "i-ghost")
-	assert.NotContains(t, got, "i-noeni")
-}
-
-func TestReportControlPlaneOrphansIncompleteListingPrintsNothing(t *testing.T) {
-	f := newEKSServiceFixture(t)
-	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
-	var buf bytes.Buffer
-	list := func(context.Context) (*ec2.DescribeInstancesOutput, error) {
-		return nil, errors.New("ServiceUnavailable")
-	}
-	orphans, unreadable, err := ReportControlPlaneOrphans(t.Context(), &buf, js, list, nil, time.Now())
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "listing incomplete")
-	assert.Zero(t, orphans+unreadable)
-	assert.Empty(t, buf.String(), "no partial list on a failed listing")
-}
-
-func TestReportControlPlaneOrphansPrintsRows(t *testing.T) {
-	f := newEKSServiceFixture(t)
-	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
-	var buf bytes.Buffer
-	list := func(context.Context) (*ec2.DescribeInstancesOutput, error) {
-		return &ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{{Instances: []*ec2.Instance{
-			orphanInstance("i-orphan", "eni-orphan"), orphanInstance("i-broken", "eni-broken"),
-		}}}}, nil
-	}
-	eni := func(_ context.Context, id string) (map[string]string, bool, error) {
-		if id == "eni-broken" {
-			return nil, false, errors.New("boom")
-		}
-		return cpTags("gone", testAccountID), true, nil
-	}
-	orphans, unreadable, err := ReportControlPlaneOrphans(t.Context(), &buf, js, list, eni, time.Unix(60, 0))
-	require.NoError(t, err)
-	assert.Equal(t, 1, orphans)
-	assert.Equal(t, 1, unreadable)
-	assert.Contains(t, buf.String(), "ORPHAN      i-orphan")
-	assert.Contains(t, buf.String(), "UNREADABLE  i-broken")
-	assert.Equal(t, 2, strings.Count(buf.String(), "az=ap-southeast-2a"), "both row kinds print the zone")
-}
-
-func TestFindControlPlaneOrphansCorruptENIRecordIsUnreadable(t *testing.T) {
-	f := newEKSServiceFixture(t)
-	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
-	enis, err := kvstore.GetOrCreateBucket(t.Context(), js, ec2vpc.KVBucketENIs, 10)
-	require.NoError(t, err)
-	good, err := json.Marshal(ec2vpc.ENIRecord{NetworkInterfaceId: "eni-good", Tags: cpTags("gone", testAccountID)})
-	require.NoError(t, err)
-	_, err = enis.Put(t.Context(), kvstore.AccountKey("000000000000", "eni-good"), good)
-	require.NoError(t, err)
-	_, err = enis.Put(t.Context(), kvstore.AccountKey("000000000000", "eni-corrupt"), []byte("{not json"))
-	require.NoError(t, err)
-	res := []*ec2.Reservation{{Instances: []*ec2.Instance{
-		orphanInstance("i-good", "eni-good"), orphanInstance("i-corrupt", "eni-corrupt"), orphanInstance("i-absent", "eni-absent"),
-	}}}
-
-	rows := FindControlPlaneOrphans(t.Context(), js, res, ec2inventory.NewENITagReader(js, "000000000000"), time.Unix(60, 0))
-
-	require.Len(t, rows, 2, "an absent ENI is skipped, the other two are reported")
-	assert.Equal(t, "i-corrupt", rows[0].InstanceID)
-	assert.True(t, rows[0].Unreadable, "a corrupt ENI record is unreadable, never gone")
-	assert.Equal(t, "i-good", rows[1].InstanceID)
-	assert.False(t, rows[1].Unreadable)
-}
-
-func TestFindControlPlaneOrphansOnlyReportsRunningInstances(t *testing.T) {
-	f := newEKSServiceFixture(t)
-	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
-	eni := func(context.Context, string) (map[string]string, bool, error) {
-		return cpTags("gone", testAccountID), true, nil
-	}
-	mk := func(id, state string) *ec2.Instance {
-		i := orphanInstance(id, "eni-"+id)
-		i.State = &ec2.InstanceState{Name: aws.String(state)}
-		return i
-	}
-	noState := orphanInstance("i-nostate", "eni-nostate")
-	noState.State = nil
-	res := []*ec2.Reservation{{Instances: []*ec2.Instance{
-		mk("i-running", ec2.InstanceStateNameRunning),
-		mk("i-stopped", ec2.InstanceStateNameStopped),
-		mk("i-pending", ec2.InstanceStateNamePending),
-		noState,
-	}}}
-
-	rows := FindControlPlaneOrphans(t.Context(), js, res, eni, time.Unix(60, 0))
-
-	require.Len(t, rows, 1)
-	assert.Equal(t, "i-running", rows[0].InstanceID)
-}

spinifex/domains/eks/cluster/delete_cluster_async_test.go

@@ -12,6 +12,8 @@ import (
 	"github.com/aws/aws-sdk-go/service/eks"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -132,22 +134,18 @@ func TestS8_EKSSecondDeleteAnswersFromTheRecordWithoutASecondTeardown(t *testing
 	assert.Len(t, f.inst.terminated(), 1, "the repeat request must start no teardown of its own")
 }
 
-// TestS8_EKSTeardownLeaseOutlivesTheRequest: the teardown lease renews on the
-// service's context, so cancelling the request context mid-teardown does not
-// let the lease expire and a second node start a duplicate teardown.
+// TestS8_EKSTeardownLeaseOutlivesTheRequest: the teardown lease renews on its
+// own, so cancelling the request context mid-teardown does not let the lease
+// lapse and a second node start a duplicate teardown.
 func TestS8_EKSTeardownLeaseOutlivesTheRequest(t *testing.T) {
 	f := newDeleteClusterFixture(t, "alpha")
-	_, nc, _ := testutil.StartTestJetStream(t)
-	js := testutil.NewJetStream(t, nc)
-	// A short TTL with frequent renewals keeps the wait brief while leaving
-	// several renewal chances inside each TTL window.
+	// A short TTL keeps the wait brief while leaving several renewal chances
+	// inside each TTL window.
 	const ttl = 400 * time.Millisecond
-	short, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: "eks-teardown-short-ttl", TTL: ttl})
-	require.NoError(t, err)
-	f.svc.leaderKV = short
-	origTTL, origRenew := teardownLeaseTTL, teardownLeaseRenew
-	teardownLeaseTTL, teardownLeaseRenew = ttl, 50*time.Millisecond
-	t.Cleanup(func() { teardownLeaseTTL, teardownLeaseRenew = origTTL, origRenew })
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+	f.svc.leases = kvsync.NewLeaseGroup(kvstore.OpenSyncBucket(js), KeyPrefixCluster, "node-1", ttl)
+	other := kvsync.NewLeaseGroup(kvstore.OpenSyncBucket(js), KeyPrefixCluster, "node-2", ttl).
+		Get(teardownLeaseName(testAccountID, "alpha"))
 
 	entered, release := blockTerminate(t, f.inst)
 	ctx, cancel := context.WithCancel(context.Background())
@@ -155,19 +153,22 @@ func TestS8_EKSTeardownLeaseOutlivesTheRequest(t *testing.T) {
 	waitEntered(t, entered)
 	cancel()
 
+	// The other node first sees the key, then sees it again past a full TTL: it
+	// takes over only if no renewal moved the key in between.
+	_, err := other.TryLock(t.Context())
+	require.ErrorAs(t, err, new(*kvsync.HeldError))
 	time.Sleep(ttl * 5 / 2)
-	_, err = short.Create(t.Context(), teardownLeaderKey(testAccountID, "alpha"), []byte("node-2"))
-	require.ErrorIs(t, err, jetstream.ErrKeyExists,
+	_, err = other.TryLock(t.Context())
+	require.ErrorAs(t, err, new(*kvsync.HeldError),
 		"the running teardown must still hold its lease after the request context is gone")
 
 	release()
 	f.svc.WaitTeardowns()
 }
 
-// TestS8_EKSKickAndSameNodeSweepRunOneTeardown: the teardown lease re-adopts a
-// key its own node holds, so only the in-process gate stops a same-node reaper
-// sweep from starting a second teardown beside the request's. Each member must
-// be terminated exactly once.
+// TestS8_EKSKickAndSameNodeSweepRunOneTeardown: a same-node reaper sweep must
+// not start a second teardown beside the request's. Each member must be
+// terminated exactly once.
 func TestS8_EKSKickAndSameNodeSweepRunOneTeardown(t *testing.T) {
 	t.Parallel()
 	f := newDeleteClusterFixture(t, "alpha")

spinifex/domains/eks/cluster/delete_cluster_review2_test.go

@@ -180,7 +180,7 @@ func TestEKSBootstrapRunRemovesItsOwnEntryOnExit(t *testing.T) {
 	eventually(t, func() bool {
 		f.svc.bootstrapMu.Lock()
 		defer f.svc.bootstrapMu.Unlock()
-		_, ok := f.svc.bootstraps[reconcilerLeaderKey(testAccountID, "alpha")]
+		_, ok := f.svc.bootstraps[clusterLeaseName(testAccountID, "alpha")]
 		return !ok
 	}, "an exited bootstrap must remove its own entry")
 }

spinifex/domains/eks/cluster/delete_cluster_review_test.go

@@ -252,8 +252,7 @@ func TestS2_EKSFailedClusterReclaimRunsUnderTheTeardownLease(t *testing.T) {
 	require.NoError(t, err)
 	meta.Status = ClusterStatusFailed
 	require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta))
-	_, err = f.svc.leaderKV.Create(t.Context(), teardownLeaderKey(testAccountID, "alpha"), []byte("node-2"))
-	require.NoError(t, err)
+	holdElsewhere(t, f.svc, teardownLeaseName(testAccountID, "alpha"))
 
 	_, err = f.svc.CreateCluster(context.Background(), createInput("alpha"), testAccountID, "")
 	code, _, _ := awserrors.ResolveErrorDetail(err)

spinifex/domains/eks/cluster/delete_cluster_test.go

@@ -15,6 +15,8 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/systeminstance"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
@@ -83,6 +85,7 @@ func newEKSServiceFixture(t *testing.T) *eksServiceFixture {
 		GatewayBaseURL:      "https://gw.local:9999",
 		Region:              "us-east-1",
 		HolderID:            "node-1",
+		Leases:              NewClusterLeases(kvstore.OpenSyncBucket(js), "node-1"),
 		SystemGatewayURL:    "https://gw.local:9999",
 		SystemAccessKey:     "AKIAEXAMPLE",
 		SystemSecretKey:     "s3cr3t-key",
@@ -371,6 +374,33 @@ func TestDeleteCluster_EgressEIPReleaseRealErrorLeavesMeta(t *testing.T) {
 	assert.NotEmpty(t, meta.EgressEIPAllocationID, "alloc ID must remain for retry")
 }
 
+// otherNodeLeases is the cluster lease group as another node holding holder
+// sees it, on its own sync bucket over the same JetStream as svc.
+func otherNodeLeases(t *testing.T, svc *EKSServiceImpl, holder string) *kvsync.LeaseGroup {
+	t.Helper()
+	return NewClusterLeases(kvstore.OpenSyncBucket(testutil.NewJetStream(t, svc.deps.NATSConn)), holder)
+}
+
+// holdElsewhere has another node take the named lease until the test ends.
+func holdElsewhere(t *testing.T, svc *EKSServiceImpl, name string) {
+	t.Helper()
+	lease := otherNodeLeases(t, svc, "node-2").Get(name)
+	tok, err := lease.TryLock(t.Context())
+	require.NoError(t, err)
+	t.Cleanup(func() { lease.Unlock(context.Background(), tok) })
+}
+
+// leaseKeyHeld reports whether any node holds the named lease's key.
+func leaseKeyHeld(t *testing.T, svc *EKSServiceImpl, name string) bool {
+	t.Helper()
+	_, _, err := kvstore.OpenSyncBucket(testutil.NewJetStream(t, svc.deps.NATSConn)).Load(t.Context(), KeyPrefixCluster+name)
+	if errors.Is(err, kvsync.ErrNotFound) {
+		return false
+	}
+	require.NoError(t, err)
+	return true
+}
+
 // TestDeleteClusterLeaseHeldElsewhereLeavesTeardownToTheHolder: when another
 // node holds the teardown lease, DeleteCluster still records DELETING and
 // returns it, but runs no teardown of its own and leaves the record for the
@@ -379,8 +409,7 @@ func TestDeleteClusterLeaseHeldElsewhereLeavesTeardownToTheHolder(t *testing.T)
 	t.Parallel()
 	f := newDeleteClusterFixture(t, "alpha")
 
-	_, err := f.svc.leaderKV.Create(t.Context(), teardownLeaderKey(testAccountID, "alpha"), []byte("node-2"))
-	require.NoError(t, err)
+	holdElsewhere(t, f.svc, teardownLeaseName(testAccountID, "alpha"))
 
 	deleteAndSettle(t, f.svc, "alpha")
 
@@ -399,8 +428,73 @@ func TestDeleteClusterTeardownReleasesItsLease(t *testing.T) {
 
 	deleteAndSettle(t, f.svc, "alpha")
 
-	_, getErr := f.svc.leaderKV.Get(t.Context(), teardownLeaderKey(testAccountID, "alpha"))
-	assert.ErrorIs(t, getErr, jetstream.ErrKeyNotFound, "the teardown lease must be released after the teardown")
+	assert.False(t, leaseKeyHeld(t, f.svc, teardownLeaseName(testAccountID, "alpha")),
+		"the teardown lease must be released after the teardown")
+}
+
+// TestDeleteClusterForgetsTheClusterLeases: a deleted cluster's reconciler and
+// teardown leases leave the group, so a cluster recreated under the same name
+// starts from fresh ones.
+func TestDeleteClusterForgetsTheClusterLeases(t *testing.T) {
+	t.Parallel()
+	f := newDeleteClusterFixture(t, "alpha")
+	names := []string{clusterLeaseName(testAccountID, "alpha"), teardownLeaseName(testAccountID, "alpha")}
+	before := make([]*kvsync.Lease, len(names))
+	for i, name := range names {
+		before[i] = f.svc.leases.Get(name)
+	}
+
+	deleteAndSettle(t, f.svc, "alpha")
+
+	_, err := GetClusterMeta(t.Context(), f.kv, "alpha")
+	require.ErrorIs(t, err, ErrClusterNotFound, "the teardown must have completed")
+	for i, name := range names {
+		assert.NotSame(t, before[i], f.svc.leases.Get(name), "%s must be forgotten", name)
+	}
+}
+
+// TestDeleteClusterCleanupLeavesARecreatedClustersLease: once the record is
+// gone a same-name cluster may lead, and the old teardown's cleanup, still
+// running, must not release the lease it took.
+func TestDeleteClusterCleanupLeavesARecreatedClustersLease(t *testing.T) {
+	t.Parallel()
+	f := newDeleteClusterFixture(t, "alpha")
+	entered, release := blockTerminate(t, f.inst)
+	_, err := f.svc.DeleteCluster(context.Background(), deleteInput("alpha"), testAccountID)
+	require.NoError(t, err)
+	waitEntered(t, entered)
+
+	// Holding the gate's mutex parks the old teardown before its gate opens.
+	f.svc.teardownMu.Lock()
+	release()
+	require.Eventually(t, func() bool {
+		_, err := GetClusterMeta(t.Context(), f.kv, "alpha")
+		return errors.Is(err, ErrClusterNotFound)
+	}, 10*time.Second, 10*time.Millisecond, "the old teardown must remove the record")
+
+	replacement := f.svc.leases.Get(clusterLeaseName(testAccountID, "alpha"))
+	tok, err := replacement.TryLock(t.Context())
+	require.NoError(t, err, "the recreated cluster's reconciler takes the free lease")
+	f.svc.teardownMu.Unlock()
+	f.svc.WaitTeardowns()
+
+	assert.True(t, replacement.Held(), "the old teardown's cleanup released the recreated cluster's lease")
+	replacement.Unlock(t.Context(), tok)
+}
+
+// TestDeleteClusterKeepsTheLeasesOfAnIncompleteTeardown: a teardown that
+// fails leaves the record DELETING for retry, so its leases stay in the group.
+func TestDeleteClusterKeepsTheLeasesOfAnIncompleteTeardown(t *testing.T) {
+	t.Parallel()
+	f := newDeleteClusterFixture(t, "alpha")
+	f.eip.releaseErr = errors.New("AddressLimitExceeded")
+	name := teardownLeaseName(testAccountID, "alpha")
+	before := f.svc.leases.Get(name)
+
+	deleteAndSettle(t, f.svc, "alpha")
+
+	requireTeardownRetained(t, f.kv, "alpha")
+	assert.Same(t, before, f.svc.leases.Get(name))
 }
 
 func TestTeardownLeaseReleaseSurvivesCallerCancellation(t *testing.T) {
@@ -413,8 +507,7 @@ func TestTeardownLeaseReleaseSurvivesCallerCancellation(t *testing.T) {
 	cancel()
 	lease.release()
 
-	_, err := f.svc.leaderKV.Get(t.Context(), teardownLeaderKey(testAccountID, "alpha"))
-	require.ErrorIs(t, err, jetstream.ErrKeyNotFound)
+	assert.False(t, leaseKeyHeld(t, f.svc, teardownLeaseName(testAccountID, "alpha")))
 }
 
 // TestDeleteCluster_ManagedCPVPCReclaimsCustomerVPCSGs guards the contract: in

spinifex/domains/eks/cluster/eks_billable_reaper.go

@@ -2,6 +2,7 @@ package cluster
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"log/slog"
 
@@ -12,10 +13,13 @@ import (
 	"github.com/nats-io/nats.go/jetstream"
 )
 
-// EKSBillableReaper raises the data-safety alarm for a running EKS control-plane
-// VM whose cluster record cannot be found. It never terminates anything: a
-// missing record cannot tell a completed DeleteCluster from a lost bucket or an
-// out-of-band delete, so removal is an operator decision. It is node-local.
+// EKSBillableReaper is the meta-independent billable cleanup from historic PROP-LIFECYCLE-006 §5 (behaviour per ADR-0003:S2), built on
+// the resource lifecycle GC backstop. A normal DeleteCluster can leave a
+// running control-plane VM behind (terminate completes async; a daemon restart
+// re-launches it from saved running-state) after the cluster meta is already
+// swept — a billable orphan with no owner to drive a retry. This reaper finds a
+// running EKS control-plane VM whose cluster meta is DEFINITIVELY GONE and
+// terminates it. It is node-local: each node reaps the orphans it runs.
 type EKSBillableReaper struct {
 	svc *EKSServiceImpl
 	// running returns this node's running VMs. The reaper filters to the EKS
@@ -34,11 +38,10 @@ func (s *EKSServiceImpl) NewBillableReaper(running func() ([]*vm.VM, error)) *EK
 func (r *EKSBillableReaper) Class() string         { return "eks-billable" }
 func (r *EKSBillableReaper) Scope() vm.ReaperScope { return vm.ScopeNodeLocal }
 
-// Sweep alarms on every running EKS control-plane VM whose cluster record, or
-// whole account bucket, is missing, and removes nothing. A VM whose cluster
-// exists is left to the cluster's own teardown; an ENI that is gone or untagged
-// is skipped, and a read that fails is logged and skipped. It returns 0 because
-// it never removes a VM.
+// Sweep terminates every running EKS control-plane VM whose cluster meta is
+// absent. A VM whose cluster still exists (CREATING or DELETING) is left to the
+// cluster's own teardown/retry; an ENI that is gone or untagged is skipped, so
+// the reaper never reaps on uncertainty.
 func (r *EKSBillableReaper) Sweep(ctx context.Context) (int, error) {
 	vms, err := r.running()
 	if err != nil {
@@ -50,10 +53,11 @@ func (r *EKSBillableReaper) Sweep(ctx context.Context) (int, error) {
 		return 0, fmt.Errorf("jetstream: %w", err)
 	}
 
+	reaped := 0
 	for _, v := range vms {
 		select {
 		case <-ctx.Done():
-			return 0, ctx.Err()
+			return reaped, ctx.Err()
 		default:
 		}
 		if v == nil || v.ManagedBy != tags.ManagedByEKS || v.ENIId == "" {
@@ -62,23 +66,47 @@ func (r *EKSBillableReaper) Sweep(ctx context.Context) (int, error) {
 
 		clusterName, clusterAccount, ok := r.clusterRefFromENI(ctx, v)
 		if !ok {
-			continue // ENI gone or untagged: cannot tie the VM to a cluster
+			continue // ENI gone or untagged: cannot confirm orphan-hood, never reap
 		}
 
-		verdict := JudgeClusterRecord(ctx, js, clusterAccount, clusterName)
-		switch verdict.State {
-		case ClusterRecordPresent:
-			continue // cluster exists (live, CREATING or DELETING)
-		case ClusterRecordUnreadable:
-			slog.WarnContext(ctx, "eks-billable: "+verdict.Reason, "cluster", clusterName, "account", clusterAccount, "err", verdict.Err)
+		acctKV, err := GetOrCreateAccountBucket(ctx, js, clusterAccount)
+		if err != nil {
+			slog.WarnContext(ctx, "eks-billable: account bucket lookup failed", "account", clusterAccount, "err", err)
+			continue
+		}
+		if _, err := GetClusterMeta(ctx, acctKV, clusterName); !errors.Is(err, ErrClusterNotFound) {
+			// Meta present (live/CREATING/DELETING) or unreadable: not a definitive
+			// orphan — leave it to the cluster's own teardown/retry.
 			continue
 		}
-		reason := verdict.Reason
 
-		slog.WarnContext(ctx, "DATA-SAFETY ALARM: EKS control-plane VM has no cluster record; retained for an operator",
-			"reason", reason, "instanceId", v.ID, "cluster", clusterName, "account", clusterAccount, "node", v.LastNode)
+		// ALARM + reap: the cluster is gone but its control-plane VM still runs.
+		slog.WarnContext(ctx, "DATA-SAFETY ALARM: orphaned EKS control-plane VM reaped (cluster meta gone)",
+			"instanceId", v.ID, "cluster", clusterName, "account", clusterAccount, "node", v.LastNode)
+		if err := TerminateK3sServerVM(ctx, r.svc.deps.VPCK3s, r.svc.deps.Instance, v.AccountID, v.ID, v.ENIId); err != nil {
+			slog.WarnContext(ctx, "eks-billable: failed to terminate orphan CP VM", "instanceId", v.ID, "err", err)
+			continue
+		}
+		reaped++
+		r.reclaimOrphanInfra(ctx, v.AccountID, clusterName)
+	}
+	return reaped, nil
+}
+
+// reclaimOrphanInfra tears down the billable infra a reaped orphan's swept meta
+// no longer anchors: the NLB front-end NAT and the managed CP VPC's NAT-GW EIP.
+// Both are keyed by cluster name and idempotent, so a clean prior delete is a
+// no-op. Best-effort — the costliest resource (the VM) is already gone, and
+// DeleteClusterCPVPC releases the NAT-GW EIP before the VPC delete, so the
+// billable address is reclaimed even if the now-empty VPC delete trails. account
+// is the infra account (system account for the managed CP VPC topology).
+func (r *EKSBillableReaper) reclaimOrphanInfra(ctx context.Context, account, clusterName string) {
+	if err := DeleteClusterNLB(ctx, r.svc.deps.NLB, account, clusterName); err != nil {
+		slog.WarnContext(ctx, "eks-billable: reclaim orphan NLB failed", "cluster", clusterName, "err", err)
+	}
+	if err := DeleteClusterCPVPC(ctx, r.svc.cpVPCDeps(), account, clusterName, nil); err != nil {
+		slog.WarnContext(ctx, "eks-billable: reclaim orphan CP VPC (NAT-GW EIP) failed", "cluster", clusterName, "err", err)
 	}
-	return 0, nil
 }
 
 // clusterRefFromENI reads the cluster name + customer account from the VM's

spinifex/domains/eks/cluster/eks_billable_reaper_test.go

@@ -1,19 +1,14 @@
 package cluster
 
 import (
-	"bytes"
 	"context"
-	"log/slog"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/elbv2"
-	testutil "github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
-	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -38,119 +33,53 @@ func cpVM(id, eniID string) *vm.VM {
 	return &vm.VM{ID: id, ManagedBy: tags.ManagedByEKS, ENIId: eniID, AccountID: reaperSysAccount, LastNode: "node-1"}
 }
 
-// ADR-0003:S7. A running control-plane VM whose cluster record cannot be found
-// is retained and alarmed, never terminated and never has its NLB or VPC torn
-// down: a missing record cannot tell a completed DeleteCluster from a lost
-// bucket or an out-of-band delete.
-func TestS7_EKSBillableReaperNeverTerminatesAControlPlane(t *testing.T) {
-	const lostAccount = "999999999999"
-
-	cases := []struct {
-		name      string
-		account   string
-		prepare   func(t *testing.T, f *eksServiceFixture)
-		wantAlarm string
-		wantLog   string
-	}{
-		{
-			name:      "account bucket missing",
-			account:   lostAccount,
-			prepare:   func(*testing.T, *eksServiceFixture) {},
-			wantAlarm: "account bucket missing",
-		},
-		{
-			name:      "bucket has no meta key",
-			account:   testAccountID,
-			prepare:   func(*testing.T, *eksServiceFixture) {},
-			wantAlarm: "cluster record missing",
-		},
-		{
-			name:    "meta removed by the teardown revision-checked delete",
-			account: testAccountID,
-			prepare: func(t *testing.T, f *eksServiceFixture) {
-				require.NoError(t, PutClusterMeta(t.Context(), f.kv, sampleClusterMeta("orphan")))
-				_, err := kvstore.DeleteIf(t.Context(), f.kv, ClusterMetaKey("orphan"), kvstore.CASConfig{
-					Attempts: maxClusterStateCASRetries,
-					NotFound: ErrClusterNotFound,
-				}, func(*ClusterMeta) error { return nil })
-				require.NoError(t, err)
-			},
-			wantAlarm: "cluster record missing",
-		},
-		{
-			name:    "meta removed out of band",
-			account: testAccountID,
-			prepare: func(t *testing.T, f *eksServiceFixture) {
-				require.NoError(t, PutClusterMeta(t.Context(), f.kv, sampleClusterMeta("orphan")))
-				require.NoError(t, f.kv.Delete(t.Context(), ClusterMetaKey("orphan")))
-			},
-			wantAlarm: "cluster record missing",
-		},
-		{
-			name:    "meta unreadable",
-			account: testAccountID,
-			prepare: func(t *testing.T, f *eksServiceFixture) {
-				_, err := f.kv.Put(t.Context(), ClusterMetaKey("orphan"), []byte("{not json"))
-				require.NoError(t, err)
-			},
-			wantLog: "cluster record read failed",
-		},
+// TestRLC5_EKSBillableReaperTerminatesOrphanCPVM exercises the historic PROP-LIFECYCLE-006 §5 (behaviour per ADR-0003:S2)
+// meta-independent billable cleanup: a running EKS control-plane VM whose
+// cluster meta is DEFINITIVELY GONE is a billable orphan and must be terminated
+// by the GC backstop — the real fix for the orphan CP VM surviving a
+// daemon restart after DeleteCluster swept the meta.
+func TestRLC5_EKSBillableReaperTerminatesOrphanCPVM(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	f.vpc.describeByENI = map[string]*ec2.NetworkInterface{
+		"eni-orphan": cpENI("eni-orphan", "gone-cluster", testAccountID),
 	}
+	orphan := cpVM("i-orphan", "eni-orphan")
 
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			// slog.SetDefault is process-wide, so this test is not parallel.
-			var buf bytes.Buffer
-			restore := slog.Default()
-			slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo})))
-			t.Cleanup(func() { slog.SetDefault(restore) })
-
-			f := newEKSServiceFixture(t)
-			f.vpc.describeByENI = map[string]*ec2.NetworkInterface{
-				"eni-cp": cpENI("eni-cp", "orphan", tc.account),
-			}
-			nlbName := ClusterNLBName("orphan")
-			f.nlb.lbByName[nlbName] = &elbv2.LoadBalancer{
-				LoadBalancerArn:  aws.String("arn:lb-orphan"),
-				LoadBalancerName: aws.String(nlbName),
-			}
-			f.ngw.gws = []*fakeCPNatGateway{{
-				id:    "nat-orphan",
-				tags:  map[string]string{clusterEKSClusterTagKey: "orphan", clusterEKSRoleTagKey: cpVPCRoles.NatGW},
-				state: "available",
-				addrs: []*ec2.NatGatewayAddress{{AllocationId: aws.String("eipalloc-orphan")}},
-			}}
-			tc.prepare(t, f)
-
-			reaper := f.svc.NewBillableReaper(func() ([]*vm.VM, error) { return []*vm.VM{cpVM("i-cp", "eni-cp")}, nil })
-			for range 2 {
-				removed, err := reaper.Sweep(context.Background())
-				require.NoError(t, err)
-				assert.Zero(t, removed, "ADR-0003:S7: the reaper removes nothing")
-			}
-
-			assert.Empty(t, f.inst.terminateCalls, "ADR-0003:S7: a reaper's judgement is never the decision to terminate")
-			assert.Empty(t, f.vpc.deleteCalls, "the control-plane ENI must stay")
-			assert.Contains(t, f.nlb.lbByName, nlbName, "the NLB must stay")
-			assert.Empty(t, f.eip.releaseCalls, "no EIP may be released")
-
-			js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
-			if tc.account == lostAccount {
-				_, err := OpenAccountBucket(t.Context(), js, lostAccount)
-				assert.ErrorIs(t, err, jetstream.ErrBucketNotFound, "a sweep must not create the bucket it found missing")
-			}
-
-			out := buf.String()
-			if tc.wantAlarm != "" {
-				assert.Contains(t, out, "DATA-SAFETY ALARM")
-				assert.Contains(t, out, tc.wantAlarm)
-				assert.Contains(t, out, "i-cp")
-			} else {
-				assert.NotContains(t, out, "DATA-SAFETY ALARM", "an unreadable record is not a missing one")
-				assert.Contains(t, out, tc.wantLog)
-			}
-		})
+	// Seed the orphan cluster's name/tag-driven billable infra the swept meta no
+	// longer anchors: an NLB front-end and a tagged NAT gateway holding a billable
+	// EIP. The reaper's post-terminate reclaim must sweep both.
+	nlbName := ClusterNLBName("gone-cluster")
+	f.nlb.lbByName[nlbName] = &elbv2.LoadBalancer{
+		LoadBalancerArn:  aws.String("arn:lb-orphan"),
+		LoadBalancerName: aws.String(nlbName),
 	}
+	f.ngw.gws = []*fakeCPNatGateway{{
+		id:    "nat-orphan",
+		tags:  map[string]string{clusterEKSClusterTagKey: "gone-cluster", clusterEKSRoleTagKey: cpVPCRoles.NatGW},
+		state: "available",
+		addrs: []*ec2.NatGatewayAddress{{AllocationId: aws.String("eipalloc-orphan")}},
+	}}
+
+	reaper := f.svc.NewBillableReaper(func() ([]*vm.VM, error) { return []*vm.VM{orphan}, nil })
+
+	reaped, err := reaper.Sweep(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, 1, reaped, "ADR-0003:S2 behaviour; historic PROP-LIFECYCLE-006 §5 mechanism: a CP VM whose cluster meta is gone must be reaped")
+	assert.Contains(t, f.inst.terminateCalls, "i-orphan", "the orphan CP VM must be terminated")
+	require.Len(t, f.vpc.deleteCalls, 1, "the orphan CP ENI must be deleted")
+	assert.Equal(t, "eni-orphan", aws.StringValue(f.vpc.deleteCalls[0].NetworkInterfaceId))
+
+	// The orphan's billable NLB + NAT-GW EIP must also be reclaimed.
+	assert.NotContains(t, f.nlb.lbByName, nlbName, "the orphan NLB front-end must be reclaimed")
+	require.Len(t, f.eip.releaseCalls, 1, "the orphan NAT-GW EIP must be released")
+	assert.Equal(t, "eipalloc-orphan", aws.StringValue(f.eip.releaseCalls[0].AllocationId))
+
+	// Idempotent: a repeat sweep over the same (still meta-absent) VM is safe —
+	// the terminate + reclaim paths tolerate already-gone infra (no second release).
+	_, err = reaper.Sweep(context.Background())
+	require.NoError(t, err)
+	assert.Len(t, f.eip.releaseCalls, 1, "reclaim must not double-release an already-freed EIP")
 }
 
 // TestRLC5_EKSBillableReaperSpareLiveAndUncertain enforces the reaper's no-false-

spinifex/domains/eks/cluster/eks_deleting_reaper_test.go

@@ -62,8 +62,7 @@ func TestDeletingReaperLeavesAHeldTeardownAlone(t *testing.T) {
 	t.Parallel()
 	f := newDeleteClusterFixture(t, "beta")
 	markDeleting(t, f, "beta", time.Second)
-	_, err := f.svc.leaderKV.Create(t.Context(), teardownLeaderKey(testAccountID, "beta"), []byte("node-2"))
-	require.NoError(t, err)
+	holdElsewhere(t, f.svc, teardownLeaseName(testAccountID, "beta"))
 
 	assert.Equal(t, 0, sweepAndSettle(t, f))
 

spinifex/domains/eks/cluster/eks_nodegroup_deleting_reaper.go

@@ -106,18 +106,22 @@ func (r *EKSNodegroupDeletingReaper) due(rec *NodegroupRecord) bool {
 // startNodegroupDelete runs one converger pass in the background and reports
 // whether it started; a pass already running here or on another node refuses.
 func (s *EKSServiceImpl) startNodegroupDelete(accountID string, kv jetstream.KeyValue, cluster, ng string) bool {
-	release, _, ok := s.holdTeardown(accountID, cluster+"/nodegroup/"+ng, false)
+	gate := cluster + "/nodegroup/" + ng
+	release, _, ok := s.holdTeardown(accountID, gate, false)
 	if !ok {
 		return false
 	}
 	s.teardownWG.Go(func() {
-		defer release()
+		deleted := false
+		defer func() { release(deleted) }()
 		ctx, cancel := context.WithTimeout(s.bgCtx, teardownPassTimeout)
 		defer cancel()
 		if err := s.convergeNodegroupDeletion(ctx, accountID, kv, cluster, ng); err != nil {
 			slog.Warn("eks: nodegroup delete incomplete, the record stays DELETING for retry",
 				"cluster", cluster, "nodegroup", ng, "account", accountID, "err", err)
+			return
 		}
+		deleted = true
 	})
 	return true
 }

spinifex/domains/eks/cluster/leases.go

@@ -0,0 +1,48 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+)
+
+// KeyPrefixCluster is the prefix of every per-cluster lease in the sync bucket.
+const KeyPrefixCluster = "eks.cluster."
+
+// clusterLeaseTTL is every per-cluster lease's time to live.
+const clusterLeaseTTL = 60 * time.Second
+
+// NewClusterLeases returns the per-cluster lease group for holder. The EKS
+// owner builds it once per process: a cluster's reconciler and teardown leases
+// are members of it, and a deleted cluster's are forgotten.
+func NewClusterLeases(store kvsync.KVStore, holder string) *kvsync.LeaseGroup {
+	return kvsync.NewLeaseGroup(store, KeyPrefixCluster, holder, clusterLeaseTTL)
+}
+
+// tryLease makes one attempt at lease, logging any failure but a held lease,
+// and returns its release, which survives a cancelled ctx, and its Lost.
+func tryLease(ctx context.Context, lease *kvsync.Lease) (func(), <-chan struct{}, bool) {
+	tok, err := lease.TryLock(ctx)
+	if err != nil {
+		if _, held := errors.AsType[*kvsync.HeldError](err); !held {
+			slog.WarnContext(ctx, "eks: lease attempt failed", "key", lease.Key(), "err", err)
+		}
+		return nil, nil, false
+	}
+	return func() { lease.Unlock(ctx, tok) }, lease.Lost(), true
+}
+
+// clusterLeaseName names a cluster's reconciler lease in the group.
+func clusterLeaseName(accountID, clusterName string) string {
+	return accountID + "/" + clusterName
+}
+
+// teardownLeaseName names the per-cluster cross-node gate for a DELETING
+// teardown. It is distinct from clusterLeaseName so a delete never contends
+// with the still-live reconciler, which holds that lease until it reads DELETING.
+func teardownLeaseName(accountID, clusterName string) string {
+	return clusterLeaseName(accountID, clusterName) + "/teardown"
+}

spinifex/domains/eks/cluster/nodegroup_delete_test.go

@@ -662,3 +662,23 @@ func TestDeleteNodegroup_WorkerStuckPastDeadlineEndsDeleteFailedKeepingProfile(t
 	requireErrorCode(t, err, awserrors.ErrorEKSResourceNotFound)
 	assert.NotContains(t, fixtureIAM(t, f).Profiles, rec.InstanceProfileName)
 }
+
+// A finished nodegroup delete drops its teardown lease from the group, so the
+// group does not grow with every nodegroup ever deleted and a recreated
+// nodegroup of the same name starts from a fresh lease.
+func TestNodegroupDelete_RetiresItsTeardownLease(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	seedActiveClusterWithToken(t, f, "c1")
+	createReadyNodegroup(t, f, "c1", "ng1", 1)
+	name := teardownLeaseName(testAccountID, "c1/nodegroup/ng1")
+	before := f.svc.leases.Get(name)
+
+	_, err := deleteNodegroupCall(f, "c1", "ng1")
+	require.NoError(t, err)
+	sweepNodegroups(t, f)
+
+	_, err = GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
+	require.ErrorIs(t, err, ErrNodegroupNotFound, "the delete must have completed")
+	assert.NotSame(t, before, f.svc.leases.Get(name), "the finished delete's teardown lease must leave the group")
+}

spinifex/domains/eks/cluster/reconciler_registry.go

@@ -5,14 +5,12 @@ import (
 	"errors"
 	"log/slog"
 	"sync"
+	"time"
 
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
-// ErrLeaseHeld signals the leader lease is held by another node.
-// Spawn treats it as benign and retries on the next tick.
-var ErrLeaseHeld = errors.New("eks: reconciler lease held by another node")
-
 // ReconcilerRegistry tracks active per-cluster reconciler goroutines; one entry per (accountID, clusterName).
 type ReconcilerRegistry struct {
 	mu      sync.Mutex
@@ -64,9 +62,6 @@ func (r *ReconcilerRegistry) Spawn(parent context.Context, accountID, clusterNam
 		r.mu.Unlock()
 		cancel()
 		close(done)
-		if errors.Is(err, ErrLeaseHeld) {
-			return nil
-		}
 		return err
 	}
 
@@ -151,33 +146,67 @@ func registryKey(accountID, clusterName string) string {
 	return accountID + "/" + clusterName
 }
 
-// RunClusterReconciler is the production SpawnReconcilerFn: constructs a ClusterReconciler,
-// acquires the lease, and drives Run in a goroutine. Returns ErrLeaseHeld if another node wins.
+// RunClusterReconciler contends for the cluster's lease and drives Run while it
+// holds it, in a goroutine, until ctx ends or the cluster leaves CREATING and
+// ACTIVE. A held lease is asked for again every ttl/3, so a dead holder's key is
+// taken over once it has sat a TTL, and a lost one is contended for again.
+// release stops the goroutine and returns once the lease is unlocked.
 func RunClusterReconciler(
 	ctx context.Context,
-	leaderKV, acctKV jetstream.KeyValue,
-	accountID, clusterName, holderID, healthURL string,
+	lease *kvsync.Lease, acctKV jetstream.KeyValue,
+	accountID, clusterName, healthURL string,
 	opts ...ReconcilerOption,
 ) (func(), <-chan struct{}, error) {
-	r, err := NewClusterReconciler(leaderKV, acctKV, accountID, clusterName, holderID, healthURL, opts...)
+	newRun := func() (*ClusterReconciler, error) {
+		return NewClusterReconciler(lease, acctKV, accountID, clusterName, healthURL, opts...)
+	}
+	r, err := newRun()
 	if err != nil {
 		return nil, nil, err
 	}
-	release, ok := r.AcquireLease(ctx)
-	if !ok {
-		slog.Info("RunClusterReconciler: lease held elsewhere, skipping spawn",
-			"accountID", accountID, "cluster", clusterName)
-		return nil, nil, ErrLeaseHeld
-	}
-	// Closed when Run returns so the registry can drop this entry on a self-exit
-	// (terminal state / lost lease), letting a later same-name create re-spawn.
+	runCtx, cancel := context.WithCancel(ctx)
 	finished := make(chan struct{})
 	go func() {
 		defer close(finished)
-		if runErr := r.Run(ctx); runErr != nil && !errors.Is(runErr, context.Canceled) {
-			slog.Info("RunClusterReconciler: Run exited",
-				"accountID", accountID, "cluster", clusterName, "err", runErr)
+		for {
+			if release, ok := r.AcquireLease(runCtx); ok {
+				err := r.Run(runCtx)
+				release()
+				if !errors.Is(err, ErrReconcilerLeaseLost) {
+					if err != nil && !errors.Is(err, context.Canceled) {
+						slog.Info("RunClusterReconciler: Run exited",
+							"accountID", accountID, "cluster", clusterName, "err", err)
+					}
+					return
+				}
+				slog.Info("RunClusterReconciler: lease lost, contending again",
+					"accountID", accountID, "cluster", clusterName)
+			}
+			if !clusterContended(runCtx, acctKV, clusterName) {
+				return
+			}
+			select {
+			case <-runCtx.Done():
+				return
+			case <-time.After(lease.TTL() / 3):
+			}
+			// A fresh reconciler per holding: a finished Run keeps its exit reason.
+			r, _ = newRun()
 		}
 	}()
-	return release, finished, nil
+	return func() { cancel(); <-finished }, finished, nil
+}
+
+// clusterContended reports whether the cluster is still one a reconciler is
+// wanted for. A read that fails is not evidence it is gone, so it keeps trying.
+func clusterContended(ctx context.Context, acctKV jetstream.KeyValue, clusterName string) bool {
+	meta, err := GetClusterMeta(ctx, acctKV, clusterName)
+	switch {
+	case errors.Is(err, ErrClusterNotFound):
+		return false
+	case err != nil:
+		return ctx.Err() == nil
+	default:
+		return meta.Status == ClusterStatusCreating || meta.Status == ClusterStatusActive
+	}
 }

spinifex/domains/eks/cluster/reconciler_registry_test.go

@@ -63,34 +63,6 @@ func TestReconcilerRegistry_SpawnFnErrorRemovesEntry(t *testing.T) {
 	assert.False(t, reg.Has("111122223333", "alpha"), "entry must not linger after spawn failure")
 }
 
-func TestReconcilerRegistry_LeaseHeldDropsEntryAndRetries(t *testing.T) {
-	t.Parallel()
-	reg := NewReconcilerRegistry()
-	var spawnCalls atomic.Int32
-	leaseHeld := atomic.Bool{}
-	leaseHeld.Store(true)
-	fn := func(ctx context.Context, _, _ string) (func(), <-chan struct{}, error) {
-		spawnCalls.Add(1)
-		if leaseHeld.Load() {
-			return nil, nil, ErrLeaseHeld
-		}
-		go func() { <-ctx.Done() }()
-		return func() {}, nil, nil
-	}
-
-	// Lease held elsewhere: Spawn is benign (nil), records no phantom holder.
-	require.NoError(t, reg.Spawn(t.Context(), "111122223333", "alpha", fn))
-	assert.False(t, reg.Has("111122223333", "alpha"), "lease-held must not leave a holder")
-
-	// Holder's TTL lapses; a later Spawn re-invokes fn and now takes over —
-	// no daemon restart needed.
-	leaseHeld.Store(false)
-	require.NoError(t, reg.Spawn(t.Context(), "111122223333", "alpha", fn))
-	assert.True(t, reg.Has("111122223333", "alpha"))
-	assert.EqualValues(t, 2, spawnCalls.Load(), "second Spawn must re-attempt after lease-held")
-	reg.StopAll()
-}
-
 // TestReconcilerRegistry_SelfExitDropsEntryAndRespawns covers a reconciler that
 // exits on its own — a terminal cluster state or a lost lease — with no Stop()
 // to cancel its ctx. The registry must drop the entry (and release the lease) so

spinifex/domains/eks/cluster/service_impl.go

@@ -34,6 +34,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/paging"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/idempotency"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -55,6 +56,9 @@ type EKSServiceDeps struct {
 	Region         string
 	HolderID       string
 
+	// Leases is the per-cluster lease group, from NewClusterLeases.
+	Leases *kvsync.LeaseGroup
+
 	// InternalSuffix is the AWS-parity internal DNS suffix (e.g. spinifex.internal)
 	// used to compose the worker's ECR registry host.
 	InternalSuffix string
@@ -262,7 +266,7 @@ type csiVolumeReclaimer interface {
 // EKSServiceImpl is the daemon-side EKSService implementation.
 type EKSServiceImpl struct {
 	deps     EKSServiceDeps
-	leaderKV jetstream.KeyValue
+	leases   *kvsync.LeaseGroup
 	registry *ReconcilerRegistry
 	owner    *access.Owner
 
@@ -340,7 +344,7 @@ const (
 	defaultLaunchStaleAfter     = 90 * time.Second
 )
 
-// NewEKSServiceImpl initialises EKSServiceImpl, wiring the leader KV and reconciler registry.
+// NewEKSServiceImpl initialises EKSServiceImpl, wiring the lease group and reconciler registry.
 func NewEKSServiceImpl(deps EKSServiceDeps) (*EKSServiceImpl, error) {
 	if err := addon.ValidateCatalog(); err != nil {
 		return nil, fmt.Errorf("eks: validate add-on catalog: %w", err)
@@ -348,21 +352,13 @@ func NewEKSServiceImpl(deps EKSServiceDeps) (*EKSServiceImpl, error) {
 	if deps.NATSConn == nil {
 		return nil, errors.New("eks: NewEKSServiceImpl nil NATSConn")
 	}
-	js, err := jetstream.New(deps.NATSConn)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
+	if deps.Leases == nil {
+		return nil, errors.New("eks: NewEKSServiceImpl nil Leases")
 	}
-	// The leader bucket outlives every request, so its open is bounded by the
-	// service's own background context rather than a caller's.
 	ctx, cancel := context.WithCancel(context.Background())
-	leaderKV, err := InitLeaderBucket(ctx, js)
-	if err != nil {
-		cancel()
-		return nil, err
-	}
 	return &EKSServiceImpl{
 		deps:                     deps,
-		leaderKV:                 leaderKV,
+		leases:                   deps.Leases,
 		registry:                 NewReconcilerRegistry(),
 		owner:                    access.New(deps.Region),
 		bgCtx:                    ctx,
@@ -1384,7 +1380,7 @@ func (s *EKSServiceImpl) claimClusterName(ctx context.Context, accountID string,
 		if !held {
 			return errors.New(awserrors.ErrorEKSResourceInUse)
 		}
-		defer release()
+		defer release(false)
 	}
 	var oldRefs *ClusterMeta
 	reclaimed := false
@@ -2368,7 +2364,7 @@ func (s *EKSServiceImpl) spawnBootstrap(accountID, clusterName string, kv jetstr
 		return
 	}
 	ctx, cancel := context.WithCancel(s.bgCtx)
-	key := reconcilerLeaderKey(accountID, clusterName)
+	key := clusterLeaseName(accountID, clusterName)
 	entry := &bootstrapRun{cancel: cancel}
 	s.bootstrapMu.Lock()
 	if s.bootstraps == nil {
@@ -2413,7 +2409,7 @@ type bootstrapRun struct {
 // stopBootstrap cancels this process's bootstrap subscriber for a cluster, so
 // a cluster being torn down stops collecting artifacts into its prefix.
 func (s *EKSServiceImpl) stopBootstrap(accountID, clusterName string) {
-	key := reconcilerLeaderKey(accountID, clusterName)
+	key := clusterLeaseName(accountID, clusterName)
 	s.bootstrapMu.Lock()
 	run := s.bootstraps[key]
 	delete(s.bootstraps, key)
@@ -2454,7 +2450,7 @@ func (s *EKSServiceImpl) spawnReconciler(accountID, clusterName string, _ *Clust
 		opts = append(opts, WithEtcdResetRecovery(0, 0, 0))
 	}
 	spawn := func(ctx context.Context, _, _ string) (func(), <-chan struct{}, error) {
-		return RunClusterReconciler(ctx, s.leaderKV, acctKV, accountID, clusterName, s.deps.HolderID, "", opts...)
+		return RunClusterReconciler(ctx, s.leases.Get(clusterLeaseName(accountID, clusterName)), acctKV, accountID, clusterName, "", opts...)
 	}
 	if err := s.registry.Spawn(s.bgCtx, accountID, clusterName, spawn); err != nil {
 		slog.Error("spawnReconciler: registry spawn", "cluster", clusterName, "err", err)

spinifex/domains/eks/cluster/service_impl_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/domains/eks/access"
 	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
@@ -20,12 +21,20 @@ import (
 
 func setupTestService(t *testing.T) *EKSServiceImpl {
 	t.Helper()
-	_, nc, _ := testutil.StartTestJetStream(t)
-	svc, err := NewEKSServiceImpl(EKSServiceDeps{NATSConn: nc})
+	_, nc, js := testutil.StartTestJetStream(t)
+	svc, err := NewEKSServiceImpl(EKSServiceDeps{NATSConn: nc, Leases: NewClusterLeases(kvstore.OpenSyncBucket(js), "test")})
 	require.NoError(t, err)
 	return svc
 }
 
+// A service without the daemon's lease group would contend on keys no other
+// owner sees, so construction refuses it.
+func TestNewEKSServiceImpl_RequiresLeases(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	_, err := NewEKSServiceImpl(EKSServiceDeps{NATSConn: nc})
+	require.ErrorContains(t, err, "nil Leases")
+}
+
 // TestEKSServiceImpl_ClusterLifecycleShimMode covers the four lifecycle
 // methods when the service is constructed with only NATS wiring (the path the
 // daemon-handler routing test uses): orchestration deps

spinifex/domains/eks/cluster/store.go

@@ -5,9 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"strings"
-	"time"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
@@ -19,10 +17,6 @@ import (
 // Per-account bucket "eks-account-{accountID}" holds all customer-visible
 // cluster state. It is created lazily on first cluster touch (no daemon-boot
 // pre-creation) so accounts without any EKS clusters never grow a bucket.
-//
-// Shared leader bucket "spinifex-eks-leader" holds 60s-TTL CAS locks
-// keyed by "{accountID}/{clusterName}" for the per-cluster reconciler
-// leader election. Created once at daemon boot.
 const (
 	KVBucketEKSAccountPrefix  = "eks-account-"
 	KVBucketEKSAccountVersion = 1
@@ -33,10 +27,6 @@ const (
 	// survives a DeleteCluster. Keep this decoupled from the schema version so a
 	// future version bump can't silently widen history and resurrect ciphertext.
 	KVBucketEKSAccountHistory = 1
-
-	KVBucketEKSLeader        = "spinifex-eks-leader"
-	KVBucketEKSLeaderVersion = 1
-	KVBucketEKSLeaderTTL     = 60 * time.Second
 )
 
 // Key-path helpers for the per-account bucket. The layout matches the
@@ -123,15 +113,15 @@ func RecoveryDirectiveKey(cluster, instanceID string) string {
 	return RecoveryPrefix(cluster) + instanceID
 }
 
-// Store is the per-daemon EKS KV handle. Per-account and leader buckets are
-// accessed via the package-level factories below.
+// Store is the per-daemon EKS KV handle. Per-account buckets are accessed via
+// the package-level factories below.
 type Store struct {
 	nc *nats.Conn
 }
 
 // NewStore constructs a Store bound to the supplied NATS connection. It does
 // not touch JetStream — per-account buckets are created lazily by
-// GetOrCreateAccountBucket and the leader bucket by InitLeaderBucket.
+// GetOrCreateAccountBucket.
 func NewStore(nc *nats.Conn) (*Store, error) {
 	if nc == nil {
 		return nil, errors.New("eks store: nats connection is nil")
@@ -184,8 +174,7 @@ func AccountWatchBuckets(ctx context.Context, nc *nats.Conn) ([]*kvstore.Bucket,
 	}
 	buckets := make([]*kvstore.Bucket, 0, len(names))
 	for _, name := range names {
-		// Attach only: a watch on a deleted bucket must fail and retry, not recreate it.
-		buckets = append(buckets, kvstore.NewBucket(js, kvstore.Config{Name: name, History: 1, AttachOnly: true}))
+		buckets = append(buckets, kvstore.NewBucket(js, kvstore.Config{Name: name, History: 1}))
 	}
 	return buckets, nil
 }
@@ -205,20 +194,3 @@ func GetOrCreateAccountBucket(ctx context.Context, js jetstream.JetStream, accou
 	}
 	return kv, nil
 }
-
-// OpenAccountBucket returns the per-account KV bucket for accountID without
-// creating it. A bucket that does not exist is jetstream.ErrBucketNotFound, so a
-// read-only caller never turns a lost bucket into an empty one.
-func OpenAccountBucket(ctx context.Context, js jetstream.JetStream, accountID string) (jetstream.KeyValue, error) {
-	return js.KeyValue(ctx, AccountBucketName(accountID))
-}
-
-// InitLeaderBucket creates (or attaches to) the shared spinifex-eks-leader
-// bucket used for per-cluster reconciler leader-lease CAS locks.
-func InitLeaderBucket(ctx context.Context, js jetstream.JetStream) (jetstream.KeyValue, error) {
-	return kvlease.OpenBucket(ctx, js, kvlease.BucketConfig{
-		Name:    KVBucketEKSLeader,
-		TTL:     KVBucketEKSLeaderTTL,
-		Version: KVBucketEKSLeaderVersion,
-	})
-}

spinifex/domains/eks/cluster/store_test.go

@@ -49,40 +49,6 @@ func TestGetOrCreateAccountBucket_ReplicasClamped(t *testing.T) {
 	assert.Equal(t, 1, si.Config.Replicas)
 }
 
-func TestInitLeaderBucket_Idempotent(t *testing.T) {
-	t.Parallel()
-	_, nc, _ := testutil.StartTestJetStream(t)
-	js := testutil.NewJetStream(t, nc)
-
-	kv1, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
-	require.NotNil(t, kv1)
-	assert.Equal(t, KVBucketEKSLeader, kv1.Bucket())
-
-	kv2, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
-	require.NotNil(t, kv2)
-	assert.Equal(t, KVBucketEKSLeader, kv2.Bucket())
-}
-
-// TestInitLeaderBucket_ReplicasClamped confirms the leader bucket is created
-// with the requested replica count, clamped to a minimum of 1.
-func TestInitLeaderBucket_ReplicasClamped(t *testing.T) {
-	t.Parallel()
-	_, nc, _ := testutil.StartTestJetStream(t)
-	js := testutil.NewJetStream(t, nc)
-
-	kv, err := InitLeaderBucket(t.Context(), js)
-	require.NoError(t, err)
-	require.NotNil(t, kv)
-
-	stream, err := js.Stream(t.Context(), "KV_"+KVBucketEKSLeader)
-	require.NoError(t, err)
-	si, err := stream.Info(t.Context())
-	require.NoError(t, err)
-	assert.Equal(t, 1, si.Config.Replicas)
-}
-
 func TestKeyPaths_MatchQ2Spec(t *testing.T) {
 	cases := []struct {
 		name string

spinifex/domains/eks/cluster/watch_source_test.go

@@ -7,7 +7,6 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit"
 	ekscluster "github.com/mulgadc/spinifex/spinifex/domains/eks/cluster"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -80,26 +79,3 @@ func TestAccountWatchBuckets_EnumerationErrorPropagates(t *testing.T) {
 	_, err := ekscluster.AccountWatchBuckets(ctx, nc)
 	require.Error(t, err)
 }
-
-// TestAccountWatchBuckets_WatchOnDeletedBucketDoesNotRecreateIt pins attach-only
-// handles: a DNS watch that outlives its bucket must fail and retry, because
-// recreating it hides a deleted account bucket behind an empty one.
-func TestAccountWatchBuckets_WatchOnDeletedBucketDoesNotRecreateIt(t *testing.T) {
-	t.Parallel()
-	_, nc, _ := testutil.StartTestJetStream(t)
-	js := testutil.NewJetStream(t, nc)
-	_, err := ekscluster.GetOrCreateAccountBucket(t.Context(), js, "111111111111")
-	require.NoError(t, err)
-
-	buckets, err := ekscluster.AccountWatchBuckets(t.Context(), nc)
-	require.NoError(t, err)
-	require.Len(t, buckets, 1)
-
-	require.NoError(t, js.DeleteKeyValue(t.Context(), ekscluster.AccountBucketName("111111111111")))
-
-	_, err = buckets[0].Watch(t.Context(), ">")
-	require.Error(t, err, "a watch on a deleted bucket must fail")
-
-	_, err = ekscluster.OpenAccountBucket(t.Context(), js, "111111111111")
-	assert.ErrorIs(t, err, jetstream.ErrBucketNotFound, "the watch must not have recreated the bucket")
-}

spinifex/domains/elbv2/agent_heartbeat_wire_test.go

@@ -1,73 +0,0 @@
-package elbv2
-
-import (
-	"testing"
-
-	"github.com/mulgadc/spinifex/spinifex/ingress/aws/query"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestLBAgentHeartbeatInput_OldAgentDecodesUnchanged(t *testing.T) {
-	t.Parallel()
-	// Exactly the parameters an agent without the data-path probe sends.
-	params := map[string]string{
-		"Action":                   "LBAgentHeartbeat",
-		"Version":                  "2015-12-01",
-		"LBID":                     "lb-abc123",
-		"Servers.member.1.Backend": "bk_tg1",
-		"Servers.member.1.Server":  "srv_i-web1",
-		"Servers.member.1.Status":  "UP",
-		"Servers.member.2.Backend": "bk_tg1",
-		"Servers.member.2.Server":  "srv_i-web2",
-		"Servers.member.2.Status":  "DOWN",
-	}
-
-	var in LBAgentHeartbeatInput
-	require.NoError(t, query.QueryParamsToStruct(params, &in))
-
-	require.NotNil(t, in.LBID)
-	assert.Equal(t, "lb-abc123", *in.LBID)
-	require.Len(t, in.Servers, 2)
-	assert.Equal(t, "DOWN", *in.Servers[1].Status)
-	assert.Nil(t, in.InstanceID, "an old agent reports no instance ID")
-	assert.Empty(t, in.NICs, "an old agent reports no NICs")
-	assert.Len(t, in.toHealthReport().Servers, 2)
-}
-
-func TestLBAgentHeartbeatInput_DecodesDataPathEvidence(t *testing.T) {
-	t.Parallel()
-	params := map[string]string{
-		"LBID":                              "lb-abc123",
-		"InstanceID":                        "i-0123456789abcdef0",
-		"NICs.member.1.MAC":                 "02:00:00:aa:00:01",
-		"NICs.member.1.ProbeOK":             "false",
-		"NICs.member.1.ConsecutiveFailures": "4",
-		"NICs.member.1.RxPackets":           "100",
-		"NICs.member.1.TxPackets":           "200",
-		"NICs.member.1.RxDropped":           "3",
-		"NICs.member.1.TxDropped":           "9",
-		"NICs.member.2.MAC":                 "02:00:00:aa:00:03",
-		"NICs.member.2.ProbeOK":             "true",
-		"NICs.member.2.ConsecutiveFailures": "0",
-	}
-
-	var in LBAgentHeartbeatInput
-	require.NoError(t, query.QueryParamsToStruct(params, &in))
-
-	require.NotNil(t, in.InstanceID)
-	assert.Equal(t, "i-0123456789abcdef0", *in.InstanceID)
-	require.Len(t, in.NICs, 2)
-
-	n := in.NICs[0]
-	assert.Equal(t, "02:00:00:aa:00:01", *n.MAC)
-	assert.False(t, *n.ProbeOK)
-	assert.EqualValues(t, 4, *n.ConsecutiveFailures)
-	assert.EqualValues(t, 100, *n.RxPackets)
-	assert.EqualValues(t, 200, *n.TxPackets)
-	assert.EqualValues(t, 3, *n.RxDropped)
-	assert.EqualValues(t, 9, *n.TxDropped)
-
-	assert.True(t, *in.NICs[1].ProbeOK)
-	assert.Nil(t, in.NICs[1].RxPackets, "an omitted counter stays unset")
-}

spinifex/domains/elbv2/agent_types.go

@@ -7,25 +7,9 @@ import (
 
 // LBAgentHeartbeatInput is sent by the LB agent on each heartbeat tick,
 // including its health report so the daemon can update target health.
-//
-// InstanceID and NICs are optional: an older agent sends neither.
 type LBAgentHeartbeatInput struct {
-	LBID       *string                `locationName:"LBID" type:"string"`
-	Servers    []*LBAgentServerStatus `locationName:"Servers" type:"list"`
-	InstanceID *string                `locationName:"InstanceID" type:"string"`
-	NICs       []*LBAgentNICStatus    `locationName:"NICs" type:"list"`
-}
-
-// LBAgentNICStatus is the gateway probe result and interface counters of one
-// data NIC, as reported by the LB agent.
-type LBAgentNICStatus struct {
-	MAC                 *string `locationName:"MAC" type:"string"`
-	ProbeOK             *bool   `locationName:"ProbeOK" type:"boolean"`
-	ConsecutiveFailures *int64  `locationName:"ConsecutiveFailures" type:"long"`
-	RxPackets           *int64  `locationName:"RxPackets" type:"long"`
-	TxPackets           *int64  `locationName:"TxPackets" type:"long"`
-	RxDropped           *int64  `locationName:"RxDropped" type:"long"`
-	TxDropped           *int64  `locationName:"TxDropped" type:"long"`
+	LBID    *string                `locationName:"LBID" type:"string"`
+	Servers []*LBAgentServerStatus `locationName:"Servers" type:"list"`
 }
 
 // LBAgentServerStatus represents a single backend server's health status as reported by the LB agent.

spinifex/domains/elbv2/launcher_test.go

@@ -14,9 +14,6 @@ type mockSystemInstanceLauncher struct {
 	launchErr      error
 	terminateErr   error
 	terminateDone  chan struct{} // closed after TerminateSystemInstance completes
-	onTerminate    func(instanceID string)
-	rebootCalls    []string
-	rebootErr      error
 }
 
 func (m *mockSystemInstanceLauncher) LaunchSystemInstance(input *SystemInstanceInput) (*SystemInstanceOutput, error) {
@@ -28,22 +25,12 @@ func (m *mockSystemInstanceLauncher) TerminateSystemInstance(instanceID string)
 	m.mu.Lock()
 	m.terminateCalls = append(m.terminateCalls, instanceID)
 	m.mu.Unlock()
-	if m.onTerminate != nil {
-		m.onTerminate(instanceID)
-	}
 	if m.terminateDone != nil {
 		close(m.terminateDone)
 	}
 	return m.terminateErr
 }
 
-func (m *mockSystemInstanceLauncher) RebootSystemInstance(instanceID string) error {
-	m.mu.Lock()
-	m.rebootCalls = append(m.rebootCalls, instanceID)
-	m.mu.Unlock()
-	return m.rebootErr
-}
-
 // waitTerminate blocks until TerminateSystemInstance has been called.
 func (m *mockSystemInstanceLauncher) waitTerminate() {
 	if m.terminateDone != nil {

spinifex/domains/elbv2/lb_converge_test.go

@@ -1,192 +0,0 @@
-package elbv2
-
-import (
-	"context"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestLBRepair_AnotherReservationWinsTheFirstCAS(t *testing.T) {
-	t.Parallel()
-	e := newRepairEnv(t)
-	lb := e.seed("lbr20", true)
-	cands := e.candidates()
-	require.Len(t, cands, 1)
-	require.True(t, e.rec.eligible(context.Background(), cands[0], e.clock))
-
-	_, err := e.svc.store.UpdateLBHealth(context.Background(), lb.LoadBalancerID, func(rec *LBDataPathRecord) (bool, error) {
-		rec.Repairs = append(rec.Repairs, LBRepair{At: e.clock, InstanceID: lb.InstanceID, Outcome: RepairReserved})
-		return true, nil
-	})
-	require.NoError(t, err)
-
-	assert.False(t, e.rec.repair(context.Background(), cands[0]))
-	assert.Empty(t, e.reboots())
-	assert.Len(t, e.health(lb.LoadBalancerID).Repairs, 1, "the losing reservation must not add an entry")
-}
-
-func TestLBRepair_LeaseLostAfterReservationKeepsItCountedAndDoesNotDispatch(t *testing.T) {
-	t.Parallel()
-	e := newRepairEnv(t)
-	e.seed("lbr21", true)
-	cands := e.candidates()
-	require.Len(t, cands, 1)
-
-	calls := 0
-	e.rec.isLeader = func() bool {
-		calls++
-		return calls == 1
-	}
-	assert.True(t, e.rec.repair(context.Background(), cands[0]), "a written reservation is reported as spent")
-	assert.Empty(t, e.reboots())
-
-	repairs := e.health("lbr21").Repairs
-	require.Len(t, repairs, 1)
-	assert.Equal(t, RepairReserved, repairs[0].Outcome, "the entry stays counted with no outcome")
-}
-
-func TestLBRepair_NonLeaderNeverReserves(t *testing.T) {
-	t.Parallel()
-	e := newRepairEnv(t)
-	e.seed("lbr22", true)
-	cands := e.candidates()
-	require.Len(t, cands, 1)
-
-	e.rec.isLeader = func() bool { return false }
-	assert.False(t, e.rec.repair(context.Background(), cands[0]))
-	assert.Empty(t, e.health("lbr22").Repairs)
-}
-
-func TestLBHealth_FailedResumeThenNewVMHeartbeatsConverge(t *testing.T) {
-	t.Parallel()
-	svc := setupTestService(t)
-	lb := seedHealthLB(t, svc)
-	ctx := context.Background()
-
-	// The old VM's evidence was suspended by SetSubnets, whose resume write then failed.
-	_, err := svc.store.UpdateLBHealth(ctx, lb.LoadBalancerID, func(rec *LBDataPathRecord) (bool, error) {
-		rec.InstanceID = "i-oldvm"
-		rec.Suspended = true
-		rec.SuspendedAt = time.Now().UTC()
-		return true, nil
-	})
-	require.NoError(t, err)
-
-	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, true, 0, 10))
-	rec, err := svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
-	require.NoError(t, err)
-	assert.False(t, rec.Suspended, "a heartbeat from the VM the LB names lifts the stale suspension")
-	assert.Equal(t, healthInst, rec.InstanceID)
-
-	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, 3, 11))
-	rec, err = svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
-	require.NoError(t, err)
-	require.Contains(t, rec.NICs, healthMACData, "a later NIC failure is recorded and repairable")
-}
-
-func TestLBHealth_SuspensionIsNotLiftedByAHeartbeatFromAVMTheLBDoesNotName(t *testing.T) {
-	t.Parallel()
-	svc := setupTestService(t)
-	lb := seedHealthLB(t, svc)
-	ctx := context.Background()
-
-	_, err := svc.store.UpdateLBHealth(ctx, lb.LoadBalancerID, func(rec *LBDataPathRecord) (bool, error) {
-		rec.InstanceID = "i-oldvm"
-		rec.Suspended = true
-		rec.SuspendedAt = time.Now().UTC()
-		return true, nil
-	})
-	require.NoError(t, err)
-
-	healthBeat(t, svc, aws.String("i-oldvm"), nicStatus(healthMACData, false, 3, 1))
-	rec, err := svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
-	require.NoError(t, err)
-	assert.True(t, rec.Suspended)
-	assert.Equal(t, "i-oldvm", rec.InstanceID)
-}
-
-func TestLBHealth_SuspensionBackstopLiftsOnlyAfterThirtyMinutes(t *testing.T) {
-	t.Parallel()
-	svc := setupTestService(t)
-	lb := seedHealthLB(t, svc)
-	ctx := context.Background()
-
-	suspend := func(age time.Duration) {
-		_, err := svc.store.UpdateLBHealth(ctx, lb.LoadBalancerID, func(rec *LBDataPathRecord) (bool, error) {
-			rec.InstanceID = healthInst
-			rec.Suspended = true
-			rec.SuspendedAt = time.Now().UTC().Add(-age)
-			return true, nil
-		})
-		require.NoError(t, err)
-	}
-
-	suspend(29 * time.Minute)
-	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, 3, 1))
-	rec, err := svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
-	require.NoError(t, err)
-	assert.True(t, rec.Suspended, "inside the backstop the suspension holds")
-
-	suspend(31 * time.Minute)
-	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, 3, 2))
-	rec, err = svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
-	require.NoError(t, err)
-	assert.False(t, rec.Suspended, "past the backstop the current VM's heartbeat lifts it")
-	assert.Contains(t, rec.NICs, healthMACData)
-}
-
-func TestLBHealth_EstablishedFailureSurvivesAnAgentRestart(t *testing.T) {
-	t.Parallel()
-	svc := setupTestService(t)
-	lb := seedHealthLB(t, svc)
-	ctx := context.Background()
-
-	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, 3, 1))
-	first, err := svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
-	require.NoError(t, err)
-	since := first.NICs[healthMACData].FailingSince
-
-	// A restarted agent reports healthy until it has missed three probes again.
-	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, true, 1, 2))
-	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, true, 2, 3))
-	rec, err := svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
-	require.NoError(t, err)
-	require.Contains(t, rec.NICs, healthMACData)
-	assert.True(t, since.Equal(rec.NICs[healthMACData].FailingSince), "FailingSince is kept")
-
-	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, true, 0, 4))
-	rec, err = svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
-	require.NoError(t, err)
-	assert.Empty(t, rec.NICs, "only a clean probe clears the failure")
-}
-
-func TestLBHealth_OldVMHeartbeatPausedAcrossPublicationLeavesTheNewBinding(t *testing.T) {
-	t.Parallel()
-	svc := setupTestService(t)
-	lb := seedHealthLB(t, svc)
-	ctx := context.Background()
-
-	// SetSubnets has already published the new VM and rebound the evidence.
-	lb.InstanceID = "i-newvm"
-	require.NoError(t, svc.store.PutLoadBalancer(ctx, lb))
-	require.NoError(t, svc.store.ResumeLBHealth(ctx, lb.LoadBalancerID, "i-newvm"))
-
-	// The old VM's heartbeat read the LB before that and runs now.
-	stale := *lb
-	stale.InstanceID = healthInst
-	svc.recordDataPathEvidence(ctx, &stale, &LBAgentHeartbeatInput{
-		LBID:       aws.String(lb.LoadBalancerID),
-		InstanceID: aws.String(healthInst),
-		NICs:       []*LBAgentNICStatus{nicStatus(healthMACData, false, 5, 1)},
-	})
-
-	rec, err := svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
-	require.NoError(t, err)
-	require.NotNil(t, rec)
-	assert.Equal(t, "i-newvm", rec.InstanceID)
-	assert.Empty(t, rec.NICs, "the old VM's failure must not land on the new binding")
-}

spinifex/domains/elbv2/lb_health.go

@@ -1,333 +0,0 @@
-package elbv2
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"log/slog"
-	"strings"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/nats-io/nats.go/jetstream"
-)
-
-const (
-	// KeyPrefixLBHealth holds the data-path evidence of one load balancer. It is
-	// a separate key from the LB record so whole-record writes cannot drop it.
-	KeyPrefixLBHealth = "lbhealth."
-
-	// StateActiveImpaired is what DescribeLoadBalancers reports while a data NIC
-	// of the LB's current VM is failing its gateway probe.
-	StateActiveImpaired = "active_impaired"
-
-	// lbSuspendBackstop is how long a suspension may outlive its mutation before
-	// a heartbeat from the current VM lifts it.
-	lbSuspendBackstop = 30 * time.Minute
-)
-
-// Outcomes of a recorded repair. Reserved is written before the repair is
-// dispatched, so an entry whose outcome was never updated still counts.
-const (
-	RepairReserved    = "reserved"
-	RepairDispatched  = "dispatched"
-	RepairRefused     = "refused"
-	RepairUnreachable = "unreachable"
-)
-
-// LBNICFailure is one data NIC that was failing its gateway probe when the
-// transition was recorded.
-type LBNICFailure struct {
-	ENIID               string    `json:"eni_id,omitempty"`
-	FailingSince        time.Time `json:"failing_since"`
-	ConsecutiveFailures int64     `json:"consecutive_failures"`
-	RxPackets           int64     `json:"rx_packets"`
-	TxPackets           int64     `json:"tx_packets"`
-	RxDropped           int64     `json:"rx_dropped"`
-	TxDropped           int64     `json:"tx_dropped"`
-}
-
-// LBRepair is one repair of an LB's VM and how it ended.
-type LBRepair struct {
-	At         time.Time `json:"at"`
-	InstanceID string    `json:"instance_id"`
-	ENIID      string    `json:"eni_id,omitempty"`
-	Reason     string    `json:"reason,omitempty"`
-	Outcome    string    `json:"outcome"`
-}
-
-// LBDataPathRecord is the data-path evidence of one load balancer VM. NICs holds
-// only failing NICs, keyed by lower-case MAC; an absent NIC is healthy.
-type LBDataPathRecord struct {
-	InstanceID  string                   `json:"instance_id"`
-	Suspended   bool                     `json:"suspended,omitempty"`
-	SuspendedAt time.Time                `json:"suspended_at,omitzero"`
-	BoundAt     time.Time                `json:"bound_at,omitzero"`
-	NICs        map[string]*LBNICFailure `json:"nics,omitempty"`
-	Repairs     []LBRepair               `json:"repairs,omitempty"`
-}
-
-// failing reports whether any NIC of the record is failing.
-func (r *LBDataPathRecord) failing() bool {
-	return r != nil && len(r.NICs) > 0
-}
-
-// GetLBHealth returns the evidence record of a load balancer, or nil when none
-// has been written.
-func (s *Store) GetLBHealth(ctx context.Context, lbID string) (*LBDataPathRecord, error) {
-	entry, err := s.kv.Get(ctx, KeyPrefixLBHealth+lbID)
-	if err != nil {
-		if errors.Is(err, jetstream.ErrKeyNotFound) {
-			return nil, nil
-		}
-		return nil, err
-	}
-	var rec LBDataPathRecord
-	if err := json.Unmarshal(entry.Value(), &rec); err != nil {
-		return nil, fmt.Errorf("unmarshal lb health: %w", err)
-	}
-	return &rec, nil
-}
-
-// UpdateLBHealth applies mutate to the evidence record under optimistic
-// concurrency, creating it when absent. mutate returns false to write nothing.
-func (s *Store) UpdateLBHealth(ctx context.Context, lbID string, mutate func(*LBDataPathRecord) (bool, error)) (*LBDataPathRecord, error) {
-	return kvstore.Update(ctx, s.kv, KeyPrefixLBHealth+lbID, kvstore.CASConfig{
-		Attempts:       targetGroupCASAttempts,
-		CreateIfAbsent: true,
-		Exhausted: func(_ string, attempts int) error {
-			return fmt.Errorf("elbv2: lb health %s contended, CAS exhausted after %d attempts", lbID, attempts)
-		},
-	}, mutate)
-}
-
-// DeleteLBHealth removes the evidence record of a load balancer.
-func (s *Store) DeleteLBHealth(ctx context.Context, lbID string) error {
-	err := s.kv.Delete(ctx, KeyPrefixLBHealth+lbID)
-	if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
-		return err
-	}
-	return nil
-}
-
-// SuspendLBHealth marks the load balancer as being torn down or reshaped, so
-// no repair is admitted against its VM until the mutation finishes.
-func (s *Store) SuspendLBHealth(ctx context.Context, lbID string) error {
-	_, err := s.UpdateLBHealth(ctx, lbID, func(rec *LBDataPathRecord) (bool, error) {
-		if rec.Suspended {
-			return false, nil
-		}
-		rec.Suspended = true
-		rec.SuspendedAt = time.Now().UTC()
-		return true, nil
-	})
-	return err
-}
-
-// ResumeLBHealth lifts a suspension. A non-empty instanceID also binds the
-// record to that VM and drops the old VM's failing NICs.
-func (s *Store) ResumeLBHealth(ctx context.Context, lbID, instanceID string) error {
-	_, err := s.UpdateLBHealth(ctx, lbID, func(rec *LBDataPathRecord) (bool, error) {
-		changed := rec.Suspended
-		rec.Suspended = false
-		rec.SuspendedAt = time.Time{}
-		if instanceID != "" && rec.InstanceID != instanceID {
-			rec.InstanceID = instanceID
-			rec.BoundAt = time.Now().UTC()
-			rec.NICs = nil
-			changed = true
-		}
-		return changed, nil
-	})
-	return err
-}
-
-// failingNICMACs returns the lower-case MACs that are failing. A probe that
-// failed starts a failure. An established failure ends only when the latest
-// probe succeeded, because an agent restarted by a repair reports healthy for
-// its first misses.
-func failingNICMACs(nics []*LBAgentNICStatus, established map[string]*LBNICFailure) map[string]*LBAgentNICStatus {
-	out := make(map[string]*LBAgentNICStatus)
-	for _, n := range nics {
-		if n == nil || n.MAC == nil || n.ProbeOK == nil {
-			continue
-		}
-		mac := strings.ToLower(*n.MAC)
-		if !*n.ProbeOK || (established[mac] != nil && aws.Int64Value(n.ConsecutiveFailures) != 0) {
-			out[mac] = n
-		}
-	}
-	return out
-}
-
-// recordDataPathEvidence stores the data-path evidence of one heartbeat. It
-// writes only when the set of failing NICs changes, and ignores evidence that
-// does not come from the VM the LB record currently names.
-func (s *ELBv2ServiceImpl) recordDataPathEvidence(ctx context.Context, lb *LoadBalancerRecord, input *LBAgentHeartbeatInput) {
-	if lb.InstanceID == "" || input.InstanceID == nil || *input.InstanceID != lb.InstanceID || len(input.NICs) == 0 {
-		return
-	}
-
-	rec, err := s.store.GetLBHealth(ctx, lb.LoadBalancerID)
-	if err != nil {
-		slog.WarnContext(ctx, "LBAgentHeartbeat: failed to read data-path evidence", "lbId", lb.LoadBalancerID, "err", err)
-		return
-	}
-	converge := false
-	if rec != nil && rec.Suspended {
-		if !s.suspensionConverges(ctx, lb, rec, *input.InstanceID) {
-			return
-		}
-		converge = true
-	}
-	var known map[string]*LBNICFailure
-	if rec != nil && rec.InstanceID == lb.InstanceID {
-		known = rec.NICs
-	}
-	failing := failingNICMACs(input.NICs, known)
-	if !converge && sameMACs(known, failing) && (len(failing) == 0 || rec != nil) {
-		return
-	}
-
-	eniByMAC := s.resolveLBENIs(ctx, lb, failing)
-	now := time.Now().UTC()
-	_, err = s.store.UpdateLBHealth(ctx, lb.LoadBalancerID, func(cur *LBDataPathRecord) (bool, error) {
-		changed := false
-		if cur.Suspended {
-			if !converge || !suspensionLifts(cur, lb.InstanceID, now) {
-				return false, nil
-			}
-			cur.Suspended = false
-			cur.SuspendedAt = time.Time{}
-			changed = true
-			if cur.InstanceID != lb.InstanceID {
-				cur.InstanceID = lb.InstanceID
-				cur.BoundAt = now
-				cur.NICs = nil
-			}
-		}
-		if cur.InstanceID != "" && cur.InstanceID != lb.InstanceID {
-			return changed, nil
-		}
-		prev := cur.NICs
-		failing := failingNICMACs(input.NICs, prev)
-		if cur.InstanceID == lb.InstanceID && sameMACs(prev, failing) {
-			return changed, nil
-		}
-		next := make(map[string]*LBNICFailure, len(failing))
-		for mac, n := range failing {
-			f := &LBNICFailure{
-				ENIID:               eniByMAC[mac],
-				FailingSince:        now,
-				ConsecutiveFailures: aws.Int64Value(n.ConsecutiveFailures),
-				RxPackets:           aws.Int64Value(n.RxPackets),
-				TxPackets:           aws.Int64Value(n.TxPackets),
-				RxDropped:           aws.Int64Value(n.RxDropped),
-				TxDropped:           aws.Int64Value(n.TxDropped),
-			}
-			if old, ok := prev[mac]; ok {
-				f.FailingSince = old.FailingSince
-			}
-			next[mac] = f
-		}
-		cur.InstanceID = lb.InstanceID
-		cur.NICs = next
-		return true, nil
-	})
-	if err != nil {
-		slog.WarnContext(ctx, "LBAgentHeartbeat: failed to store data-path evidence", "lbId", lb.LoadBalancerID, "err", err)
-		return
-	}
-	for mac, n := range failing {
-		if _, ok := known[mac]; !ok {
-			slog.WarnContext(ctx, "ELBv2 data-path probe failing",
-				"lbArn", lb.LoadBalancerArn, "lbId", lb.LoadBalancerID, "instanceId", lb.InstanceID,
-				"eniId", eniByMAC[mac], "mac", mac, "consecutive_failures", aws.Int64Value(n.ConsecutiveFailures),
-				"rx_packets", aws.Int64Value(n.RxPackets), "tx_packets", aws.Int64Value(n.TxPackets),
-				"rx_dropped", aws.Int64Value(n.RxDropped), "tx_dropped", aws.Int64Value(n.TxDropped))
-		}
-	}
-	for mac := range known {
-		if _, ok := failing[mac]; !ok {
-			slog.InfoContext(ctx, "ELBv2 data-path probe recovered",
-				"lbArn", lb.LoadBalancerArn, "lbId", lb.LoadBalancerID, "instanceId", lb.InstanceID, "mac", mac)
-		}
-	}
-}
-
-// suspensionLifts reports whether a suspended record may be lifted by a
-// heartbeat from instanceID: it is bound to an older VM, or it has outlived the
-// backstop while bound to the current one.
-func suspensionLifts(rec *LBDataPathRecord, instanceID string, now time.Time) bool {
-	if rec.InstanceID != "" && rec.InstanceID != instanceID {
-		return true
-	}
-	return !rec.SuspendedAt.IsZero() && now.Sub(rec.SuspendedAt) >= lbSuspendBackstop
-}
-
-// suspensionConverges confirms against a fresh read of the LB record that the
-// VM the heartbeat reports is the one the LB currently names.
-func (s *ELBv2ServiceImpl) suspensionConverges(ctx context.Context, lb *LoadBalancerRecord, rec *LBDataPathRecord, heartbeatInstance string) bool {
-	if !suspensionLifts(rec, heartbeatInstance, time.Now().UTC()) {
-		return false
-	}
-	fresh, err := s.store.GetLoadBalancer(ctx, lb.LoadBalancerID)
-	return err == nil && fresh != nil && fresh.InstanceID == heartbeatInstance
-}
-
-// sameMACs reports whether two failing sets name the same NICs.
-func sameMACs[A, B any](a map[string]A, b map[string]B) bool {
-	if len(a) != len(b) {
-		return false
-	}
-	for mac := range a {
-		if _, ok := b[mac]; !ok {
-			return false
-		}
-	}
-	return true
-}
-
-// resolveLBENIs maps failing MACs to ENI IDs from the LB's cross-account ENIs
-// and one describe of its own ENIs. A MAC it cannot resolve maps to nothing.
-func (s *ELBv2ServiceImpl) resolveLBENIs(ctx context.Context, lb *LoadBalancerRecord, failing map[string]*LBAgentNICStatus) map[string]string {
-	out := make(map[string]string, len(failing))
-	for _, x := range lb.CrossAccountENIs {
-		mac := strings.ToLower(x.ENIMac)
-		if _, ok := failing[mac]; ok {
-			out[mac] = x.ENIID
-		}
-	}
-	if len(out) == len(failing) {
-		return out
-	}
-	for id, eni := range s.describeENIs(ctx, lb.ENIs, lb.AccountID) {
-		mac := strings.ToLower(aws.StringValue(eni.MacAddress))
-		if _, ok := failing[mac]; ok && out[mac] == "" {
-			out[mac] = id
-		}
-	}
-	return out
-}
-
-// impairedByEvidence reports whether the LB's current VM has a failing data NIC.
-func (s *ELBv2ServiceImpl) impairedByEvidence(ctx context.Context, r *LoadBalancerRecord) bool {
-	if r.State != StateActive || r.InstanceID == "" {
-		return false
-	}
-	rec, err := s.store.GetLBHealth(ctx, r.LoadBalancerID)
-	if err != nil {
-		slog.DebugContext(ctx, "lbRecordToSDK: failed to read data-path evidence", "lbId", r.LoadBalancerID, "err", err)
-		return false
-	}
-	return rec != nil && !rec.Suspended && rec.InstanceID == r.InstanceID && rec.failing()
-}
-
-// resumeLBHealth lifts a suspension after a failure that left the VM untouched.
-func (s *ELBv2ServiceImpl) resumeLBHealth(ctx context.Context, lbID string) {
-	if err := s.store.ResumeLBHealth(ctx, lbID, ""); err != nil {
-		slog.WarnContext(ctx, "failed to lift data-path suspension", "lbId", lbID, "err", err)
-	}
-}

spinifex/domains/elbv2/lb_health_test.go

@@ -1,370 +0,0 @@
-package elbv2
-
-import (
-	"context"
-	"errors"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/elbv2"
-	"github.com/nats-io/nats.go/jetstream"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-const (
-	healthLBID    = "lbhealth01"
-	healthInst    = "i-lbvm01"
-	healthMACData = "02:00:00:aa:00:01"
-	healthMACX    = "02:00:00:aa:00:02"
-)
-
-func seedHealthLB(t *testing.T, svc *ELBv2ServiceImpl) *LoadBalancerRecord {
-	t.Helper()
-	lb := &LoadBalancerRecord{
-		LoadBalancerArn: "arn:aws:elasticloadbalancing:us-east-1:" + testAccountID + ":loadbalancer/app/health-lb/" + healthLBID,
-		LoadBalancerID:  healthLBID,
-		Name:            "health-lb",
-		State:           StateActive,
-		InstanceID:      healthInst,
-		AccountID:       testAccountID,
-		CrossAccountENIs: []ExtraENIInput{
-			{ENIID: "eni-cross01", ENIMac: healthMACX},
-		},
-		CreatedAt: time.Now().UTC(),
-	}
-	require.NoError(t, svc.store.PutLoadBalancer(context.Background(), lb))
-	return lb
-}
-
-func nicStatus(mac string, ok bool, failures, rx int64) *LBAgentNICStatus {
-	return &LBAgentNICStatus{
-		MAC:                 aws.String(mac),
-		ProbeOK:             aws.Bool(ok),
-		ConsecutiveFailures: aws.Int64(failures),
-		RxPackets:           aws.Int64(rx),
-		TxPackets:           aws.Int64(rx + 5),
-		RxDropped:           aws.Int64(0),
-		TxDropped:           aws.Int64(2),
-	}
-}
-
-func healthBeat(t *testing.T, svc *ELBv2ServiceImpl, instanceID *string, nics ...*LBAgentNICStatus) {
-	t.Helper()
-	_, err := svc.LBAgentHeartbeat(context.Background(), &LBAgentHeartbeatInput{
-		LBID:       aws.String(healthLBID),
-		InstanceID: instanceID,
-		NICs:       nics,
-	}, testAccountID)
-	require.NoError(t, err)
-}
-
-func healthRevision(t *testing.T, svc *ELBv2ServiceImpl) uint64 {
-	t.Helper()
-	entry, err := svc.store.kv.Get(context.Background(), KeyPrefixLBHealth+healthLBID)
-	require.NoError(t, err)
-	return entry.Revision()
-}
-
-func TestLBHealth_SteadyHeartbeatsWriteNothing(t *testing.T) {
-	t.Parallel()
-	svc := setupTestService(t)
-	seedHealthLB(t, svc)
-
-	for range 5 {
-		healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, true, 0, 100))
-	}
-	rec, err := svc.store.GetLBHealth(context.Background(), healthLBID)
-	require.NoError(t, err)
-	assert.Nil(t, rec, "healthy heartbeats must not create an evidence record")
-
-	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, 3, 100))
-	rev := healthRevision(t, svc)
-	for i := range 5 {
-		healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, int64(4+i), int64(100+i)))
-	}
-	assert.Equal(t, rev, healthRevision(t, svc), "a NIC that stays failing must not rewrite the record")
-}
-
-func TestLBHealth_FailingNICWritesFailingSinceOnceAndRecoveryClears(t *testing.T) {
-	t.Parallel()
-	svc := setupTestService(t)
-	seedHealthLB(t, svc)
-
-	before := time.Now().UTC().Add(-time.Second)
-	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, 3, 777), nicStatus(healthMACX, true, 0, 1))
-
-	rec, err := svc.store.GetLBHealth(context.Background(), healthLBID)
-	require.NoError(t, err)
-	require.NotNil(t, rec)
-	assert.Equal(t, healthInst, rec.InstanceID)
-	require.Len(t, rec.NICs, 1)
-	f := rec.NICs[healthMACData]
-	require.NotNil(t, f)
-	assert.True(t, f.FailingSince.After(before))
-	assert.EqualValues(t, 777, f.RxPackets)
-	assert.EqualValues(t, 782, f.TxPackets)
-	assert.EqualValues(t, 2, f.TxDropped)
-	assert.EqualValues(t, 3, f.ConsecutiveFailures)
-
-	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, true, 0, 900))
-	rec, err = svc.store.GetLBHealth(context.Background(), healthLBID)
-	require.NoError(t, err)
-	require.NotNil(t, rec)
-	assert.Empty(t, rec.NICs, "recovery clears the failing NIC")
-}
-
-func TestLBHealth_SecondNICFailingKeepsFirstFailingSince(t *testing.T) {
-	t.Parallel()
-	svc := setupTestService(t)
-	seedHealthLB(t, svc)
-
-	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, 3, 1))
-	first, err := svc.store.GetLBHealth(context.Background(), healthLBID)
-	require.NoError(t, err)
-	since := first.NICs[healthMACData].FailingSince
-
-	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, 9, 2), nicStatus(healthMACX, false, 3, 2))
-	rec, err := svc.store.GetLBHealth(context.Background(), healthLBID)
-	require.NoError(t, err)
-	require.Len(t, rec.NICs, 2)
-	assert.True(t, since.Equal(rec.NICs[healthMACData].FailingSince))
-	assert.Equal(t, "eni-cross01", rec.NICs[healthMACX].ENIID, "a cross-account NIC resolves to its ENI without a describe")
-}
-
-func TestLBHealth_WrongOrMissingInstanceIDIsIgnored(t *testing.T) {
-	t.Parallel()
-	svc := setupTestService(t)
-	seedHealthLB(t, svc)
-
-	failing := nicStatus(healthMACData, false, 5, 1)
-	healthBeat(t, svc, aws.String("i-someoneelse"), failing)
-	healthBeat(t, svc, aws.String(""), failing)
-	healthBeat(t, svc, nil, failing)
-
-	rec, err := svc.store.GetLBHealth(context.Background(), healthLBID)
-	require.NoError(t, err)
-	assert.Nil(t, rec, "evidence from another VM, or with no instance ID, must be ignored")
-}
-
-func TestLBHealth_WholeRecordPutLeavesEvidenceIntact(t *testing.T) {
-	t.Parallel()
-	svc := setupTestService(t)
-	lb := seedHealthLB(t, svc)
-
-	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, 3, 1))
-	want, err := svc.store.GetLBHealth(context.Background(), healthLBID)
-	require.NoError(t, err)
-	rev := healthRevision(t, svc)
-
-	lb.StateReason = "rewritten by an older daemon"
-	require.NoError(t, svc.store.PutLoadBalancer(context.Background(), lb))
-
-	got, err := svc.store.GetLBHealth(context.Background(), healthLBID)
-	require.NoError(t, err)
-	assert.Equal(t, want, got)
-	assert.Equal(t, rev, healthRevision(t, svc))
-}
-
-func TestLBHealth_DeleteLoadBalancerRemovesTheKey(t *testing.T) {
-	t.Parallel()
-	svc := setupTestService(t)
-	lb := seedHealthLB(t, svc)
-	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, 3, 1))
-
-	_, err := svc.DeleteLoadBalancer(context.Background(), &elbv2.DeleteLoadBalancerInput{
-		LoadBalancerArn: aws.String(lb.LoadBalancerArn),
-	}, testAccountID)
-	require.NoError(t, err)
-
-	rec, err := svc.store.GetLBHealth(context.Background(), healthLBID)
-	require.NoError(t, err)
-	assert.Nil(t, rec)
-}
-
-func TestLBHealth_DeleteLoadBalancerSuspendsBeforeTerminatingTheVM(t *testing.T) {
-	t.Parallel()
-	svc := setupTestService(t)
-	lb := seedHealthLB(t, svc)
-	seen := make(chan bool, 1)
-	svc.InstanceLauncher = &mockSystemInstanceLauncher{
-		onTerminate: func(string) {
-			rec, err := svc.store.GetLBHealth(context.Background(), healthLBID)
-			seen <- err == nil && rec != nil && rec.Suspended
-		},
-	}
-
-	_, err := svc.DeleteLoadBalancer(context.Background(), &elbv2.DeleteLoadBalancerInput{
-		LoadBalancerArn: aws.String(lb.LoadBalancerArn),
-	}, testAccountID)
-	require.NoError(t, err)
-
-	select {
-	case suspended := <-seen:
-		assert.True(t, suspended, "repair must be excluded before the VM is terminated")
-	case <-time.After(5 * time.Second):
-		t.Fatal("the LB VM was never terminated")
-	}
-}
-
-func TestLBHealth_SetSubnetsSuspendsFirstAndResumesOnTheNewVM(t *testing.T) {
-	t.Parallel()
-	svc, vpcSvc, mock := setupSubnetTestService(t)
-	vid := vpcID(t, vpcSvc)
-	sub1 := getTestSubnetID(t, vpcSvc, vid, "10.0.40.0/24", "us-east-1a")
-	sub2 := getTestSubnetID(t, vpcSvc, vid, "10.0.41.0/24", "us-east-1b")
-
-	out, err := svc.CreateLoadBalancer(context.Background(), &elbv2.CreateLoadBalancerInput{
-		Name:    aws.String("health-sub-lb"),
-		Subnets: []*string{aws.String(sub1)},
-	}, testAccountID)
-	require.NoError(t, err)
-	svc.WaitLaunches()
-	lbID := aws.StringValue(out.LoadBalancers[0].LoadBalancerArn)
-	rec0, err := svc.store.GetLoadBalancerByArn(context.Background(), lbID)
-	require.NoError(t, err)
-	require.Equal(t, "i-lb", rec0.InstanceID)
-
-	_, err = svc.LBAgentHeartbeat(context.Background(), &LBAgentHeartbeatInput{
-		LBID:       aws.String(rec0.LoadBalancerID),
-		InstanceID: aws.String("i-lb"),
-		NICs:       []*LBAgentNICStatus{nicStatus(healthMACData, false, 3, 1)},
-	}, testAccountID)
-	require.NoError(t, err)
-
-	suspendedAtTerminate := false
-	mock.onTerminate = func(string) {
-		rec, gerr := svc.store.GetLBHealth(context.Background(), rec0.LoadBalancerID)
-		suspendedAtTerminate = gerr == nil && rec != nil && rec.Suspended
-	}
-	mock.launchResult = &SystemInstanceOutput{InstanceID: "i-new", PrivateIP: "10.0.1.6"}
-
-	_, err = svc.SetSubnets(context.Background(), &elbv2.SetSubnetsInput{
-		LoadBalancerArn: aws.String(lbID),
-		Subnets:         []*string{aws.String(sub1), aws.String(sub2)},
-	}, testAccountID)
-	require.NoError(t, err)
-	assert.True(t, suspendedAtTerminate, "repair must be excluded before the VM is terminated")
-
-	rec, err := svc.store.GetLBHealth(context.Background(), rec0.LoadBalancerID)
-	require.NoError(t, err)
-	require.NotNil(t, rec)
-	assert.False(t, rec.Suspended, "the suspension is lifted once the new VM is published")
-	assert.Equal(t, "i-new", rec.InstanceID)
-	assert.Empty(t, rec.NICs, "the old VM's failing NIC does not carry over")
-}
-
-func TestLBHealth_SetSubnetsFailureBeforeTheVMIsTouchedResumes(t *testing.T) {
-	t.Parallel()
-	svc, vpcSvc, mock := setupSubnetTestService(t)
-	vid := vpcID(t, vpcSvc)
-	sub1 := getTestSubnetID(t, vpcSvc, vid, "10.0.42.0/24", "us-east-1a")
-	sub2 := getTestSubnetID(t, vpcSvc, vid, "10.0.43.0/24", "us-east-1b")
-
-	out, err := svc.CreateLoadBalancer(context.Background(), &elbv2.CreateLoadBalancerInput{
-		Name:    aws.String("health-resume-lb"),
-		Subnets: []*string{aws.String(sub1)},
-	}, testAccountID)
-	require.NoError(t, err)
-	svc.WaitLaunches()
-	arn := aws.StringValue(out.LoadBalancers[0].LoadBalancerArn)
-	rec0, err := svc.store.GetLoadBalancerByArn(context.Background(), arn)
-	require.NoError(t, err)
-
-	mock.terminateErr = assert.AnError
-	_, err = svc.SetSubnets(context.Background(), &elbv2.SetSubnetsInput{
-		LoadBalancerArn: aws.String(arn),
-		Subnets:         []*string{aws.String(sub1), aws.String(sub2)},
-	}, testAccountID)
-	require.Error(t, err)
-
-	rec, err := svc.store.GetLBHealth(context.Background(), rec0.LoadBalancerID)
-	require.NoError(t, err)
-	require.NotNil(t, rec)
-	assert.False(t, rec.Suspended, "a failed reshape that left the VM running must not leave repair excluded")
-}
-
-func TestLBHealth_DescribeShowsImpairedOnlyForFailingNICOfCurrentVM(t *testing.T) {
-	t.Parallel()
-	svc := setupTestService(t)
-	lb := seedHealthLB(t, svc)
-	code := func() string {
-		out, err := svc.DescribeLoadBalancers(context.Background(), &elbv2.DescribeLoadBalancersInput{
-			LoadBalancerArns: []*string{aws.String(lb.LoadBalancerArn)},
-		}, testAccountID)
-		require.NoError(t, err)
-		require.Len(t, out.LoadBalancers, 1)
-		return aws.StringValue(out.LoadBalancers[0].State.Code)
-	}
-
-	assert.Equal(t, StateActive, code())
-
-	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, 3, 1))
-	assert.Equal(t, StateActiveImpaired, code())
-
-	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, true, 0, 2))
-	assert.Equal(t, StateActive, code(), "active again once the probe recovers")
-
-	// Evidence recorded against a VM the LB no longer names never impairs it.
-	_, err := svc.store.UpdateLBHealth(context.Background(), healthLBID, func(r *LBDataPathRecord) (bool, error) {
-		r.InstanceID = "i-oldvm"
-		r.NICs = map[string]*LBNICFailure{healthMACData: {FailingSince: time.Now().UTC()}}
-		return true, nil
-	})
-	require.NoError(t, err)
-	assert.Equal(t, StateActive, code())
-
-	// A LB that is not active is reported as it is.
-	_, err = svc.store.UpdateLBHealth(context.Background(), healthLBID, func(r *LBDataPathRecord) (bool, error) {
-		r.InstanceID = healthInst
-		return true, nil
-	})
-	require.NoError(t, err)
-	require.Equal(t, StateActiveImpaired, code())
-	_, err = svc.store.UpdateLoadBalancer(context.Background(), healthLBID, func(r *LoadBalancerRecord) (bool, error) {
-		r.State = StateFailed
-		return true, nil
-	})
-	require.NoError(t, err)
-	assert.Equal(t, StateFailed, code())
-}
-
-// failKeysKV fails key listing once armed, which is what the listener cascade
-// of DeleteLoadBalancer reads first.
-type failKeysKV struct {
-	jetstream.KeyValue
-
-	armed atomic.Bool
-}
-
-func (k *failKeysKV) Keys(ctx context.Context, opts ...jetstream.WatchOpt) ([]string, error) {
-	if k.armed.Load() {
-		return nil, errors.New("injected key listing failure")
-	}
-	return k.KeyValue.Keys(ctx, opts...)
-}
-
-func TestLBHealth_DeleteLoadBalancerListenerCascadeFailureResumes(t *testing.T) {
-	t.Parallel()
-	svc := setupTestService(t)
-	lb := seedHealthLB(t, svc)
-	kv := &failKeysKV{KeyValue: svc.store.kv}
-	svc.store.kv = kv
-	mock := &mockSystemInstanceLauncher{}
-	svc.InstanceLauncher = mock
-	kv.armed.Store(true)
-
-	_, err := svc.DeleteLoadBalancer(context.Background(), &elbv2.DeleteLoadBalancerInput{
-		LoadBalancerArn: aws.String(lb.LoadBalancerArn),
-	}, testAccountID)
-	require.Error(t, err)
-	kv.armed.Store(false)
-
-	rec, err := svc.store.GetLBHealth(context.Background(), healthLBID)
-	require.NoError(t, err)
-	require.NotNil(t, rec)
-	assert.False(t, rec.Suspended, "a delete that failed before touching the VM must not leave repair excluded")
-	assert.Empty(t, mock.terminateCalls)
-}

spinifex/domains/elbv2/lb_repair.go

@@ -1,377 +0,0 @@
-package elbv2
-
-import (
-	"context"
-	"errors"
-	"log/slog"
-	"sort"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/mulgadc/spinifex/spinifex/domains/ec2/systeminstance"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
-	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
-	"github.com/nats-io/nats.go"
-	"github.com/nats-io/nats.go/jetstream"
-)
-
-const (
-	// KVBucketELBv2Leader holds the lease key of the data-path reconciler.
-	KVBucketELBv2Leader        = "spinifex-elbv2-leader"
-	KVBucketELBv2LeaderVersion = 1
-	KVBucketELBv2LeaderTTL     = 60 * time.Second
-
-	lbRepairLeaderKey     = "datapath"
-	lbRepairLeaseRenew    = 20 * time.Second
-	lbRepairPassInterval  = 30 * time.Second
-	lbRepairMinFailing    = 60 * time.Second
-	lbRepairMinVMAge      = 5 * time.Minute
-	lbRepairCooldown      = 10 * time.Minute
-	lbRepairBudgetWindow  = 24 * time.Hour
-	lbRepairBudget        = 3
-	lbRepairFleetMinCount = 3
-
-	// A heartbeat that persists only every heartbeatPersistInterval is still
-	// fresh for twice that long.
-	lbRepairHeartbeatMaxAge = 2 * heartbeatPersistInterval
-)
-
-// SystemInstanceRebooter is the only capability the repair path has over a VM.
-type SystemInstanceRebooter interface {
-	RebootSystemInstance(instanceID string) error
-}
-
-var _ SystemInstanceRebooter = SystemInstanceLauncher(nil)
-
-// counts reports whether a repair entry uses up cooldown and budget. Only a
-// repair no node received is free.
-func (r LBRepair) counts() bool {
-	return r.Outcome != RepairUnreachable
-}
-
-// repairWindow counts the entries inside window ending at now, and returns the
-// newest of them.
-func repairWindow(repairs []LBRepair, now time.Time, window time.Duration) (n int, newest time.Time) {
-	for _, r := range repairs {
-		if !r.counts() || now.Sub(r.At) >= window {
-			continue
-		}
-		n++
-		if r.At.After(newest) {
-			newest = r.At
-		}
-	}
-	return n, newest
-}
-
-// lbRepairReconciler restarts the VM of a load balancer whose data NIC has been
-// failing its gateway probe. Only the lease holder acts.
-type lbRepairReconciler struct {
-	svc      *ELBv2ServiceImpl
-	rebooter SystemInstanceRebooter
-	isLeader func() bool
-	now      func() time.Time
-
-	mu             sync.Mutex
-	budgetReported map[string]bool
-}
-
-func newLBRepairReconciler(svc *ELBv2ServiceImpl, rebooter SystemInstanceRebooter, isLeader func() bool) *lbRepairReconciler {
-	return &lbRepairReconciler{
-		svc:            svc,
-		rebooter:       rebooter,
-		isLeader:       isLeader,
-		now:            func() time.Time { return time.Now().UTC() },
-		budgetReported: make(map[string]bool),
-	}
-}
-
-// StartDataPathReconciler elects a leader among the daemons and runs the repair
-// pass on it every lbRepairPassInterval until ctx or the service is cancelled.
-func (s *ELBv2ServiceImpl) StartDataPathReconciler(ctx context.Context, holder string) {
-	if s.InstanceLauncher == nil || s.store == nil || s.store.js == nil {
-		return
-	}
-	lease, err := kvlease.New(kvlease.Config{
-		Name:   "elbv2/datapath",
-		Bucket: s.leaderBucket,
-		Key:    lbRepairLeaderKey,
-		Holder: holder,
-		TTL:    KVBucketELBv2LeaderTTL,
-		Renew:  lbRepairLeaseRenew,
-		Retry:  lbRepairLeaseRenew,
-	})
-	if err != nil {
-		slog.ErrorContext(ctx, "ELBv2 data-path reconciler: lease config invalid", "err", err)
-		return
-	}
-	rec := newLBRepairReconciler(s, s.InstanceLauncher, lease.Held)
-	go lease.Run(ctx)
-	go func() {
-		ticker := time.NewTicker(lbRepairPassInterval)
-		defer ticker.Stop()
-		for {
-			select {
-			case <-s.ctx.Done():
-				return
-			case <-ctx.Done():
-				return
-			case <-ticker.C:
-				rec.pass(ctx)
-			}
-		}
-	}()
-}
-
-func (s *ELBv2ServiceImpl) leaderBucket(ctx context.Context) (jetstream.KeyValue, error) {
-	return kvlease.OpenBucket(ctx, s.store.js, kvlease.BucketConfig{
-		Name:    KVBucketELBv2Leader,
-		TTL:     KVBucketELBv2LeaderTTL,
-		Version: KVBucketELBv2LeaderVersion,
-	})
-}
-
-// candidate is a load balancer that is failing and may be repaired.
-type lbRepairCandidate struct {
-	lb    *LoadBalancerRecord
-	rec   *LBDataPathRecord
-	mac   string
-	nic   *LBNICFailure
-	since time.Time
-}
-
-// pass runs one repair pass. It starts at most one repair.
-func (r *lbRepairReconciler) pass(ctx context.Context) {
-	if !r.isLeader() {
-		return
-	}
-	lbs, err := r.svc.store.ListLoadBalancersStrict(ctx)
-	if err != nil {
-		slog.WarnContext(ctx, "ELBv2 data-path reconciler: failed to list load balancers", "err", err)
-		return
-	}
-	now := r.now()
-
-	var heartbeating, failing int
-	var cands []lbRepairCandidate
-	live := make(map[string]bool, len(lbs))
-	for _, lb := range lbs {
-		live[lb.LoadBalancerID] = true
-		if lb.State != StateActive || lb.InstanceID == "" || now.Sub(lb.LastHeartbeat) >= lbRepairHeartbeatMaxAge {
-			continue
-		}
-		heartbeating++
-		rec, err := r.svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
-		if err != nil || rec == nil || rec.Suspended || rec.InstanceID != lb.InstanceID || !rec.failing() {
-			continue
-		}
-		failing++
-		c := lbRepairCandidate{lb: lb, rec: rec}
-		for mac, nic := range rec.NICs {
-			if c.nic == nil || nic.FailingSince.Before(c.since) {
-				c.mac, c.nic, c.since = mac, nic, nic.FailingSince
-			}
-		}
-		cands = append(cands, c)
-	}
-	r.forgetBudgetReports(live)
-
-	if failing >= lbRepairFleetMinCount && failing*2 > heartbeating {
-		slog.WarnContext(ctx, "ELBv2 data-path repair suppressed: fleet-wide probe failure",
-			"failing_lbs", failing, "heartbeating_lbs", heartbeating)
-		return
-	}
-
-	sort.Slice(cands, func(i, j int) bool { return cands[i].since.Before(cands[j].since) })
-	for _, c := range cands {
-		if r.eligible(ctx, c, now) && r.repair(ctx, c) {
-			return
-		}
-	}
-}
-
-// eligible applies the per-LB conditions that need no write.
-func (r *lbRepairReconciler) eligible(ctx context.Context, c lbRepairCandidate, now time.Time) bool {
-	if now.Sub(c.since) < lbRepairMinFailing {
-		return false
-	}
-	if now.Sub(vmRunningSince(c.lb, c.rec)) < lbRepairMinVMAge {
-		return false
-	}
-	n, newest := repairWindow(c.rec.Repairs, now, lbRepairBudgetWindow)
-	if n >= lbRepairBudget {
-		r.reportBudgetSpent(ctx, c, n)
-		return false
-	}
-	r.clearBudgetReport(c.lb.LoadBalancerID)
-	return now.Sub(newest) >= lbRepairCooldown || newest.IsZero()
-}
-
-// vmRunningSince is when the LB's current VM last came up as far as the control
-// plane can tell: its activation, a relaunch, or the last repair.
-func vmRunningSince(lb *LoadBalancerRecord, rec *LBDataPathRecord) time.Time {
-	since := lb.ActiveSince
-	if since.IsZero() {
-		since = lb.CreatedAt
-	}
-	if rec.BoundAt.After(since) {
-		since = rec.BoundAt
-	}
-	for _, e := range rec.Repairs {
-		if e.counts() && e.At.After(since) {
-			since = e.At
-		}
-	}
-	return since
-}
-
-// repair reserves the attempt durably, then dispatches it. It reports whether a
-// repair was started, which ends the pass.
-func (r *lbRepairReconciler) repair(ctx context.Context, c lbRepairCandidate) bool {
-	if !r.isLeader() {
-		return false
-	}
-	lb, err := r.svc.store.GetLoadBalancer(ctx, c.lb.LoadBalancerID)
-	if err != nil || lb == nil || lb.State != StateActive || lb.InstanceID != c.lb.InstanceID {
-		return false
-	}
-	now := r.now()
-	reason := "data NIC failing gateway probe"
-	var attempt int
-	reserved := false
-	_, err = r.svc.store.UpdateLBHealth(ctx, lb.LoadBalancerID, func(cur *LBDataPathRecord) (bool, error) {
-		// The callback reruns after a conflict, so admission is judged by the run
-		// that wrote the reservation and nothing earlier.
-		reserved, attempt = false, 0
-		nic := cur.NICs[c.mac]
-		if cur.Suspended || cur.InstanceID != lb.InstanceID || nic == nil || now.Sub(nic.FailingSince) < lbRepairMinFailing {
-			return false, nil
-		}
-		n, newest := repairWindow(cur.Repairs, now, lbRepairBudgetWindow)
-		if n >= lbRepairBudget || (!newest.IsZero() && now.Sub(newest) < lbRepairCooldown) {
-			return false, nil
-		}
-		cur.Repairs = append(pruneRepairs(cur.Repairs, now), LBRepair{
-			At: now, InstanceID: lb.InstanceID, ENIID: nic.ENIID, Reason: reason, Outcome: RepairReserved,
-		})
-		attempt = n + 1
-		reserved = true
-		return true, nil
-	})
-	if err != nil {
-		slog.WarnContext(ctx, "ELBv2 data-path repair: reservation failed", "lbId", lb.LoadBalancerID, "err", err)
-		return false
-	}
-	if !reserved {
-		return false
-	}
-
-	attrs := repairLogAttrs(lb, c, attempt, reason, now)
-	if !r.isLeader() {
-		slog.WarnContext(ctx, "ELBv2 data-path repair not dispatched: leadership lost after reservation", attrs...)
-		return true
-	}
-	slog.WarnContext(ctx, "ELBv2 data-path repair started", attrs...)
-	outcome := r.dispatch(lb.InstanceID)
-	if outcome != "" {
-		r.recordOutcome(ctx, lb.LoadBalancerID, now, lb.InstanceID, outcome)
-	}
-	switch outcome {
-	case RepairDispatched:
-		slog.InfoContext(ctx, "ELBv2 data-path repair dispatched", attrs...)
-	case RepairRefused:
-		slog.ErrorContext(ctx, "ELBv2 data-path repair refused by the owning node", attrs...)
-	case RepairUnreachable:
-		slog.WarnContext(ctx, "ELBv2 data-path repair: owning node unreachable, will retry", attrs...)
-	default:
-		slog.WarnContext(ctx, "ELBv2 data-path repair: outcome unknown, counted", attrs...)
-	}
-	return true
-}
-
-// dispatch asks the owning node to restart exactly the LB's own VM. It returns
-// an empty outcome when the reply was lost, which leaves the entry reserved.
-func (r *lbRepairReconciler) dispatch(instanceID string) string {
-	err := r.rebooter.RebootSystemInstance(instanceID)
-	switch {
-	case err == nil:
-		return RepairDispatched
-	case errors.Is(err, nats.ErrNoResponders), strings.Contains(err.Error(), "no responders"):
-		return RepairUnreachable
-	case errors.Is(err, systeminstance.ErrSystemInstanceNotFound),
-		errors.Is(err, systeminstance.ErrSystemInstanceNotSystem),
-		errors.Is(err, systeminstance.ErrSystemInstanceOwnershipUnknown):
-		return RepairRefused
-	default:
-		return ""
-	}
-}
-
-func (r *lbRepairReconciler) recordOutcome(ctx context.Context, lbID string, at time.Time, instanceID, outcome string) {
-	_, err := r.svc.store.UpdateLBHealth(ctx, lbID, func(cur *LBDataPathRecord) (bool, error) {
-		for i := range cur.Repairs {
-			e := &cur.Repairs[i]
-			if e.At.Equal(at) && e.InstanceID == instanceID && e.Outcome == RepairReserved {
-				e.Outcome = outcome
-				return true, nil
-			}
-		}
-		return false, nil
-	})
-	if err != nil {
-		slog.WarnContext(ctx, "ELBv2 data-path repair: failed to record outcome", "lbId", lbID, "outcome", outcome, "err", err)
-	}
-}
-
-// pruneRepairs drops entries older than the budget window.
-func pruneRepairs(repairs []LBRepair, now time.Time) []LBRepair {
-	kept := repairs[:0:0]
-	for _, e := range repairs {
-		if now.Sub(e.At) < lbRepairBudgetWindow {
-			kept = append(kept, e)
-		}
-	}
-	return kept
-}
-
-func repairLogAttrs(lb *LoadBalancerRecord, c lbRepairCandidate, attempt int, reason string, now time.Time) []any {
-	return []any{
-		"lbArn", lb.LoadBalancerArn, "lbId", lb.LoadBalancerID, "instanceId", lb.InstanceID, "nodeId", lb.NodeID,
-		"eniId", c.nic.ENIID, "mac", c.mac, "failing_since", c.nic.FailingSince,
-		"failing_ms", otelsetup.Millis(now.Sub(c.nic.FailingSince)),
-		"consecutive_failures", c.nic.ConsecutiveFailures,
-		"rx_packets", c.nic.RxPackets, "tx_packets", c.nic.TxPackets,
-		"rx_dropped", c.nic.RxDropped, "tx_dropped", c.nic.TxDropped,
-		"repair_attempt", attempt, "reason", reason,
-	}
-}
-
-func (r *lbRepairReconciler) reportBudgetSpent(ctx context.Context, c lbRepairCandidate, n int) {
-	r.mu.Lock()
-	already := r.budgetReported[c.lb.LoadBalancerID]
-	r.budgetReported[c.lb.LoadBalancerID] = true
-	r.mu.Unlock()
-	if already {
-		return
-	}
-	slog.ErrorContext(ctx, "ELBv2 data-path repair budget exhausted",
-		"lbArn", c.lb.LoadBalancerArn, "lbId", c.lb.LoadBalancerID, "instanceId", c.lb.InstanceID,
-		"eniId", c.nic.ENIID, "mac", c.mac, "repairs_24h", n)
-}
-
-func (r *lbRepairReconciler) clearBudgetReport(lbID string) {
-	r.mu.Lock()
-	delete(r.budgetReported, lbID)
-	r.mu.Unlock()
-}
-
-func (r *lbRepairReconciler) forgetBudgetReports(live map[string]bool) {
-	r.mu.Lock()
-	defer r.mu.Unlock()
-	for id := range r.budgetReported {
-		if !live[id] {
-			delete(r.budgetReported, id)
-		}
-	}
-}