feat(lambda): provide test-link SDK gateway settings (#1655)

* fix(eks): alarm on a control-plane VM with no cluster record instead of terminating it

* chore(eks): order test imports

* feat(eks): add spx admin eks orphans and share the cluster-record judgement with the billable reaper

* fix(eks): list orphan control planes through an EC2-owned inventory capability

* test(eks): cover the orphan report ENI reader and the inventory fail-closed path

* fix(eks): report only running control planes and treat a missing ENI as gone

* fix(eks): open DNS watch buckets attach-only so a deleted account bucket is not recreated

* fix(eks): read control-plane ENIs from their record so a corrupt one is unreadable, and report the zone

Makefile

@@ -90,6 +90,7 @@ go_build:
 LAMBDA_TESTLINK_DIR := ./bin/lambda-testlink
 LAMBDA_TESTLINK_PKG := github.com/mulgadc/spinifex/spinifex/runtime/daemon
 LAMBDA_TESTLINK_PIN := $(LAMBDA_TESTLINK_PKG).lambdaHardKillSHA256
+LAMBDA_TESTLINK_GW := github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw
 LAMBDA_DIGEST_RE := ^[0-9a-f]{64}$$
 build-lambda-testlink: override export LAMBDA_IMAGE_DIGEST := $(value LAMBDA_IMAGE_DIGEST)
 build-lambda-testlink:
@@ -104,8 +105,8 @@ build-lambda-testlink:
 	@digest=$$(sha256sum $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so | cut -d' ' -f1); \
 	test $${#digest} -eq 64 || { echo "initializer digest missing"; exit 1; }; \
 	echo "initializer sha256 $$digest"; \
-	pin="-X $(LAMBDA_TESTLINK_PIN)=$$digest -X $(LAMBDA_TESTLINK_PKG).lambdaImageDigest=$$LAMBDA_IMAGE_DIGEST"; \
+	pin="-X $(LAMBDA_TESTLINK_PIN)=$$digest -X $(LAMBDA_TESTLINK_PKG).lambdaImageDigest=$$LAMBDA_IMAGE_DIGEST -X $(LAMBDA_TESTLINK_GW).lambdaImageDigest=$$LAMBDA_IMAGE_DIGEST"; \
-	LAMBDA_TESTLINK_INITIALIZER=$(abspath $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so) go test -count=1 -tags lambdatestlink -ldflags "$$pin" -run '^TestLambda(HardKillPin|ImageDigestPin)$$' ./spinifex/runtime/daemon/ && \
+	LAMBDA_TESTLINK_INITIALIZER=$(abspath $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so) go test -count=1 -tags lambdatestlink -ldflags "$$pin" -run '^TestLambda(HardKillPin|ImageDigestPin)$$' ./spinifex/runtime/daemon/ ./spinifex/runtime/roles/awsgw/ && \
 	go build $(GO_BUILD_MOD) -tags lambdatestlink -ldflags "$(subst -s -w ,,$(LDFLAGS)) $$pin" -o $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME) cmd/spinifex/main.go
 	@mkdir -p $(LAMBDA_TESTLINK_DIR)
 	mv $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME) $(LAMBDA_TESTLINK_DIR)/

architecture/layering/allowlist_test.go

@@ -17,7 +17,6 @@ var allowlist = []allowance{
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/instance", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/routetable", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/snapshot", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
-	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/volume", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/vpc", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 
 	{From: "agents/ecs/agent", To: "domains/ecs", Debt: "agents/ecs/agent", Reason: "assignment/state wire types from the ECS implementation; needs a guest/controller contract"},

build/microvm/init.sh

@@ -32,6 +32,8 @@ load_mod qemu_fw_cfg || load_mod fw_cfg_sysfs || true
 load_mod failover || true
 load_mod net_failover || true
 load_mod virtio_net || true
+# lb-agent probes each data NIC's gateway on an AF_PACKET socket.
+load_mod af_packet || true
 
 NETCFG=/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
 i=0

build/microvm/lambda-runtime/init.sh

@@ -16,8 +16,8 @@ load_mod() {
     done
     return 1
 }
-# lambda-agent reads its identity from fw_cfg and unloads qemu_fw_cfg before
+# lambda-agent reads its identity from fw_cfg, or the configuration disk, and
-# anything else runs. virtio_net needs net_failover and failover, and the
+# seals it before anything else runs. virtio_net needs net_failover and failover, and the
 # insmod fallback resolves no dependencies, so load the chain bottom-up.
 load_mod qemu_fw_cfg || echo "[init] qemu_fw_cfg did not load" >/dev/console
 load_mod failover || true
@@ -26,8 +26,30 @@ load_mod virtio_net || echo "[init] virtio_net did not load" >/dev/console
 
 ip link set lo up
 
+# Without fw_cfg data the executor attaches the same entries as a tar archive
+# on the second disk; lambda-agent reads them here and erases the disk before
+# the function starts. qemu_fw_cfg loads without a device, so test the entry.
+CFG_ROOT=/spinifex-cfg
+FWCFG_ROOT=/sys/firmware/qemu_fw_cfg
+if [ ! -f "$FWCFG_ROOT/by_name/opt/spinifex/netcfg/raw" ]; then
+    load_mod virtio_blk || echo "[init] virtio_blk did not load" >/dev/console
+    i=0
+    while [ "$i" -lt 25 ] && [ ! -b /dev/vdb ]; do
+        sleep 0.04
+        i=$((i + 1))
+    done
+    if [ -b /dev/vdb ]; then
+        mkdir -m 0700 "$CFG_ROOT"
+        if ! mount -t tmpfs -o mode=0700,size=1m tmpfs "$CFG_ROOT" ||
+            ! tar -xf /dev/vdb -C "$CFG_ROOT"; then
+            echo "[init] configuration disk not readable" >/dev/console
+        fi
+        FWCFG_ROOT=$CFG_ROOT
+    fi
+fi
+
 # The guest's addresses come from the same netcfg entry as other system VMs.
-NETCFG=/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
+NETCFG=$FWCFG_ROOT/by_name/opt/spinifex/netcfg/raw
 i=0
 while [ "$i" -lt 25 ] && [ ! -f "$NETCFG" ]; do
     sleep 0.04

cmd/lambda-agent/connect.go

@@ -5,6 +5,7 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
+	"os"
 
 	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
@@ -25,9 +26,22 @@ type startup struct {
 }
 
 func defaultStartup() startup {
+	if _, err := os.Stat(hostlink.ConfigDiskRoot); err == nil {
+		return configDiskStartup(hostlink.ConfigDiskRoot, hostlink.ConfigDiskDevice)
+	}
 	return startup{fwCfgDir: hostlink.FwCfgDir, fwCfgRoot: hostlink.FwCfgRoot, unload: hostlink.UnloadFwCfg}
 }
 
+// configDiskStartup reads the entries init extracted from the configuration
+// disk, and seals by erasing the disk and removing the extracted tree.
+func configDiskStartup(root, device string) startup {
+	return startup{
+		fwCfgDir:  root + "/by_name/" + lambdav1.FwCfgPrefix,
+		fwCfgRoot: root,
+		unload:    func() error { return hostlink.SealConfigDisk(device, root) },
+	}
+}
+
 // connect loads the identity, seals fw_cfg whether or not the load succeeded,
 // then dials the host. Under TLS 1.3 a host refusal is seen only on first read.
 func (s startup) connect(ctx context.Context) (*tls.Conn, lambdav1.Scope, int, error) {

cmd/lambda-agent/connect_configdisk_test.go

@@ -0,0 +1,43 @@
+//test:in-package — package main cannot be imported by an external test package.
+
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// An identity that does not load still seals the configuration disk: the
+// guest key on it is erased before the agent refuses to start.
+func TestConfigDiskStartup_RefusedLoadErasesDisk(t *testing.T) {
+	dir := t.TempDir()
+	root := filepath.Join(dir, "cfg")
+	require.NoError(t, os.MkdirAll(filepath.Join(root, "by_name"), 0o700))
+	device := filepath.Join(dir, "vdb")
+	require.NoError(t, os.WriteFile(device, bytes.Repeat([]byte("guest-key"), 512), 0o600))
+
+	_, _, code, err := configDiskStartup(root, device).connect(t.Context())
+	require.Error(t, err)
+	require.Equal(t, exitRefused, code)
+	got, rerr := os.ReadFile(device)
+	require.NoError(t, rerr)
+	require.Len(t, got, 9*512)
+	require.Equal(t, make([]byte, len(got)), got, "configuration disk erased")
+}
+
+// A disk that cannot be erased fails closed: the agent refuses, and the
+// extracted tree is kept so the seal check sees it.
+func TestConfigDiskStartup_UnerasableDiskRefuses(t *testing.T) {
+	dir := t.TempDir()
+	root := filepath.Join(dir, "cfg")
+	require.NoError(t, os.MkdirAll(filepath.Join(root, "by_name"), 0o700))
+
+	_, _, code, err := configDiskStartup(root, filepath.Join(dir, "missing")).connect(t.Context())
+	require.ErrorContains(t, err, "erase configuration disk")
+	require.Equal(t, exitRefused, code)
+	require.DirExists(t, root)
+}

cmd/lambda-agent/sdk_config_testlink.go

@@ -0,0 +1,65 @@
+//go:build lambdatestlink
+
+package main
+
+import (
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"net"
+	"net/url"
+	"os"
+	"path/filepath"
+)
+
+const (
+	sdkEndpointEntry = "test-sdk-endpoint"
+	sdkCAEntry       = "test-sdk-ca"
+	sdkBundlePath    = "/run/lambda-sdk/ca-bundle.pem"
+	systemBundlePath = "/etc/ssl/certs/ca-certificates.crt"
+)
+
+type sdkConfig struct {
+	endpoint string
+	ca       []byte
+}
+
+// readSDKConfig reads optional public settings before the boot config is sealed.
+func readSDKConfig(dir string) (*sdkConfig, error) {
+	endpoint, endpointErr := os.ReadFile(filepath.Join(dir, sdkEndpointEntry, "raw"))
+	ca, caErr := os.ReadFile(filepath.Join(dir, sdkCAEntry, "raw"))
+	if errors.Is(endpointErr, os.ErrNotExist) && errors.Is(caErr, os.ErrNotExist) {
+		return nil, nil
+	}
+	if endpointErr != nil || caErr != nil {
+		return nil, errors.New("incomplete SDK gateway settings")
+	}
+	u, err := url.Parse(string(endpoint))
+	if err != nil || u.Scheme != "https" || u.User != nil || u.Path != "" || u.RawQuery != "" || u.Fragment != "" ||
+		net.ParseIP(u.Hostname()) == nil || u.Port() == "" {
+		return nil, errors.New("invalid SDK gateway endpoint")
+	}
+	if len(ca) > 64<<10 || !x509.NewCertPool().AppendCertsFromPEM(ca) {
+		return nil, errors.New("invalid SDK gateway CA")
+	}
+	return &sdkConfig{endpoint: string(endpoint), ca: ca}, nil
+}
+
+// writeBundle combines the system roots with the cluster's public root.
+func (c *sdkConfig) writeBundle(path, systemPath string) error {
+	system, err := os.ReadFile(systemPath)
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("read system CA bundle: %w", err)
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		return fmt.Errorf("create SDK CA directory: %w", err)
+	}
+	if err := os.Chmod(filepath.Dir(path), 0o755); err != nil {
+		return fmt.Errorf("make SDK CA directory readable: %w", err)
+	}
+	bundle := append(append(system, '\n'), c.ca...)
+	if err := os.WriteFile(path, bundle, 0o644); err != nil {
+		return fmt.Errorf("write SDK CA bundle: %w", err)
+	}
+	return os.Chmod(path, 0o644)
+}

cmd/lambda-agent/sdk_config_testlink_test.go

@@ -0,0 +1,106 @@
+//go:build lambdatestlink
+
+//test:in-package exercises the unexported boot configuration reader
+package main
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSDKConfig_OptionalAndInvalid(t *testing.T) {
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
+	defer server.Close()
+	ca := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw})
+	for _, tc := range []struct {
+		name     string
+		endpoint []byte
+		ca       []byte
+		wantErr  bool
+		want     bool
+	}{
+		{name: "absent"},
+		{name: "valid", endpoint: []byte(server.URL), ca: ca, want: true},
+		{name: "missing ca", endpoint: []byte(server.URL), wantErr: true},
+		{name: "missing endpoint", ca: ca, wantErr: true},
+		{name: "invalid endpoint", endpoint: []byte("http://127.0.0.1:1234"), ca: ca, wantErr: true},
+		{name: "invalid ca", endpoint: []byte(server.URL), ca: []byte("not a certificate"), wantErr: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			dir := t.TempDir()
+			for name, value := range map[string][]byte{sdkEndpointEntry: tc.endpoint, sdkCAEntry: tc.ca} {
+				if value != nil {
+					require.NoError(t, os.MkdirAll(filepath.Join(dir, name), 0o700))
+					require.NoError(t, os.WriteFile(filepath.Join(dir, name, "raw"), value, 0o600))
+				}
+			}
+			cfg, err := readSDKConfig(dir)
+			if tc.wantErr {
+				require.Error(t, err)
+				require.Nil(t, cfg)
+				return
+			}
+			require.NoError(t, err)
+			if !tc.want {
+				require.Nil(t, cfg)
+				return
+			}
+			bundle := filepath.Join(t.TempDir(), "sdk", "ca.pem")
+			require.NoError(t, cfg.writeBundle(bundle, filepath.Join(t.TempDir(), "absent")))
+			data, err := os.ReadFile(bundle)
+			require.NoError(t, err)
+			roots := x509.NewCertPool()
+			require.True(t, roots.AppendCertsFromPEM(data))
+			client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: roots}}}
+			resp, err := client.Get(cfg.endpoint)
+			require.NoError(t, err)
+			defer resp.Body.Close()
+			assert.Equal(t, http.StatusOK, resp.StatusCode)
+		})
+	}
+}
+
+func TestSDKConfig_BundleKeepsSystemRoots(t *testing.T) {
+	cluster := httptest.NewTLSServer(nil)
+	defer cluster.Close()
+	clusterCA := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cluster.Certificate().Raw})
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+	template := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test system root"},
+		NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour),
+		IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign}
+	der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
+	require.NoError(t, err)
+	systemCA := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+	dir := t.TempDir()
+	systemPath, bundlePath := filepath.Join(dir, "system.pem"), filepath.Join(dir, "sdk", "ca.pem")
+	require.NoError(t, os.WriteFile(systemPath, systemCA, 0o600))
+	require.NoError(t, (&sdkConfig{ca: clusterCA}).writeBundle(bundlePath, systemPath))
+	bundle, err := os.ReadFile(bundlePath)
+	require.NoError(t, err)
+	roots := x509.NewCertPool()
+	require.True(t, roots.AppendCertsFromPEM(bundle))
+	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: roots}}}
+	resp, err := client.Get(cluster.URL)
+	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
+	systemCert, err := x509.ParseCertificate(der)
+	require.NoError(t, err)
+	_, err = systemCert.Verify(x509.VerifyOptions{Roots: roots})
+	require.NoError(t, err)
+}

cmd/lambda-agent/start_testlink.go

@@ -12,6 +12,7 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
+	"strings"
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/agent"
@@ -31,13 +32,17 @@ type testlink struct {
 	pkg            taskPackage
 	runtimeAddr    string
 	startBootstrap func(ctx context.Context, env []string) (*exec.Cmd, error)
+	caBundlePath   string
+	systemCAPath   string
 }
 
 func start(ctx context.Context) int {
 	t := testlink{
-		startup:     defaultStartup(),
+		startup:      defaultStartup(),
-		pkg:         devTaskPackage(),
+		pkg:          devTaskPackage(),
-		runtimeAddr: runtimeAddr,
+		runtimeAddr:  runtimeAddr,
+		caBundlePath: sdkBundlePath,
+		systemCAPath: systemBundlePath,
 		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
 			return agent.StartBootstrap(ctx, taskRoot+"/bootstrap", env, os.Stdout, os.Stderr)
 		},
@@ -53,6 +58,7 @@ func (t testlink) run(parent context.Context) (int, error) {
 	// The package values are fw_cfg entries, so read them before connect seals it.
 	spec, specErr := readPackageSpec(t.fwCfgDir)
 	grant, grantErr := readGrant(t.fwCfgDir)
+	sdk, sdkErr := readSDKConfig(t.fwCfgDir)
 	conn, scope, code, err := t.connect(parent)
 	if err != nil {
 		return code, errors.Join(err, specErr, grantErr)
@@ -85,6 +91,15 @@ func (t testlink) run(parent context.Context) (int, error) {
 	}
 	cred := *first.Credential
 	first.Credential = nil
+	if sdkErr != nil {
+		slog.Warn("lambda-agent: SDK gateway settings unavailable", "err", sdkErr)
+	}
+	if sdk != nil {
+		if err := sdk.writeBundle(t.caBundlePath, t.systemCAPath); err != nil {
+			slog.Warn("lambda-agent: SDK gateway settings unavailable", "err", err)
+			sdk = nil
+		}
+	}
 	if err := t.pkg.install(spec); err != nil {
 		return exitRefused, fmt.Errorf("install package: %w", err)
 	}
@@ -100,7 +115,8 @@ func (t testlink) run(parent context.Context) (int, error) {
 
 	ctx, cancel := context.WithCancelCause(parent)
 	defer cancel(nil)
-	cmd, err := t.startBootstrap(ctx, bootstrapEnv(ln.Addr().String(), spec.handler, cred))
+	region, _, _ := strings.Cut(string(scope), "/")
+	cmd, err := t.startBootstrap(ctx, bootstrapEnv(ln.Addr().String(), spec.handler, cred, region, sdk, t.caBundlePath))
 	if err != nil {
 		return exitLink, fmt.Errorf("start bootstrap: %w", err)
 	}
@@ -124,10 +140,9 @@ func (t testlink) run(parent context.Context) (int, error) {
 	return exitClosed, nil
 }
 
-// bootstrapEnv is the bootstrap's whole environment: the runtime variables and
+// bootstrapEnv supplies the runtime, execution-role and SDK settings.
-// the execution role's three credential variables.
+func bootstrapEnv(runtimeAPI, handler string, cred hostlink.Credential, region string, sdk *sdkConfig, caPath string) []string {
-func bootstrapEnv(runtimeAPI, handler string, cred hostlink.Credential) []string {
+	env := []string{
-	return []string{
 		"AWS_LAMBDA_RUNTIME_API=" + runtimeAPI,
 		"_HANDLER=" + handler,
 		"LAMBDA_TASK_ROOT=" + taskRoot,
@@ -135,7 +150,13 @@ func bootstrapEnv(runtimeAPI, handler string, cred hostlink.Credential) []string
 		"AWS_ACCESS_KEY_ID=" + cred.AccessKeyID,
 		"AWS_SECRET_ACCESS_KEY=" + cred.SecretAccessKey,
 		"AWS_SESSION_TOKEN=" + cred.SessionToken,
+		"AWS_REGION=" + region,
+		"AWS_DEFAULT_REGION=" + region,
 	}
+	if sdk != nil {
+		env = append(env, "AWS_ENDPOINT_URL="+sdk.endpoint, "AWS_CA_BUNDLE="+caPath)
+	}
+	return env
 }
 
 // shutdownGrace is how long past the invocation deadline the agent waits for

cmd/lambda-agent/start_testlink_test.go

@@ -7,8 +7,10 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
+	"encoding/pem"
 	"io"
 	"net/http"
+	"net/http/httptest"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -70,6 +72,7 @@ type bootstrapStarts struct {
 	sealed  bool
 	// handlerOut receives the _HANDLER the bootstrap process itself saw.
 	handlerOut string
+	caBundle   string
 	// env is the environment the agent gave the bootstrap.
 	env []string
 }
@@ -94,6 +97,7 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 	b.started = make(chan struct{}, 1)
 	b.task = t.TempDir()
 	b.handlerOut = filepath.Join(t.TempDir(), "handler")
+	b.caBundle = filepath.Join(t.TempDir(), "sdk", "ca-bundle.pem")
 	return testlink{
 		startup: g.startup(),
 		pkg: taskPackage{
@@ -102,7 +106,9 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 			mountDir:   func(string) error { return nil },
 			sealDir:    func(string) error { b.sealed = true; return nil },
 		},
-		runtimeAddr: "127.0.0.1:0",
+		runtimeAddr:  "127.0.0.1:0",
+		caBundlePath: b.caBundle,
+		systemCAPath: filepath.Join(t.TempDir(), "system-ca.pem"),
 		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
 			b.n++
 			b.env = env
@@ -176,10 +182,45 @@ func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
 		"AWS_ACCESS_KEY_ID=" + testCredential.AccessKeyID,
 		"AWS_SECRET_ACCESS_KEY=" + testCredential.SecretAccessKey,
 		"AWS_SESSION_TOKEN=" + testCredential.SessionToken,
+		"AWS_REGION=ap-southeast-2",
+		"AWS_DEFAULT_REGION=ap-southeast-2",
 	}, b.env, "the bootstrap gets exactly the runtime and credential variables")
 	assert.NotEmpty(t, envValue(b.env, "AWS_LAMBDA_RUNTIME_API"))
 }
 
+func TestTestlinkProvidesSDKSettingsToBootstrap(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
+	defer server.Close()
+	h := newHost(t)
+	h.accept(ctx)
+	entries := h.entries()
+	entries[sdkEndpointEntry] = []byte(server.URL)
+	entries[sdkCAEntry] = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw})
+	var b bootstrapStarts
+	done := runTestlinkAsync(ctx, b.testlink(t, entries))
+	conn := h.guest(t)
+	defer conn.Close()
+	hs := hostlink.NewTestStream(conn)
+	req := hostlink.Request{Credential: &testCredential, GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: []byte(`{}`)}
+	require.NoError(t, hs.HostSend(req))
+	_, err := hs.HostResult(ctx)
+	require.NoError(t, err)
+	res := wait(t, done, "agent to stop")
+	require.NoError(t, res.err)
+	assert.Equal(t, server.URL, envValue(b.env, "AWS_ENDPOINT_URL"))
+	assert.Equal(t, b.caBundle, envValue(b.env, "AWS_CA_BUNDLE"))
+	assert.Equal(t, "ap-southeast-2", envValue(b.env, "AWS_REGION"))
+	assert.Equal(t, "ap-southeast-2", envValue(b.env, "AWS_DEFAULT_REGION"))
+	ca, err := os.ReadFile(b.caBundle)
+	require.NoError(t, err)
+	assert.Contains(t, string(ca), string(entries[sdkCAEntry]))
+	info, err := os.Stat(b.caBundle)
+	require.NoError(t, err)
+	assert.Equal(t, os.FileMode(0o644), info.Mode().Perm())
+}
+
 var testCredential = hostlink.Credential{AccessKeyID: "ASIAGUESTKEY", SecretAccessKey: "guest-secret", SessionToken: "guest-token"}
 
 func envValue(env []string, name string) string {

contracts/ec2/v1/commands.go

@@ -12,6 +12,11 @@ const (
 	// this contract family. It is the NATS permission/subscription pattern for
 	// the per-instance command route, rather than the broader descendant match.
 	InstanceCommandSubjectWildcard = InstanceCommandSubjectPrefix + "*"
+
+	// HandledHeader is set to "1" on every reply an owner sends to a
+	// modify-instance-metadata-options or set-disable-api-termination command.
+	// An owner that predates those commands cannot set it.
+	HandledHeader = "Spinifex-Ec2-Cmd-Handled"
 )
 
 // InstanceCommandSubject returns the targeted command subject for instanceID.
@@ -23,7 +28,9 @@ func InstanceCommandSubject(instanceID string) string {
 }
 
 // EC2InstanceCommand is the NATS wire format for EC2 instance commands
-// (stop, terminate, start, attach/detach-volume, attach/detach-eni).
+// (stop, terminate, start, reboot, attach/detach-volume, attach/detach-eni,
+// IAM profile, spot lineage, tags, monitoring, metadata options and
+// termination protection).
 type EC2InstanceCommand struct {
 	ID                        string                     `json:"id"`
 	Attributes                EC2CommandAttributes       `json:"attributes"`
@@ -35,6 +42,9 @@ type EC2InstanceCommand struct {
 	SpotLineageData           *SpotLineageData           `json:"spot_lineage_data,omitempty"`
 	InstanceTagsData          *InstanceTagsData          `json:"instance_tags_data,omitempty"`
 	InstanceMonitoringData    *InstanceMonitoringData    `json:"instance_monitoring_data,omitempty"`
+
+	InstanceMetadataOptionsData *InstanceMetadataOptionsData `json:"instance_metadata_options_data,omitempty"`
+	DisableApiTerminationData   *DisableApiTerminationData   `json:"disable_api_termination_data,omitempty"`
 }
 
 // EC2CommandAttributes indicates which action the daemon should perform.
@@ -52,6 +62,9 @@ type EC2CommandAttributes struct {
 	SetInstanceTags             bool `json:"set_instance_tags,omitempty"`
 	RemoveInstanceTags          bool `json:"remove_instance_tags,omitempty"`
 	SetInstanceMonitoring       bool `json:"set_instance_monitoring,omitempty"`
+
+	ModifyInstanceMetadataOptions bool `json:"modify_instance_metadata_options,omitempty"`
+	SetDisableApiTermination      bool `json:"set_disable_api_termination,omitempty"`
 }
 
 // AttachVolumeData carries parameters for an attach-volume command.
@@ -103,6 +116,23 @@ type InstanceMonitoringData struct {
 	Enabled bool `json:"enabled"`
 }
 
+// InstanceMetadataOptionsData carries the requested metadata options for a
+// modify-instance-metadata-options command. Unset fields are left unchanged.
+type InstanceMetadataOptionsData struct {
+	HttpTokens              string `json:"http_tokens,omitempty"`
+	HttpEndpoint            string `json:"http_endpoint,omitempty"`
+	HttpProtocolIpv6        string `json:"http_protocol_ipv6,omitempty"`
+	InstanceMetadataTags    string `json:"instance_metadata_tags,omitempty"`
+	HttpPutResponseHopLimit *int64 `json:"http_put_response_hop_limit,omitempty"`
+}
+
+// DisableApiTerminationData carries the desired termination-protection value
+// for a set-disable-api-termination command. Carried as data so "false" is an
+// explicit value, not the absence of one.
+type DisableApiTerminationData struct {
+	Value bool `json:"value"`
+}
+
 // SpotLineageData carries the SIR id stamped onto a spot-launched instance.
 // Lifecycle is always "spot" for this command, so only the SIR id travels.
 type SpotLineageData struct {

contracts/ec2/v1/commands_test.go

@@ -24,6 +24,9 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 			SetInstanceTags:             true,
 			RemoveInstanceTags:          true,
 			SetInstanceMonitoring:       true,
+
+			ModifyInstanceMetadataOptions: true,
+			SetDisableApiTermination:      true,
 		},
 		AttachVolumeData:          &AttachVolumeData{VolumeID: "vol-1", Device: "/dev/sdf"},
 		DetachVolumeData:          &DetachVolumeData{VolumeID: "vol-2", Device: "/dev/sdg", Force: true},
@@ -33,6 +36,15 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 		SpotLineageData:           &SpotLineageData{SpotInstanceRequestId: "sir-1"},
 		InstanceTagsData:          &InstanceTagsData{Tags: map[string]string{"Name": "example"}, TagKeys: []string{"old"}},
 		InstanceMonitoringData:    &InstanceMonitoringData{Enabled: true},
+
+		InstanceMetadataOptionsData: &InstanceMetadataOptionsData{
+			HttpTokens:              "required",
+			HttpEndpoint:            "enabled",
+			HttpProtocolIpv6:        "disabled",
+			InstanceMetadataTags:    "enabled",
+			HttpPutResponseHopLimit: int64Ptr(3),
+		},
+		DisableApiTerminationData: &DisableApiTerminationData{Value: false},
 	}
 
 	data, err := json.Marshal(command)
@@ -52,7 +64,9 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 			"set_spot_lineage": true,
 			"set_instance_tags": true,
 			"remove_instance_tags": true,
-			"set_instance_monitoring": true
+			"set_instance_monitoring": true,
+			"modify_instance_metadata_options": true,
+			"set_disable_api_termination": true
 		},
 		"attach_volume_data": {"volume_id": "vol-1", "device": "/dev/sdf"},
 		"detach_volume_data": {"volume_id": "vol-2", "device": "/dev/sdg", "force": true},
@@ -61,10 +75,64 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 		"iam_profile_association_data": {"instance_profile_arn": "arn:aws:iam::123456789012:instance-profile/example"},
 		"spot_lineage_data": {"spot_instance_request_id": "sir-1"},
 		"instance_tags_data": {"tags": {"Name": "example"}, "tag_keys": ["old"]},
-		"instance_monitoring_data": {"enabled": true}
+		"instance_monitoring_data": {"enabled": true},
+		"instance_metadata_options_data": {
+			"http_tokens": "required",
+			"http_endpoint": "enabled",
+			"http_protocol_ipv6": "disabled",
+			"instance_metadata_tags": "enabled",
+			"http_put_response_hop_limit": 3
+		},
+		"disable_api_termination_data": {"value": false}
 	}`, string(data))
 }
 
+func int64Ptr(v int64) *int64 { return &v }
+
+// An owner that predates the metadata and termination-protection commands
+// sends and receives payloads without the new fields.
+func TestEC2InstanceCommandNewFieldsAreOptionalOnTheWire(t *testing.T) {
+	var n1 EC2InstanceCommand
+	require.NoError(t, json.Unmarshal([]byte(`{
+		"id": "i-0123456789abcdef0",
+		"attributes": {"stop_instance": true},
+		"instance_monitoring_data": {"enabled": true}
+	}`), &n1))
+	require.Equal(t, "i-0123456789abcdef0", n1.ID)
+	require.True(t, n1.Attributes.StopInstance)
+	require.False(t, n1.Attributes.ModifyInstanceMetadataOptions)
+	require.False(t, n1.Attributes.SetDisableApiTermination)
+	require.Nil(t, n1.InstanceMetadataOptionsData)
+	require.Nil(t, n1.DisableApiTerminationData)
+
+	unset, err := json.Marshal(EC2InstanceCommand{ID: "i-1", Attributes: EC2CommandAttributes{StopInstance: true}})
+	require.NoError(t, err)
+	require.NotContains(t, string(unset), "modify_instance_metadata_options")
+	require.NotContains(t, string(unset), "set_disable_api_termination")
+	require.NotContains(t, string(unset), "instance_metadata_options_data")
+	require.NotContains(t, string(unset), "disable_api_termination_data")
+}
+
+func TestInstanceMetadataOptionsDataOmitsUnsetFields(t *testing.T) {
+	data, err := json.Marshal(InstanceMetadataOptionsData{HttpPutResponseHopLimit: int64Ptr(1)})
+	require.NoError(t, err)
+	require.JSONEq(t, `{"http_put_response_hop_limit": 1}`, string(data))
+
+	data, err = json.Marshal(InstanceMetadataOptionsData{})
+	require.NoError(t, err)
+	require.JSONEq(t, `{}`, string(data))
+}
+
+func TestDisableApiTerminationDataKeepsExplicitFalse(t *testing.T) {
+	data, err := json.Marshal(DisableApiTerminationData{Value: false})
+	require.NoError(t, err)
+	require.JSONEq(t, `{"value": false}`, string(data))
+}
+
+func TestHandledHeaderName(t *testing.T) {
+	require.Equal(t, "Spinifex-Ec2-Cmd-Handled", HandledHeader)
+}
+
 func TestInstanceCommandSubjects(t *testing.T) {
 	require.Equal(t, "ec2.cmd.i-0123456789abcdef0", InstanceCommandSubject("i-0123456789abcdef0"))
 	require.Equal(t, "ec2.cmd.*", InstanceCommandSubjectWildcard)

contracts/lambda/v1/README.md

@@ -22,7 +22,15 @@ Tenant code may be able to reach the listener, but it cannot open the agent's co
 For each generation the executor creates ephemeral self-signed host and guest certificates whose single URI SAN is `spinifex-lambda://<Region>/<account>/<function ID>/<environment ID>/<generation>/<role>`, with role `host` or `guest`.
 Each end validates the peer certificate against the one pinned peer certificate and separately requires that URI SAN with the opposite role; hostname verification is not relied on.
 ALPN carries the L-GUEST major version, `spinifex-lambda-guest/1`, and an unsupported version fails the handshake. Builds with the `lambdatestlink` tag offer only `spinifex-lambda-testlink/2` on both host and guest, whose first frame carries the execution-role credential, so a test-link end and a release end fail the handshake; release builds are unchanged.
+In the same builds, the gateway reaches the host daemon on the local abstract socket `spinifex-lambda-testlink-control/3`, and each request carries the runtime-image digest the gateway was built with; the daemon refuses a missing or different digest before starting the attempt, and a gateway and daemon on different socket names never connect.
+On that socket a request's raw payload is at most 6 MiB, checked by the gateway before encoding and by the daemon after decoding.
+Its encoded frame is at most the base64 length of 6 MiB plus 64 KiB of metadata headroom and 64 bytes of field slack, and a receiver checks the frame length before allocating; metadata has no separate bound within the frame.
+The credential travels as its own frame of at most 8 KiB, and the reply has the same payload and frame bounds as the request.
+The daemon holds at most two connections at once, from accept to close, and leaves the rest unaccepted in the listen backlog.
 Under TLS 1.3 the guest's handshake can complete before the host refuses its certificate, so the guest treats the connection as accepted only after the host's first message.
+In the same builds, the function owner moves a created function with stored code from `Pending` to `Active` once this node's daemon serves the pinned image and the cluster configuration has exactly one node; a function with environment variables or a code copy over the test-link bound stays `Pending`, and `StateReason` names the unmet condition.
+Release builds never leave `Pending`.
+The Mulga deploy role installs a test-link build with `deploy_lambda_testlink=true`; this profile does not meet Q-199 fail-closed revocation, and its guests take management addresses whose record can expire (mulga-0hsub).
 
 **Bootstrap material:** the executor passes these fw_cfg entries under `opt/spinifex/lambda/`, and the agent reads them as root and unloads `qemu_fw_cfg` before user code starts.
 
@@ -36,6 +44,8 @@ Under TLS 1.3 the guest's handshake can complete before the host refuses its cer
 
 The guest's own interface settings come from `opt/spinifex/netcfg`, the entry other system VMs use.
 The agent unloads `qemu_fw_cfg` even when an entry is missing, and refuses to start if the fw_cfg tree is still readable afterwards.
+On a VMM without fw_cfg (the test-link Firecracker executor), the executor attaches the same entries as a tar archive on the second, writable virtio-blk disk, laid out as `by_name/<entry>/raw`.
+The runtime image's init extracts it to a root-only tmpfs at `/spinifex-cfg`, and the agent reads the entries there, then overwrites the whole disk with zeros and removes the tree; it refuses to start if the tree is still present.
 Until the message format exists, the agent authenticates and holds the connection but accepts no host message and starts no user code.
 
 **Authority:** one environment's attempt only.

docs/testing/e2e-coverage.md

@@ -108,6 +108,8 @@ Three entry points, in the order they run:
 
 **Its own suite and its own cell, for instancerecovery's reasons plus one.** It takes a node away, so anything sharing the environment reports that outage as its own failure; it needs its own VPC, public subnet and a backend placement no shared fixture describes; and a failure here has to read as "the load balancer did not survive its host" rather than as one more red subtest in a suite about load balancer features.
 
+**A dead data NIC is the other half, and there the state does change.** `LBSelfHealsADeadDataNIC` disables one data NIC of a running load balancer guest through a second QMP client and runs for an internet-facing NLB and an internal ALB. The backends and an unrelated customer instance in the same subnet are launched more than 21 minutes before the fault, so a repair that reached a customer instance would find one the platform has long stopped expecting to change. A state watch samples `DescribeLoadBalancers` every two seconds and requires the sequence `active`, `active_impaired`, `active` with nothing in between, while the internet-facing run also watches the name continuously and serves traffic through it within five minutes. Afterwards the guest, its interfaces and its address are unchanged but its process is not, the backends and the bystander keep their instance ID, launch time and process (ADR-0003:S7), and the daemon journals exactly one `ELBv2 data-path repair started` line carrying the fields an operator needs. The internal ALB has no client inside the VPC, so it asserts the state sequence and target health rather than traffic through its name.
+
 ### `iam` — everything IAM and STS
 
 Policy evaluation against live requests rather than unit-level: a deny applying to the next request without a new credential, resource-scoped grants naming one key pair, a user rename needing `iam:UpdateUser` on both the old and the new name, `aws:SourceIp` seeing the real client address, assumed-role sessions carrying role policies, and revocation stopping one key alone.
@@ -129,7 +131,7 @@ The client reports on its serial console because the runner has no route into th
 
 ### `eks`, `ecs`, `ecr` — container services
 
-`eks`: cluster create/delete, access entries, addon delivery, `GetToken`, kubeconfig artifacts, the EBS CSI volume path, an IRSA web-identity exchange under the stock AWS trust policy (`StringEquals` on the issuer host-path `:sub`/`:aud`, a mismatched `:sub` denied with `AccessDenied`, a `StringLike` `:sub` granted), and an IRSA pod whose spec names only a ServiceAccount with the role-arn annotation, where the pod-identity webhook injects the token volume, `AWS_*` environment, STS endpoint and CA bundle and the pod assumes the role, while a pod of an unannotated ServiceAccount is left untouched. The unmodified aws-load-balancer-controller chart 1.11.0 installs with the values an AWS user passes plus the gateway endpoint override, the Shield and WAF switches and the single-subnet feature gate, and reconciles an Ingress into an active ALB. Worker IMDS is covered too: a managed nodegroup worker reports IMDSv2 required with hop limit 2, a pod with no IRSA resolves to the node role through IMDS, and a nodegroup built from a launch template takes the template's metadata options. Control-plane isolation is covered as well: no control-plane host registers as a Node, a DaemonSet tolerating every taint runs on workers only, metrics-server is Ready, and logs work for every kube-system pod. `ecs`: clusters, container instances, the three network modes (`awsvpc`, `bridge`, `host`), task-role credentials fetched from the container credentials endpoint, a service behind an ELB, and ENI release on terminate. `ecr`: control-plane CRUD plus a real OCI data plane — docker login/build/push/pull round trips, multi-layer images, and raw blob HTTP.
+`eks`: cluster create/delete, access entries, addon delivery, `GetToken`, kubeconfig artifacts, the EBS CSI volume path, an IRSA web-identity exchange under the stock AWS trust policy (`StringEquals` on the issuer host-path `:sub`/`:aud`, a mismatched `:sub` denied with `AccessDenied`, a `StringLike` `:sub` granted), and an IRSA pod whose spec names only a ServiceAccount with the role-arn annotation, where the pod-identity webhook injects the token volume, `AWS_*` environment, STS endpoint and CA bundle and the pod assumes the role, while a pod of an unannotated ServiceAccount is left untouched. The unmodified aws-load-balancer-controller chart 1.11.0 installs with the values an AWS user passes plus the gateway endpoint override, the Shield and WAF switches and the single-subnet feature gate, and reconciles an Ingress into an active ALB. Worker IMDS is covered too: a managed nodegroup worker reports IMDSv2 required with hop limit 2, a pod with no IRSA resolves to the node role through IMDS, and a nodegroup built from a launch template takes the template's metadata options. The private-endpoint NIC of the cluster's endpoint NLB is disabled through QMP and every worker must be Ready again within ten minutes while the cluster stays `ACTIVE` and the NLB keeps its guest instance. Control-plane isolation is covered as well: no control-plane host registers as a Node, a DaemonSet tolerating every taint runs on workers only, metrics-server is Ready, and logs work for every kube-system pod. `ecs`: clusters, container instances, the three network modes (`awsvpc`, `bridge`, `host`), task-role credentials fetched from the container credentials endpoint, a service behind an ELB, and ENI release on terminate. `ecr`: control-plane CRUD plus a real OCI data plane — docker login/build/push/pull round trips, multi-layer images, and raw blob HTTP.
 
 ### `rds` — managed databases
 

scripts/build-microvm-image.sh

@@ -209,6 +209,7 @@ echo "[build-microvm-image] nginx: binary + stream module present"
 #   virtio*.ko*    — virtio bus, net, blk, rng, console, and helpers
 #   *fw_cfg*.ko*   — QEMU fw_cfg sysfs driver (delivers boot blobs)
 #   *failover*.ko* — net_failover + failover (virtio_net deps for live migration)
+#   af_packet.ko*  — AF_PACKET sockets (lb-agent's per-NIC gateway ARP probe)
 # All other loadable modules are removed. Drivers compiled into the kernel
 # (CONFIG_VIRTIO_MMIO=y etc.) are unaffected — they live in vmlinuz, not here.
 echo "[build-microvm-image] stripping kernel modules to virtio+fw_cfg only..."
@@ -222,6 +223,7 @@ for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
     find "$kernel_dir" -name "virtio*.ko.gz" -o -name "virtio*.ko" \
                        -o -name "*fw_cfg*.ko.gz" -o -name "*fw_cfg*.ko" \
                        -o -name "*failover*.ko.gz" -o -name "*failover*.ko" \
+                       -o -name "af_packet.ko.gz" -o -name "af_packet.ko" \
         2>/dev/null | while read -r mod; do
         rel="${mod#"$kernel_dir/"}"
         dest_dir="$tmp_keep/$(dirname "$rel")"
@@ -240,6 +242,15 @@ for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
     done)
     rm -rf "$tmp_keep"
 
+    # lb-agent's data-path probe needs AF_PACKET; without it every LB would
+    # report a dead NIC. Accept a kernel with CONFIG_PACKET built in.
+    if ! find "$kernel_dir" -name "af_packet.ko*" | grep -q .; then
+        if ! grep -q "CONFIG_PACKET=y" "$KERNEL_CONFIG" 2>/dev/null; then
+            echo "ERROR: af_packet.ko missing from the kept modules — lb-agent's data-path probe cannot open its socket" >&2
+            exit 1
+        fi
+    fi
+
     # Decompress kept modules to plain .ko. Alpine ships .ko.gz, but a host
     # kmod that lacks gzip support (e.g. Debian trixie's kmod 34) silently
     # produces an EMPTY modules.dep from compressed modules — the guest's

spinifex/agents/elbv2/agent.go

@@ -21,6 +21,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"sync"
 	"syscall"
 	"time"
 
@@ -94,6 +95,18 @@ type Agent struct {
 	statsFn func(socketPath string) ([]ServerStatus, error)
 	// For testing: override the active health prober (nginx/NLB).
 	probeFn func(targets []HealthTarget) []ServerStatus
+
+	// Data-path probe: netcfg and sysfs locations, the ARP exchange, the
+	// instance ID lookup and per-NIC miss counters (overridable in tests).
+	netcfgPath   string
+	sysNetDir    string
+	probeWait    time.Duration
+	arpProbeFn   arpProbeFn
+	instanceIDFn func() (string, error)
+	instanceID   string
+	idSkip       int
+	probeMu      sync.Mutex
+	probeStates  map[string]*nicProbeState
 }
 
 // New creates a new LB agent for the given load balancer.
@@ -150,6 +163,12 @@ func New(lbID, gatewayURL, accessKey, secretKey, region string) (*Agent, error)
 		reloadNginxFn: reloadNginx,
 		statsFn:       queryHAProxyStats,
 		probeFn:       probeHealthTargets,
+		netcfgPath:    netcfgPath,
+		sysNetDir:     sysClassNet,
+		probeWait:     probeWait,
+		arpProbeFn:    arpProbe,
+		instanceIDFn:  fetchInstanceID,
+		probeStates:   map[string]*nicProbeState{},
 	}, nil
 }
 
@@ -207,7 +226,7 @@ func (a *Agent) tick() {
 		}
 	}
 
-	resp, err := a.sendHeartbeat(servers)
+	resp, err := a.sendHeartbeat(servers, a.dataPathEvidence())
 	if err != nil {
 		// Include the dial target so the serial console disambiguates a
 		// transport failure (never reached AWSGW — "send request: dial ...")
@@ -288,7 +307,7 @@ func (a *Agent) enginePaths(engine string) (configPath, pidPath, certDir string,
 }
 
 // sendHeartbeat sends a heartbeat with health report to the gateway.
-func (a *Agent) sendHeartbeat(servers []ServerStatus) (*heartbeatResponse, error) {
+func (a *Agent) sendHeartbeat(servers []ServerStatus, ev dataPathEvidence) (*heartbeatResponse, error) {
 	params := url.Values{}
 	params.Set("Action", "LBAgentHeartbeat")
 	params.Set("Version", "2015-12-01")
@@ -300,6 +319,7 @@ func (a *Agent) sendHeartbeat(servers []ServerStatus) (*heartbeatResponse, error
 		params.Set("Servers.member."+idx+".Server", s.Server)
 		params.Set("Servers.member."+idx+".Status", s.Status)
 	}
+	ev.addTo(params)
 
 	body, err := a.signedPost(params)
 	if err != nil {

spinifex/agents/elbv2/arp_linux.go

@@ -0,0 +1,62 @@
+//go:build linux
+
+package elbv2
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"net/netip"
+	"syscall"
+	"time"
+)
+
+// arpProbe sends one ARP request for gateway out of iface on an AF_PACKET
+// socket bound to it and waits for the gateway's reply on the same socket.
+func arpProbe(iface string, mac net.HardwareAddr, src, gateway netip.Addr, wait time.Duration) error {
+	ifi, err := net.InterfaceByName(iface)
+	if err != nil {
+		return fmt.Errorf("interface %s: %w", iface, err)
+	}
+
+	proto := htons(syscall.ETH_P_ARP)
+	fd, err := syscall.Socket(syscall.AF_PACKET, syscall.SOCK_RAW, int(proto))
+	if err != nil {
+		return fmt.Errorf("packet socket: %w", err)
+	}
+	defer syscall.Close(fd)
+
+	addr := &syscall.SockaddrLinklayer{Protocol: proto, Ifindex: ifi.Index}
+	if err := syscall.Bind(fd, addr); err != nil {
+		return fmt.Errorf("bind %s: %w", iface, err)
+	}
+
+	deadline := time.Now().Add(wait)
+	if err := syscall.Sendto(fd, buildARPRequest(mac, src, gateway), 0, addr); err != nil {
+		return fmt.Errorf("send ARP request: %w", err)
+	}
+
+	buf := make([]byte, 128)
+	for {
+		left := time.Until(deadline)
+		if left <= 0 {
+			return errARPNoReply
+		}
+		tv := syscall.NsecToTimeval(int64(left))
+		if err := syscall.SetsockoptTimeval(fd, syscall.SOL_SOCKET, syscall.SO_RCVTIMEO, &tv); err != nil {
+			return fmt.Errorf("set receive deadline: %w", err)
+		}
+		n, _, err := syscall.Recvfrom(fd, buf, 0)
+		if err != nil {
+			if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EINTR) {
+				continue
+			}
+			return fmt.Errorf("receive ARP reply: %w", err)
+		}
+		if isARPReplyFrom(buf[:n], mac, gateway) {
+			return nil
+		}
+	}
+}
+
+func htons(v uint16) uint16 { return v<<8 | v>>8 }

spinifex/agents/elbv2/arp_other.go

@@ -0,0 +1,15 @@
+//go:build !linux
+
+package elbv2
+
+import (
+	"errors"
+	"net"
+	"net/netip"
+	"time"
+)
+
+// arpProbe is only meaningful inside the Linux guest.
+func arpProbe(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error {
+	return errors.New("ARP probe is only supported on linux")
+}

spinifex/agents/elbv2/datapath.go

@@ -0,0 +1,303 @@
+package elbv2
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"net"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+const (
+	// netcfgPath is the fw_cfg blob the microVM init script reads to configure NICs.
+	netcfgPath = "/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw"
+
+	sysClassNet = "/sys/class/net"
+
+	// probeFailureThreshold is the number of consecutive missed probes after
+	// which a NIC is reported as failing (about 15 s at the poll interval).
+	probeFailureThreshold = 3
+
+	// arpExchangeWait bounds one ARP exchange. probeWait bounds how long a tick
+	// waits for all probes, so a stuck probe never delays the heartbeat.
+	arpExchangeWait = time.Second
+	probeWait       = 1500 * time.Millisecond
+)
+
+// nicConfig is one NIC of the netcfg blob that has a gateway to probe.
+type nicConfig struct {
+	MAC     string
+	Addr    netip.Addr
+	Gateway netip.Addr
+}
+
+// NICReport is the per-NIC data-path evidence carried in the heartbeat.
+type NICReport struct {
+	MAC                 string
+	ProbeOK             bool
+	ConsecutiveFailures int64
+	RxPackets           int64
+	TxPackets           int64
+	RxDropped           int64
+	TxDropped           int64
+}
+
+// arpProbeFn sends one ARP request for gateway out of iface and returns nil
+// when the gateway's reply arrives within timeout.
+type arpProbeFn func(iface string, mac net.HardwareAddr, src, gateway netip.Addr, timeout time.Duration) error
+
+// parseNetcfg reads the KEY=value netcfg blob and returns the NICs that have a
+// gateway, in NIC order. The mgmt NIC carries no gateway and is skipped, as
+// is any NIC without a usable MAC, address or gateway.
+func parseNetcfg(blob []byte) []nicConfig {
+	type raw struct{ mac, cidr, gw string }
+	byIndex := map[int]*raw{}
+
+	sc := bufio.NewScanner(bytes.NewReader(blob))
+	for sc.Scan() {
+		line := strings.TrimSpace(sc.Text())
+		key, val, ok := strings.Cut(line, "=")
+		if !ok || !strings.HasPrefix(key, "NIC") {
+			continue
+		}
+		idxStr, field, ok := strings.Cut(strings.TrimPrefix(key, "NIC"), "_")
+		if !ok {
+			continue
+		}
+		idx, err := strconv.Atoi(idxStr)
+		if err != nil {
+			continue
+		}
+		r := byIndex[idx]
+		if r == nil {
+			r = &raw{}
+			byIndex[idx] = r
+		}
+		switch field {
+		case "MAC":
+			r.mac = strings.ToLower(val)
+		case "CIDR":
+			r.cidr = val
+		case "GW":
+			r.gw = val
+		}
+	}
+
+	idxs := make([]int, 0, len(byIndex))
+	for i := range byIndex {
+		idxs = append(idxs, i)
+	}
+	sort.Ints(idxs)
+
+	var out []nicConfig
+	for _, i := range idxs {
+		r := byIndex[i]
+		if r.mac == "" || r.cidr == "" || r.gw == "" {
+			continue
+		}
+		prefix, err := netip.ParsePrefix(r.cidr)
+		if err != nil || !prefix.Addr().Is4() {
+			continue
+		}
+		gw, err := netip.ParseAddr(r.gw)
+		if err != nil || !gw.Is4() {
+			continue
+		}
+		out = append(out, nicConfig{MAC: r.mac, Addr: prefix.Addr(), Gateway: gw})
+	}
+	return out
+}
+
+// ifaceByMAC returns the interface under sysNetDir whose address is mac.
+func ifaceByMAC(sysNetDir, mac string) (string, error) {
+	entries, err := os.ReadDir(sysNetDir)
+	if err != nil {
+		return "", err
+	}
+	for _, e := range entries {
+		b, err := os.ReadFile(filepath.Join(sysNetDir, e.Name(), "address"))
+		if err != nil {
+			continue
+		}
+		if strings.EqualFold(strings.TrimSpace(string(b)), mac) {
+			return e.Name(), nil
+		}
+	}
+	return "", fmt.Errorf("no interface with MAC %s", mac)
+}
+
+// readCounters fills the sysfs statistics of iface into r.
+func readCounters(sysNetDir, iface string, r *NICReport) {
+	read := func(name string) int64 {
+		b, err := os.ReadFile(filepath.Join(sysNetDir, iface, "statistics", name))
+		if err != nil {
+			return 0
+		}
+		v, _ := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64)
+		return v
+	}
+	r.RxPackets = read("rx_packets")
+	r.TxPackets = read("tx_packets")
+	r.RxDropped = read("rx_dropped")
+	r.TxDropped = read("tx_dropped")
+}
+
+var errARPNoReply = errors.New("no ARP reply from the gateway")
+
+// buildARPRequest returns the Ethernet frame asking who has target.
+func buildARPRequest(mac net.HardwareAddr, src, target netip.Addr) []byte {
+	f := make([]byte, 0, 42)
+	f = append(f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff)
+	f = append(f, mac...)
+	f = binary.BigEndian.AppendUint16(f, 0x0806)
+	f = binary.BigEndian.AppendUint16(f, 1)      // Ethernet
+	f = binary.BigEndian.AppendUint16(f, 0x0800) // IPv4
+	f = append(f, 6, 4)
+	f = binary.BigEndian.AppendUint16(f, 1) // request
+	f = append(f, mac...)
+	s := src.As4()
+	f = append(f, s[:]...)
+	f = append(f, 0, 0, 0, 0, 0, 0)
+	t := target.As4()
+	f = append(f, t[:]...)
+	return f
+}
+
+// isARPReplyFrom reports whether frame is an ARP reply sent by sender to mac.
+func isARPReplyFrom(frame []byte, mac net.HardwareAddr, sender netip.Addr) bool {
+	if len(frame) < 42 || binary.BigEndian.Uint16(frame[12:14]) != 0x0806 {
+		return false
+	}
+	if binary.BigEndian.Uint16(frame[20:22]) != 2 || !bytes.Equal(frame[32:38], mac) {
+		return false
+	}
+	s := sender.As4()
+	return bytes.Equal(frame[28:32], s[:])
+}
+
+// nicProbeState carries the miss counter of one NIC across ticks. A probe that
+// has not returned is still in flight, and counts as a miss without starting
+// another goroutine behind it.
+type nicProbeState struct {
+	mu       sync.Mutex
+	failures int64
+	inflight bool
+}
+
+type probeResult struct {
+	ok  bool
+	set bool
+}
+
+// probeNICs probes every gateway-bearing data NIC once, in parallel, and
+// returns within probeWait however long a probe blocks.
+func (a *Agent) probeNICs() []NICReport {
+	blob, err := os.ReadFile(a.netcfgPath)
+	if err != nil {
+		return nil
+	}
+	nics := parseNetcfg(blob)
+	if len(nics) == 0 {
+		return nil
+	}
+
+	reports := make([]NICReport, len(nics))
+	results := make([]chan probeResult, len(nics))
+	for i, n := range nics {
+		results[i] = make(chan probeResult, 1)
+		st := a.probeState(n.MAC)
+
+		st.mu.Lock()
+		busy := st.inflight
+		if !busy {
+			st.inflight = true
+		}
+		st.mu.Unlock()
+		if busy {
+			results[i] <- probeResult{set: true}
+			continue
+		}
+
+		go func(n nicConfig, st *nicProbeState, out chan<- probeResult) {
+			ok := a.probeOne(n)
+			st.mu.Lock()
+			st.inflight = false
+			st.mu.Unlock()
+			out <- probeResult{ok: ok, set: true}
+		}(n, st, results[i])
+	}
+
+	deadline := time.NewTimer(a.probeWait)
+	defer deadline.Stop()
+	expired := false
+	for i, n := range nics {
+		var res probeResult
+		if expired {
+			select {
+			case res = <-results[i]:
+			default:
+			}
+		} else {
+			select {
+			case res = <-results[i]:
+			case <-deadline.C:
+				// Out of time: an unfinished probe is a miss for this tick.
+				expired = true
+			}
+		}
+
+		st := a.probeState(n.MAC)
+		st.mu.Lock()
+		if res.ok {
+			st.failures = 0
+		} else {
+			st.failures++
+		}
+		failures := st.failures
+		st.mu.Unlock()
+
+		r := NICReport{
+			MAC:                 n.MAC,
+			ProbeOK:             failures < probeFailureThreshold,
+			ConsecutiveFailures: failures,
+		}
+		if iface, err := ifaceByMAC(a.sysNetDir, n.MAC); err == nil {
+			readCounters(a.sysNetDir, iface, &r)
+		}
+		reports[i] = r
+	}
+	return reports
+}
+
+// probeOne runs one ARP exchange for n, false on any failure.
+func (a *Agent) probeOne(n nicConfig) bool {
+	iface, err := ifaceByMAC(a.sysNetDir, n.MAC)
+	if err != nil {
+		return false
+	}
+	hw, err := net.ParseMAC(n.MAC)
+	if err != nil {
+		return false
+	}
+	return a.arpProbeFn(iface, hw, n.Addr, n.Gateway, arpExchangeWait) == nil
+}
+
+func (a *Agent) probeState(mac string) *nicProbeState {
+	a.probeMu.Lock()
+	defer a.probeMu.Unlock()
+	st := a.probeStates[mac]
+	if st == nil {
+		st = &nicProbeState{}
+		a.probeStates[mac] = st
+	}
+	return st
+}

spinifex/agents/elbv2/datapath_test.go

@@ -0,0 +1,484 @@
+package elbv2
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/netip"
+	"net/url"
+	"os"
+	"path/filepath"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+const (
+	testDataMAC = "02:00:00:aa:00:01"
+	testMgmtMAC = "02:00:00:aa:00:02"
+	testNetcfg  = "NIC0_MAC=" + testDataMAC + "\nNIC0_CIDR=10.0.1.5/24\nNIC0_GW=10.0.1.1\nNIC0_DEFAULT=1\n" +
+		"NIC1_MAC=" + testMgmtMAC + "\nNIC1_CIDR=10.255.0.9/24\nNIC1_DEFAULT=0\n"
+)
+
+// datapathAgent returns a test agent whose netcfg and sysfs live in a temp dir.
+func datapathAgent(t *testing.T, gwURL, netcfg string) *Agent {
+	t.Helper()
+	a := newTestAgent(t, gwURL)
+	dir := t.TempDir()
+
+	a.netcfgPath = filepath.Join(dir, "netcfg")
+	if err := os.WriteFile(a.netcfgPath, []byte(netcfg), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	a.sysNetDir = filepath.Join(dir, "net")
+	for name, mac := range map[string]string{"ens3": testDataMAC, "ens4": testMgmtMAC} {
+		stats := filepath.Join(a.sysNetDir, name, "statistics")
+		if err := os.MkdirAll(stats, 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(filepath.Join(a.sysNetDir, name, "address"), []byte(mac+"\n"), 0o600); err != nil {
+			t.Fatal(err)
+		}
+		for file, v := range map[string]string{"rx_packets": "100", "tx_packets": "200", "rx_dropped": "3", "tx_dropped": "4"} {
+			if err := os.WriteFile(filepath.Join(stats, file), []byte(v+"\n"), 0o600); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+
+	a.instanceIDFn = func() (string, error) { return "i-0123456789abcdef0", nil }
+	return a
+}
+
+func TestParseNetcfg_SkipsMgmtNICAndMissingGateway(t *testing.T) {
+	blob := testNetcfg +
+		"NIC2_MAC=02:00:00:aa:00:03\nNIC2_CIDR=10.0.2.5/24\nNIC2_GW=10.0.2.1\nNIC2_DEFAULT=0\n" +
+		"NIC3_MAC=02:00:00:aa:00:04\nNIC3_DHCP=1\nNIC3_DEFAULT=0\n" +
+		"NIC4_CIDR=10.0.4.5/24\nNIC4_GW=10.0.4.1\n" +
+		"NIC5_MAC=02:00:00:aa:00:06\nNIC5_CIDR=not-a-cidr\nNIC5_GW=10.0.5.1\n"
+
+	got := parseNetcfg([]byte(blob))
+	if len(got) != 2 {
+		t.Fatalf("got %d NICs, want the 2 with a gateway: %+v", len(got), got)
+	}
+	if got[0].MAC != testDataMAC || got[0].Gateway != netip.MustParseAddr("10.0.1.1") || got[0].Addr != netip.MustParseAddr("10.0.1.5") {
+		t.Errorf("first NIC = %+v", got[0])
+	}
+	if got[1].MAC != "02:00:00:aa:00:03" || got[1].Gateway != netip.MustParseAddr("10.0.2.1") {
+		t.Errorf("second NIC = %+v", got[1])
+	}
+}
+
+func TestParseNetcfg_OrdersByNICIndexAndLowercasesMAC(t *testing.T) {
+	blob := "NIC10_MAC=02:00:00:AA:00:0A\nNIC10_CIDR=10.0.9.5/24\nNIC10_GW=10.0.9.1\n" +
+		"NIC2_MAC=02:00:00:AA:00:02\nNIC2_CIDR=10.0.2.5/24\nNIC2_GW=10.0.2.1\n"
+	got := parseNetcfg([]byte(blob))
+	if len(got) != 2 || got[0].MAC != "02:00:00:aa:00:02" || got[1].MAC != "02:00:00:aa:00:0a" {
+		t.Fatalf("order or case wrong: %+v", got)
+	}
+}
+
+func TestIfaceByMACAndCounters(t *testing.T) {
+	a := datapathAgent(t, "http://unused", testNetcfg)
+
+	iface, err := ifaceByMAC(a.sysNetDir, "02:00:00:AA:00:01")
+	if err != nil || iface != "ens3" {
+		t.Fatalf("iface = %q, err = %v", iface, err)
+	}
+	if _, err := ifaceByMAC(a.sysNetDir, "02:00:00:ff:ff:ff"); err == nil {
+		t.Error("an unknown MAC must not resolve to an interface")
+	}
+
+	var r NICReport
+	readCounters(a.sysNetDir, iface, &r)
+	if r.RxPackets != 100 || r.TxPackets != 200 || r.RxDropped != 3 || r.TxDropped != 4 {
+		t.Errorf("counters = %+v", r)
+	}
+}
+
+func TestARPFrames_RequestShapeAndReplyMatching(t *testing.T) {
+	mac, _ := net.ParseMAC(testDataMAC)
+	src, gw := netip.MustParseAddr("10.0.1.5"), netip.MustParseAddr("10.0.1.1")
+
+	req := buildARPRequest(mac, src, gw)
+	if len(req) != 42 || req[12] != 0x08 || req[13] != 0x06 || req[21] != 1 {
+		t.Fatalf("request frame malformed: % x", req)
+	}
+	if string(req[38:42]) != string(gw.AsSlice()) {
+		t.Errorf("target IP not the gateway: % x", req[38:42])
+	}
+
+	reply := make([]byte, 42)
+	copy(reply[0:6], mac)
+	reply[12], reply[13] = 0x08, 0x06
+	reply[21] = 2
+	copy(reply[28:32], gw.AsSlice())
+	copy(reply[32:38], mac)
+	if !isARPReplyFrom(reply, mac, gw) {
+		t.Error("a reply from the gateway to our MAC must match")
+	}
+	if isARPReplyFrom(reply, mac, netip.MustParseAddr("10.0.1.99")) {
+		t.Error("a reply from another host must not match")
+	}
+	reply[21] = 1
+	if isARPReplyFrom(reply, mac, gw) {
+		t.Error("a request must not count as a reply")
+	}
+	if isARPReplyFrom(reply[:20], mac, gw) {
+		t.Error("a short frame must not match")
+	}
+}
+
+func TestProbeNICs_ProbesOnlyTheGatewayNIC(t *testing.T) {
+	a := datapathAgent(t, "http://unused", testNetcfg)
+	var mu sync.Mutex
+	var probed []string
+	a.arpProbeFn = func(iface string, _ net.HardwareAddr, src, gw netip.Addr, _ time.Duration) error {
+		mu.Lock()
+		defer mu.Unlock()
+		probed = append(probed, fmt.Sprintf("%s %s>%s", iface, src, gw))
+		return nil
+	}
+
+	reports := a.probeNICs()
+	if len(reports) != 1 || reports[0].MAC != testDataMAC || !reports[0].ProbeOK || reports[0].ConsecutiveFailures != 0 {
+		t.Fatalf("reports = %+v", reports)
+	}
+	if len(probed) != 1 || probed[0] != "ens3 10.0.1.5>10.0.1.1" {
+		t.Errorf("probed = %v, want only the data NIC", probed)
+	}
+	if reports[0].RxPackets != 100 || reports[0].TxDropped != 4 {
+		t.Errorf("counters missing from report: %+v", reports[0])
+	}
+}
+
+func TestProbeNICs_TwoMissesThenSuccessNeverReportsFailing(t *testing.T) {
+	a := datapathAgent(t, "http://unused", testNetcfg)
+	results := []error{errARPNoReply, errARPNoReply, nil, errARPNoReply, errARPNoReply, nil}
+	var n atomic.Int32
+	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error {
+		return results[n.Add(1)-1]
+	}
+
+	for i, want := range []int64{1, 2, 0, 1, 2, 0} {
+		r := a.probeNICs()
+		if len(r) != 1 || !r[0].ProbeOK || r[0].ConsecutiveFailures != want {
+			t.Fatalf("tick %d: report = %+v, want ok with %d failures", i, r, want)
+		}
+	}
+}
+
+func TestProbeNICs_ThreeMissesReportFailingAndRecoveryClears(t *testing.T) {
+	a := datapathAgent(t, "http://unused", testNetcfg)
+	var fail atomic.Bool
+	fail.Store(true)
+	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error {
+		if fail.Load() {
+			return errARPNoReply
+		}
+		return nil
+	}
+
+	for i := 1; i <= 4; i++ {
+		r := a.probeNICs()
+		wantOK := i < 3 // failing from the third consecutive miss
+		if r[0].ProbeOK != wantOK || r[0].ConsecutiveFailures != int64(i) {
+			t.Fatalf("miss %d: report = %+v, want ProbeOK=%v", i, r[0], wantOK)
+		}
+	}
+
+	fail.Store(false)
+	if r := a.probeNICs(); !r[0].ProbeOK || r[0].ConsecutiveFailures != 0 {
+		t.Errorf("a successful probe must clear the failure: %+v", r[0])
+	}
+}
+
+func TestProbeNICs_BlockedProbeIsBoundedAndCountsAsMiss(t *testing.T) {
+	a := datapathAgent(t, "http://unused", testNetcfg)
+	a.probeWait = 50 * time.Millisecond
+	release := make(chan struct{})
+	defer close(release)
+	var calls atomic.Int32
+	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error {
+		calls.Add(1)
+		<-release
+		return nil
+	}
+
+	for i := 1; i <= 3; i++ {
+		start := time.Now()
+		r := a.probeNICs()
+		if d := time.Since(start); d > time.Second {
+			t.Fatalf("tick %d waited %s on a stuck probe", i, d)
+		}
+		if len(r) != 1 || r[0].ConsecutiveFailures != int64(i) {
+			t.Fatalf("tick %d: report = %+v, want %d misses", i, r, i)
+		}
+	}
+	if got := calls.Load(); got != 1 {
+		t.Errorf("a stuck probe was started %d times, want 1 (no goroutine pile-up)", got)
+	}
+}
+
+func TestTick_BlockedProbeDoesNotDelayHeartbeatBeyondOneTick(t *testing.T) {
+	var got atomic.Pointer[url.Values]
+	gw := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_ = r.ParseForm()
+		if r.FormValue("Action") == "LBAgentHeartbeat" {
+			v := r.Form
+			got.Store(&v)
+		}
+		fmt.Fprint(w, `<LBAgentHeartbeatResponse><LBAgentHeartbeatResult><Status>active</Status><ConfigHash>h1</ConfigHash></LBAgentHeartbeatResult></LBAgentHeartbeatResponse>`)
+	}))
+	defer gw.Close()
+
+	a := datapathAgent(t, gw.URL, testNetcfg)
+	a.localConfigHash = "h1"
+	a.probeWait = 50 * time.Millisecond
+	release := make(chan struct{})
+	defer close(release)
+	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error {
+		<-release
+		return nil
+	}
+
+	start := time.Now()
+	a.tick()
+	if d := time.Since(start); d >= pollInterval {
+		t.Fatalf("tick took %s with a stuck probe, want under one %s tick", d, pollInterval)
+	}
+	if got.Load() == nil {
+		t.Fatal("the heartbeat was never sent")
+	}
+}
+
+func TestHeartbeat_EncodesEvidenceWhenPresent(t *testing.T) {
+	var got atomic.Pointer[url.Values]
+	gw := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_ = r.ParseForm()
+		v := r.Form
+		got.Store(&v)
+		fmt.Fprint(w, `<LBAgentHeartbeatResponse><LBAgentHeartbeatResult><Status>active</Status><ConfigHash>h1</ConfigHash></LBAgentHeartbeatResult></LBAgentHeartbeatResponse>`)
+	}))
+	defer gw.Close()
+
+	a := datapathAgent(t, gw.URL, testNetcfg)
+	a.localConfigHash = "h1"
+	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error { return errARPNoReply }
+
+	a.tick()
+	v := got.Load()
+	if v == nil {
+		t.Fatal("no heartbeat received")
+	}
+	want := map[string]string{
+		"InstanceID":                        "i-0123456789abcdef0",
+		"NICs.member.1.MAC":                 testDataMAC,
+		"NICs.member.1.ProbeOK":             "true",
+		"NICs.member.1.ConsecutiveFailures": "1",
+		"NICs.member.1.RxPackets":           "100",
+		"NICs.member.1.TxPackets":           "200",
+		"NICs.member.1.RxDropped":           "3",
+		"NICs.member.1.TxDropped":           "4",
+	}
+	for k, w := range want {
+		if g := v.Get(k); g != w {
+			t.Errorf("%s = %q, want %q", k, g, w)
+		}
+	}
+	if v.Get("NICs.member.2.MAC") != "" {
+		t.Error("the mgmt NIC must not be reported")
+	}
+}
+
+func TestHeartbeat_OmitsEvidenceWithoutNetcfg(t *testing.T) {
+	var got atomic.Pointer[url.Values]
+	gw := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_ = r.ParseForm()
+		v := r.Form
+		got.Store(&v)
+		fmt.Fprint(w, `<LBAgentHeartbeatResponse><LBAgentHeartbeatResult><Status>active</Status><ConfigHash>h1</ConfigHash></LBAgentHeartbeatResult></LBAgentHeartbeatResponse>`)
+	}))
+	defer gw.Close()
+
+	a := datapathAgent(t, gw.URL, testNetcfg)
+	a.localConfigHash = "h1"
+	a.netcfgPath = filepath.Join(t.TempDir(), "absent")
+	a.instanceIDFn = func() (string, error) { return "", errors.New("must not be asked without NICs") }
+
+	a.tick()
+	v := got.Load()
+	if v == nil {
+		t.Fatal("no heartbeat received")
+	}
+	for k := range *v {
+		if k == "InstanceID" || len(k) >= 4 && k[:4] == "NICs" {
+			t.Errorf("unexpected evidence parameter %s", k)
+		}
+	}
+}
+
+func TestDataPathEvidence_InstanceIDRetriedUntilKnown(t *testing.T) {
+	a := datapathAgent(t, "http://unused", testNetcfg)
+	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error { return nil }
+	var calls atomic.Int32
+	a.instanceIDFn = func() (string, error) {
+		if calls.Add(1) < 3 {
+			return "", errors.New("metadata not ready")
+		}
+		return "i-late", nil
+	}
+
+	// A failure backs the lookup off for imdsRetryTicks-1 ticks.
+	ticks := func(n int) string {
+		var id string
+		for range n {
+			id = a.dataPathEvidence().InstanceID
+		}
+		return id
+	}
+	if got := ticks(imdsRetryTicks); got != "" || calls.Load() != 1 {
+		t.Fatalf("after %d ticks: id %q, calls %d; want no id and one lookup", imdsRetryTicks, got, calls.Load())
+	}
+	if got := ticks(imdsRetryTicks); got != "" || calls.Load() != 2 {
+		t.Fatalf("second window: id %q, calls %d; want no id and two lookups", got, calls.Load())
+	}
+	if got := ticks(imdsRetryTicks); got != "i-late" || calls.Load() != 3 {
+		t.Fatalf("third window: id %q, calls %d; want i-late and three lookups", got, calls.Load())
+	}
+	ticks(imdsRetryTicks)
+	if calls.Load() != 3 {
+		t.Errorf("metadata asked %d times, want 3 (stop once known)", calls.Load())
+	}
+}
+
+func TestTick_BlockedInstanceIDLookupDoesNotDelayHeartbeat(t *testing.T) {
+	var got atomic.Pointer[url.Values]
+	gw := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_ = r.ParseForm()
+		if r.FormValue("Action") == "LBAgentHeartbeat" {
+			v := r.Form
+			got.Store(&v)
+		}
+		fmt.Fprint(w, `<LBAgentHeartbeatResponse><LBAgentHeartbeatResult><Status>active</Status><ConfigHash>h1</ConfigHash></LBAgentHeartbeatResult></LBAgentHeartbeatResponse>`)
+	}))
+	defer gw.Close()
+
+	a := datapathAgent(t, gw.URL, testNetcfg)
+	a.localConfigHash = "h1"
+	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error { return nil }
+	release := make(chan struct{})
+	defer close(release)
+	var calls atomic.Int32
+	a.instanceIDFn = func() (string, error) {
+		calls.Add(1)
+		<-release
+		return "", errors.New("unreachable")
+	}
+
+	for i := range 3 {
+		start := time.Now()
+		a.tick()
+		if d := time.Since(start); d >= pollInterval {
+			t.Fatalf("tick %d took %s with IMDS blocked, want under one %s tick", i, d, pollInterval)
+		}
+	}
+	v := got.Load()
+	if v == nil {
+		t.Fatal("the heartbeat was never sent")
+	}
+	if v.Get("InstanceID") != "" {
+		t.Error("InstanceID must be omitted while the lookup fails")
+	}
+	if v.Get("NICs.member.1.MAC") == "" {
+		t.Error("NIC evidence must still be sent")
+	}
+	if calls.Load() != 1 {
+		t.Errorf("blocked lookup started %d times in 3 ticks, want 1 (backoff)", calls.Load())
+	}
+}
+
+func TestFetchInstanceID_SharesOneDeadline(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		time.Sleep(3 * imdsWait)
+	}))
+	defer srv.Close()
+	old := imdsBase
+	imdsBase = srv.URL + "/latest"
+	defer func() { imdsBase = old }()
+
+	start := time.Now()
+	if _, err := fetchInstanceID(); err == nil {
+		t.Fatal("a hung metadata service must fail the lookup")
+	}
+	if d := time.Since(start); d > imdsWait+imdsWait/2 {
+		t.Fatalf("lookup took %s, want about one %s deadline", d, imdsWait)
+	}
+}
+
+func TestARPProbe_NoResponderFailsClosed(t *testing.T) {
+	mac, _ := net.ParseMAC(testDataMAC)
+	// Nobody answers an ARP request sent on loopback, so a probe that returned
+	// nil here would mask a dead data path. It fails either on the socket (no
+	// CAP_NET_RAW) or on the missing reply.
+	err := arpProbe("lo", mac, netip.MustParseAddr("10.9.9.2"), netip.MustParseAddr("10.9.9.1"), 100*time.Millisecond)
+	if err == nil {
+		t.Fatal("a probe with no responder must fail")
+	}
+	if err := arpProbe("no-such-iface0", mac, netip.MustParseAddr("10.9.9.2"), netip.MustParseAddr("10.9.9.1"), 100*time.Millisecond); err == nil {
+		t.Fatal("a probe on a missing interface must fail")
+	}
+}
+
+func TestFetchInstanceID(t *testing.T) {
+	serve := func(tokenStatus, idStatus int, wantToken string) *httptest.Server {
+		return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			switch {
+			case r.Method == http.MethodPut && r.URL.Path == "/latest/api/token":
+				w.WriteHeader(tokenStatus)
+				fmt.Fprint(w, "tok123")
+			case r.URL.Path == "/latest/meta-data/instance-id":
+				if got := r.Header.Get("X-Aws-Ec2-Metadata-Token"); got != wantToken {
+					t.Errorf("token header = %q, want %q", got, wantToken)
+				}
+				w.WriteHeader(idStatus)
+				fmt.Fprint(w, " i-abc123 \n")
+			default:
+				http.NotFound(w, r)
+			}
+		}))
+	}
+	saved := imdsBase
+	defer func() { imdsBase = saved }()
+
+	for _, tc := range []struct {
+		name                  string
+		tokenStatus, idStatus int
+		wantToken             string
+		wantID                string
+		wantErr               bool
+	}{
+		{"token then id", http.StatusOK, http.StatusOK, "tok123", "i-abc123", false},
+		{"token refused falls back to v1", http.StatusNotFound, http.StatusOK, "", "i-abc123", false},
+		{"id endpoint error", http.StatusOK, http.StatusServiceUnavailable, "tok123", "", true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			srv := serve(tc.tokenStatus, tc.idStatus, tc.wantToken)
+			defer srv.Close()
+			imdsBase = srv.URL + "/latest"
+			id, err := fetchInstanceID()
+			if (err != nil) != tc.wantErr || id != tc.wantID {
+				t.Fatalf("id = %q, err = %v", id, err)
+			}
+		})
+	}
+
+	imdsBase = "http://127.0.0.1:1/latest"
+	if _, err := fetchInstanceID(); err == nil {
+		t.Error("an unreachable metadata service must fail")
+	}
+}

spinifex/agents/elbv2/evidence.go

@@ -0,0 +1,133 @@
+package elbv2
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	imdsWait       = time.Second
+	imdsRetryTicks = 6
+)
+
+// imdsBase is the in-VM metadata endpoint (a variable so tests can point it at a fake).
+var imdsBase = "http://169.254.169.254/latest"
+
+// dataPathEvidence is the optional part of a heartbeat: which VM the agent runs
+// in and what each data NIC's gateway probe saw. An empty value adds nothing,
+// which is what an agent without a netcfg blob sends.
+type dataPathEvidence struct {
+	InstanceID string
+	NICs       []NICReport
+}
+
+// addTo encodes the evidence as additional heartbeat query parameters.
+func (e dataPathEvidence) addTo(params url.Values) {
+	if e.InstanceID != "" {
+		params.Set("InstanceID", e.InstanceID)
+	}
+	for i, n := range e.NICs {
+		p := "NICs.member." + strconv.Itoa(i+1) + "."
+		params.Set(p+"MAC", n.MAC)
+		params.Set(p+"ProbeOK", strconv.FormatBool(n.ProbeOK))
+		params.Set(p+"ConsecutiveFailures", strconv.FormatInt(n.ConsecutiveFailures, 10))
+		params.Set(p+"RxPackets", strconv.FormatInt(n.RxPackets, 10))
+		params.Set(p+"TxPackets", strconv.FormatInt(n.TxPackets, 10))
+		params.Set(p+"RxDropped", strconv.FormatInt(n.RxDropped, 10))
+		params.Set(p+"TxDropped", strconv.FormatInt(n.TxDropped, 10))
+	}
+}
+
+// dataPathEvidence gathers the probe results and instance ID for one heartbeat.
+// Neither step can fail the heartbeat: missing evidence is simply omitted.
+func (a *Agent) dataPathEvidence() dataPathEvidence {
+	nics := a.probeNICs()
+	if len(nics) == 0 {
+		return dataPathEvidence{}
+	}
+	if a.instanceID == "" {
+		a.lookupInstanceID()
+	}
+	return dataPathEvidence{InstanceID: a.instanceID, NICs: nics}
+}
+
+// lookupInstanceID asks for the instance ID without ever holding up a tick: the
+// call is bounded by imdsWait and, after a failure, retried only every
+// imdsRetryTicks ticks.
+func (a *Agent) lookupInstanceID() {
+	if a.idSkip > 0 {
+		a.idSkip--
+		return
+	}
+	type result struct {
+		id  string
+		err error
+	}
+	ch := make(chan result, 1)
+	go func() {
+		id, err := a.instanceIDFn()
+		ch <- result{id, err}
+	}()
+	timer := time.NewTimer(imdsWait)
+	defer timer.Stop()
+	select {
+	case r := <-ch:
+		if r.err == nil && r.id != "" {
+			a.instanceID = r.id
+			return
+		}
+	case <-timer.C:
+	}
+	a.idSkip = imdsRetryTicks - 1
+}
+
+// fetchInstanceID reads the instance ID from the in-VM metadata service. The
+// token request and the read share one deadline.
+func fetchInstanceID() (string, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), imdsWait)
+	defer cancel()
+	c := &http.Client{}
+
+	var token string
+	if req, err := http.NewRequestWithContext(ctx, http.MethodPut, imdsBase+"/api/token", nil); err == nil {
+		req.Header.Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "60")
+		if resp, err := c.Do(req); err == nil {
+			b, _ := io.ReadAll(io.LimitReader(resp.Body, 256))
+			resp.Body.Close()
+			if resp.StatusCode == http.StatusOK {
+				token = strings.TrimSpace(string(b))
+			}
+		}
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, imdsBase+"/meta-data/instance-id", nil)
+	if err != nil {
+		return "", err
+	}
+	if token != "" {
+		req.Header.Set("X-Aws-Ec2-Metadata-Token", token)
+	}
+	resp, err := c.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("instance-id: status %d", resp.StatusCode)
+	}
+	b, err := io.ReadAll(io.LimitReader(resp.Body, 256))
+	if err != nil {
+		return "", err
+	}
+	id := strings.TrimSpace(string(b))
+	if id == "" {
+		return "", fmt.Errorf("instance-id: empty")
+	}
+	return id, nil
+}

spinifex/agents/lambda/hostlink/configdisk.go

@@ -0,0 +1,71 @@
+package hostlink
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+
+	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
+	"golang.org/x/sys/unix"
+)
+
+// On a VMM without fw_cfg the executor attaches the same entries as a tar
+// archive on a second disk, which the runtime image's init extracts here in
+// the fw_cfg sysfs layout, so the fw_cfg readers work unchanged.
+const (
+	ConfigDiskRoot   = "/spinifex-cfg"
+	ConfigDiskDir    = ConfigDiskRoot + "/by_name/" + lambdav1.FwCfgPrefix
+	ConfigDiskDevice = "/dev/vdb"
+)
+
+// zeroChunk bounds each write while erasing the configuration disk.
+const zeroChunk = 64 << 10
+
+// unmount is a var so tests without privilege can stand in for the tmpfs.
+var unmount = unix.Unmount
+
+// SealConfigDisk overwrites the whole configuration disk with zeros, then
+// removes the extracted tree. The tree stays if the disk cannot be erased, so
+// SealFwCfg's check on root fails closed.
+func SealConfigDisk(device, root string) error {
+	if err := zeroDevice(device); err != nil {
+		return fmt.Errorf("hostlink: erase configuration disk: %w", err)
+	}
+	if err := unmount(root, 0); err != nil && !errors.Is(err, unix.EINVAL) {
+		return fmt.Errorf("hostlink: unmount %s: %w", root, err)
+	}
+	if err := os.RemoveAll(root); err != nil {
+		return fmt.Errorf("hostlink: remove %s: %w", root, err)
+	}
+	return nil
+}
+
+func zeroDevice(device string) error {
+	f, err := os.OpenFile(device, os.O_WRONLY, 0)
+	if err != nil {
+		return err
+	}
+	size, err := f.Seek(0, io.SeekEnd)
+	if err == nil {
+		_, err = f.Seek(0, io.SeekStart)
+	}
+	if err != nil {
+		_ = f.Close()
+		return err
+	}
+	zeros := make([]byte, zeroChunk)
+	for left := size; left > 0; {
+		n := min(left, int64(len(zeros)))
+		if _, err := f.Write(zeros[:n]); err != nil {
+			_ = f.Close()
+			return err
+		}
+		left -= n
+	}
+	if err := f.Sync(); err != nil {
+		_ = f.Close()
+		return err
+	}
+	return f.Close()
+}

spinifex/agents/lambda/hostlink/configdisk_test.go

@@ -0,0 +1,63 @@
+//test:in-package reaches the unexported configDisk and fwCfgTree readers to seal and read a real archive
+
+package hostlink
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
+)
+
+// configDisk writes a fake disk holding secret and an extracted tree; the
+// stand-in unmount reports the tree is not a mount, as on a plain directory.
+func configDisk(t *testing.T, secret []byte) (device, root string) {
+	t.Helper()
+	dir := t.TempDir()
+	device = filepath.Join(dir, "vdb")
+	require.NoError(t, os.WriteFile(device, append(bytes.Repeat([]byte{0xa5}, 3*zeroChunk+7), secret...), 0o600))
+	root = fwCfgTree(t)
+	require.NoError(t, os.WriteFile(filepath.Join(root, "by_name", "key"), secret, 0o600))
+	old := unmount
+	unmount = func(string, int) error { return unix.EINVAL }
+	t.Cleanup(func() { unmount = old })
+	return device, root
+}
+
+func TestSealConfigDisk(t *testing.T) {
+	secret := []byte("guest-key-material")
+
+	t.Run("erases the disk and removes the tree", func(t *testing.T) {
+		device, root := configDisk(t, secret)
+		before, err := os.Stat(device)
+		require.NoError(t, err)
+		require.NoError(t, SealFwCfg(root, func() error { return SealConfigDisk(device, root) }))
+
+		disk, err := os.ReadFile(device)
+		require.NoError(t, err)
+		assert.Len(t, disk, int(before.Size()))
+		assert.Equal(t, make([]byte, len(disk)), disk, "configuration disk not fully erased")
+		_, err = os.Stat(root)
+		assert.ErrorIs(t, err, os.ErrNotExist)
+	})
+	t.Run("an unwritable disk keeps the tree and fails closed", func(t *testing.T) {
+		device, root := configDisk(t, secret)
+		require.NoError(t, os.Chmod(device, 0o400))
+		err := SealFwCfg(root, func() error { return SealConfigDisk(device, root) })
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "still readable")
+		_, err = os.Stat(root)
+		assert.NoError(t, err)
+	})
+	t.Run("a failed unmount keeps the tree", func(t *testing.T) {
+		device, root := configDisk(t, secret)
+		unmount = func(string, int) error { return unix.EBUSY }
+		require.Error(t, SealFwCfg(root, func() error { return SealConfigDisk(device, root) }))
+		_, err := os.Stat(root)
+		assert.NoError(t, err)
+	})
+}

spinifex/agents/lambda/hostlink/test_stream.go

@@ -321,6 +321,19 @@ func (s *TestStream) HostSend(req Request) error {
 	return s.write(frame{Type: typeInvoke, Request: &req})
 }
 
+// CheckInvokeFits encodes req as HostSend would and returns ErrFrameTooLarge
+// when the frame exceeds the bound, so a caller can refuse before launch.
+func CheckInvokeFits(req Request) error {
+	b, err := json.Marshal(frame{Type: typeInvoke, Request: &req})
+	if err != nil {
+		return err
+	}
+	if len(b) > maxFrame {
+		return ErrFrameTooLarge
+	}
+	return nil
+}
+
 // HostShutdown writes a shutdown frame, as the host side of the carrier.
 func (s *TestStream) HostShutdown() error {
 	return s.write(frame{Type: typeShutdown})

spinifex/domains/ec2/accountcleanup/remainder.go

@@ -0,0 +1,155 @@
+package accountcleanup
+
+import (
+	"context"
+	"log/slog"
+	"slices"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+)
+
+// ReasonRecordRemains marks a resource whose record the owner still holds.
+const ReasonRecordRemains = "record-remains"
+
+// ComputeRemainder reads what compute the account still holds, changing nothing.
+type ComputeRemainder interface {
+	ComputeRemaining(ctx context.Context, accountID string) (*Outcome, error)
+}
+
+// StorageRemainder reads what volumes and tag residue the account still holds,
+// changing nothing.
+type StorageRemainder interface {
+	StorageRemaining(ctx context.Context, accountID string) (*Outcome, error)
+}
+
+var (
+	_ ComputeRemainder = (*ComputeCleanup)(nil)
+	_ StorageRemainder = (*StorageCleanup)(nil)
+)
+
+func validRemainderAccount(accountID string) error {
+	if accountID == "" {
+		return awserrors.Errorf(awserrors.ErrorMissingParameter, "The request must specify an account ID.")
+	}
+	if !awsidentifiers.IsAccountID(accountID) {
+		return awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "The account ID must be 12 digits.")
+	}
+	return nil
+}
+
+// ComputeRemaining lists every instance the account still holds and every spot
+// request that can still launch. A read that does not complete is an incomplete
+// reason, never a shorter list. The result is a snapshot, not evidence of absence.
+func (c *ComputeCleanup) ComputeRemaining(ctx context.Context, accountID string) (*Outcome, error) {
+	if err := validRemainderAccount(accountID); err != nil {
+		return nil, err
+	}
+	if c == nil || c.spot == nil || c.instances == nil || c.teardown == nil {
+		return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "Compute cleanup is not fully wired.")
+	}
+	res := &Outcome{AccountID: accountID}
+	defer res.finish()
+
+	scan, err := c.spot.ScanAccount(ctx, accountID)
+	if err != nil {
+		slog.WarnContext(ctx, "ComputeRemaining: spot scan failed", "account_id", accountID, "err", err)
+		res.incomplete(ReasonSpotInventoryIncomplete)
+	} else {
+		for _, r := range slices.Concat(scan.Active, scan.Terminal) {
+			if !terminalSpotState(r.State) {
+				res.Items = append(res.Items, Item{ResourceID: r.RequestID, Kind: KindSpotRequest, State: r.State, Disposition: DispositionResidue, Reason: ReasonRecordRemains})
+			}
+		}
+	}
+
+	inventory, err := c.instances.AccountInstances(ctx, accountID)
+	if err != nil || !inventoryUsable(inventory) {
+		slog.WarnContext(ctx, "ComputeRemaining: instance inventory incomplete", "account_id", accountID, "err", err)
+		res.incomplete(ReasonInstanceInventoryIncomplete)
+	} else {
+		for _, reservation := range inventory.Reservations {
+			for _, in := range reservation.Instances {
+				state := ""
+				if in.State != nil {
+					state = aws.StringValue(in.State.Name)
+				}
+				res.Items = append(res.Items, Item{ResourceID: aws.StringValue(in.InstanceId), Kind: KindInstance, State: state, Disposition: DispositionResidue, Reason: ReasonRecordRemains})
+			}
+		}
+	}
+
+	outstanding, err := c.teardown.AccountTeardownOutstanding(accountID)
+	if err != nil {
+		slog.WarnContext(ctx, "ComputeRemaining: teardown unread", "account_id", accountID, "err", err)
+		res.incomplete(ReasonTeardownUnread)
+	} else {
+		for _, o := range outstanding {
+			if !res.hasInstance(o.InstanceID) {
+				res.Items = append(res.Items, Item{ResourceID: o.InstanceID, Kind: KindInstance, State: o.State, Disposition: DispositionResidue, Reason: ReasonTerminating})
+			}
+		}
+	}
+	return res, nil
+}
+
+// StorageRemaining lists every volume record the account still holds, deleting
+// ones included, and the tag entries of resources whose owner holds no record.
+func (c *StorageCleanup) StorageRemaining(ctx context.Context, accountID string) (*Outcome, error) {
+	if err := validRemainderAccount(accountID); err != nil {
+		return nil, err
+	}
+	if c == nil || c.volumes == nil || c.instances == nil || c.teardown == nil || c.spot == nil || c.tags == nil {
+		return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "Storage cleanup is not fully wired.")
+	}
+	res := &Outcome{AccountID: accountID}
+	defer res.finish()
+
+	volumes, err := c.volumes.VolumeInventory(ctx, accountID)
+	if err != nil {
+		slog.WarnContext(ctx, "StorageRemaining: volume inventory incomplete", "account_id", accountID, "err", err)
+		res.incomplete(ReasonVolumeInventoryIncomplete)
+	}
+	for _, v := range volumes {
+		reason := ReasonRecordRemains
+		if v.State == ec2.VolumeStateDeleting {
+			reason = ReasonVolumeDeleting
+		}
+		res.Items = append(res.Items, Item{ResourceID: v.VolumeID, Kind: KindVolume, State: v.State, Disposition: DispositionResidue, Reason: reason})
+	}
+
+	live, err := c.instances.AccountInstances(ctx, accountID)
+	if err != nil || !inventoryUsable(live) {
+		slog.WarnContext(ctx, "StorageRemaining: no complete instance view", "account_id", accountID, "err", err)
+		res.incomplete(ReasonHolderProofIncomplete)
+		return res, nil
+	}
+	outstanding, err := c.teardown.AccountTeardownOutstanding(accountID)
+	if err != nil {
+		slog.WarnContext(ctx, "StorageRemaining: teardown unread", "account_id", accountID, "err", err)
+		res.incomplete(ReasonHolderProofIncomplete)
+		return res, nil
+	}
+
+	tagged, err := c.tags.ScanAccountResourceIDs(ctx, accountID, storageTagPrefixes...)
+	if err != nil {
+		slog.WarnContext(ctx, "StorageRemaining: tag index read failed", "account_id", accountID, "err", err)
+		res.incomplete(ReasonTagIndexUnread)
+		return res, nil
+	}
+	owned, resolved := c.ownedResourceIDs(ctx, accountID, live, outstanding)
+	for _, id := range tagged {
+		switch {
+		case !resolved[tagPrefix(id)]:
+			res.incomplete(ReasonTagOwnerUnresolved)
+			res.Items = append(res.Items, Item{ResourceID: id, Kind: KindTag, Disposition: DispositionResidue, Reason: ReasonOwnerRecordsUnresolved})
+		case owned[id]:
+			// The resource itself is reported; its tags are not residue.
+		default:
+			res.Items = append(res.Items, Item{ResourceID: id, Kind: KindTag, Disposition: DispositionResidue, Reason: ReasonRecordRemains})
+		}
+	}
+	return res, nil
+}

spinifex/domains/ec2/accountcleanup/remainder_test.go

@@ -0,0 +1,97 @@
+package accountcleanup
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountinventory"
+	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
+	ec2spotinstance "github.com/mulgadc/spinifex/spinifex/domains/ec2/spotinstance"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestComputeRemainingReportsEveryHeldInstanceAndOpenSpot(t *testing.T) {
+	r := newComputeRig()
+	r.spot.scans = []*ec2spotinstance.AccountScan{{
+		Active:   []ec2spotinstance.AccountSpotRequest{{RequestID: "sir-0aaaaaaaaaaaaaaa1", State: ec2.SpotInstanceStateOpen}},
+		Terminal: []ec2spotinstance.AccountSpotRequest{{RequestID: "sir-0aaaaaaaaaaaaaaa2", State: ec2.SpotInstanceStateClosed}},
+	}}
+	r.teardown.outstanding = []ec2instance.OutstandingInstance{{InstanceID: "i-0aaaaaaaaaaaaaa99", State: ec2.InstanceStateNameTerminated}}
+
+	res, err := r.cleanup().ComputeRemaining(t.Context(), testAccount)
+	require.NoError(t, err)
+
+	assert.Equal(t, StatusIncomplete, res.Status)
+	assert.NotNil(t, itemFor(res, runningID))
+	assert.NotNil(t, itemFor(res, stoppedID))
+	assert.NotNil(t, itemFor(res, "i-0aaaaaaaaaaaaaa99"))
+	assert.NotNil(t, itemFor(res, "sir-0aaaaaaaaaaaaaaa1"))
+	assert.Nil(t, itemFor(res, "sir-0aaaaaaaaaaaaaaa2"), "a closed request launches nothing")
+	assert.Empty(t, r.terminator.terminated, "a re-read changes nothing")
+}
+
+func TestComputeRemainingCleanWhenNothingHeld(t *testing.T) {
+	r := newComputeRig()
+	r.inventory.out = listing(nil)
+	res, err := r.cleanup().ComputeRemaining(t.Context(), testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, StatusClean, res.Status)
+}
+
+func TestComputeRemainingUnreadStoreIsIncompleteNeverClean(t *testing.T) {
+	for name, mutate := range map[string]func(*computeRig){
+		"instances": func(r *computeRig) { r.inventory.out, r.inventory.err = nil, errors.New("node silent") },
+		"spot":      func(r *computeRig) { r.spot.scanErr = errors.New("spot unreadable") },
+		"teardown":  func(r *computeRig) { r.teardown.err = errors.New("no responder") },
+	} {
+		t.Run(name, func(t *testing.T) {
+			r := newComputeRig()
+			r.inventory.out = listing(nil)
+			mutate(r)
+			res, err := r.cleanup().ComputeRemaining(t.Context(), testAccount)
+			require.NoError(t, err)
+			assert.Equal(t, StatusIncomplete, res.Status)
+			assert.NotEmpty(t, res.IncompleteReasons)
+		})
+	}
+}
+
+func TestStorageRemainingReportsVolumeRecordsAndTagResidue(t *testing.T) {
+	r := newStorageRig()
+	r.volumes.vols = []accountinventory.VolumeItem{
+		volume(freeVol, ec2.VolumeStateAvailable, ""),
+		volume("vol-0aaaaaaaaaaaaaaa2", ec2.VolumeStateDeleting, ""),
+	}
+	r.tags.ids = []string{"i-0aaaaaaaaaaaaaaaf", freeVol}
+
+	res, err := r.cleanup().StorageRemaining(t.Context(), testAccount)
+	require.NoError(t, err)
+
+	assert.Equal(t, StatusIncomplete, res.Status)
+	assert.Equal(t, ReasonRecordRemains, itemFor(res, freeVol).Reason)
+	assert.Equal(t, ReasonVolumeDeleting, itemFor(res, "vol-0aaaaaaaaaaaaaaa2").Reason)
+	assert.Equal(t, KindTag, itemFor(res, "i-0aaaaaaaaaaaaaaaf").Kind, "tags of a resource with no record are residue")
+	assert.Empty(t, r.deleter.deleted, "a re-read deletes nothing")
+	assert.Empty(t, r.tags.deleted, "a re-read clears no tag")
+}
+
+func TestStorageRemainingUnreadStoreIsIncompleteNeverClean(t *testing.T) {
+	for name, mutate := range map[string]func(*storageRig){
+		"volumes":   func(r *storageRig) { r.volumes.failAt = -1 },
+		"instances": func(r *storageRig) { r.instances.out, r.instances.err = nil, errors.New("node silent") },
+		"teardown":  func(r *storageRig) { r.teardown.err = errors.New("no responder") },
+		"tags":      func(r *storageRig) { r.tags.scanErr = errors.New("tag index unreadable") },
+	} {
+		t.Run(name, func(t *testing.T) {
+			r := newStorageRig()
+			r.volumes.vols = nil
+			mutate(r)
+			res, err := r.cleanup().StorageRemaining(t.Context(), testAccount)
+			require.NoError(t, err)
+			assert.Equal(t, StatusIncomplete, res.Status)
+			assert.NotEmpty(t, res.IncompleteReasons)
+		})
+	}
+}

spinifex/domains/ec2/awsapi/instance/AccountTeardownOutstanding.go

@@ -0,0 +1,28 @@
+package instance
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/nats-io/nats.go"
+)
+
+// AccountTeardownOutstandingSubject serves the instances of an account whose
+// teardown is not complete, from the shared record store.
+const AccountTeardownOutstandingSubject = "ec2.AccountTeardownOutstanding"
+
+var accountTeardownOutstandingTimeout = 15 * time.Second
+
+// AccountTeardownOutstanding reads the account's unfinished instance teardowns.
+// No responder, a timeout or an error reply is an error, never an empty list.
+func AccountTeardownOutstanding(ctx context.Context, natsConn *nats.Conn, accountID string) ([]ec2instance.OutstandingInstance, error) {
+	out, err := natsmsg.NATSRequest[ec2instance.TeardownOutstandingOutput](ctx, natsConn, AccountTeardownOutstandingSubject,
+		&ec2instance.TeardownOutstandingInput{}, accountTeardownOutstandingTimeout, accountID)
+	if err != nil {
+		return nil, fmt.Errorf("read outstanding teardown: %w", err)
+	}
+	return out.Instances, nil
+}

spinifex/domains/ec2/awsapi/instance/AccountTeardownOutstanding_test.go

@@ -0,0 +1,88 @@
+package instance
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func serveTeardownOutstanding(t *testing.T, nc *nats.Conn, reply []byte) {
+	t.Helper()
+	_, err := nc.Subscribe(AccountTeardownOutstandingSubject, func(msg *nats.Msg) { _ = msg.Respond(reply) })
+	require.NoError(t, err)
+	require.NoError(t, nc.Flush())
+}
+
+func TestAccountTeardownOutstanding_ReturnsTheServedInstances(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	reply, err := json.Marshal(ec2instance.TeardownOutstandingOutput{Instances: []ec2instance.OutstandingInstance{{InstanceID: "i-1"}}})
+	require.NoError(t, err)
+	serveTeardownOutstanding(t, nc, reply)
+
+	got, err := AccountTeardownOutstanding(context.Background(), nc, "123456789012")
+
+	require.NoError(t, err)
+	require.Len(t, got, 1)
+	assert.Equal(t, "i-1", got[0].InstanceID)
+}
+
+// A read that cannot be answered must never look like an empty list, because an
+// empty list lets the storage cleanup believe no QEMU still holds a volume.
+func TestAccountTeardownOutstanding_UnansweredReadIsAnError(t *testing.T) {
+	t.Parallel()
+
+	t.Run("no responder", func(t *testing.T) {
+		t.Parallel()
+		_, nc := startTestNATSServer(t)
+
+		got, err := AccountTeardownOutstanding(context.Background(), nc, "123456789012")
+
+		require.Error(t, err)
+		assert.Nil(t, got)
+		assert.ErrorIs(t, err, nats.ErrNoResponders)
+	})
+	t.Run("error reply", func(t *testing.T) {
+		t.Parallel()
+		_, nc := startTestNATSServer(t)
+		serveTeardownOutstanding(t, nc, awserrors.GenerateErrorPayload(awserrors.ErrorServerInternal))
+
+		got, err := AccountTeardownOutstanding(context.Background(), nc, "123456789012")
+
+		require.Error(t, err)
+		assert.Nil(t, got)
+	})
+	t.Run("unreadable reply", func(t *testing.T) {
+		t.Parallel()
+		_, nc := startTestNATSServer(t)
+		serveTeardownOutstanding(t, nc, []byte("not json"))
+
+		got, err := AccountTeardownOutstanding(context.Background(), nc, "123456789012")
+
+		require.Error(t, err)
+		assert.Nil(t, got)
+	})
+}
+
+// Not parallel: it shortens the package timeout.
+func TestAccountTeardownOutstanding_TimeoutIsAnError(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+	_, err := nc.Subscribe(AccountTeardownOutstandingSubject, func(*nats.Msg) {})
+	require.NoError(t, err)
+	require.NoError(t, nc.Flush())
+	previous := accountTeardownOutstandingTimeout
+	accountTeardownOutstandingTimeout = 200 * time.Millisecond
+	t.Cleanup(func() { accountTeardownOutstandingTimeout = previous })
+
+	got, err := AccountTeardownOutstanding(context.Background(), nc, "123456789012")
+
+	require.Error(t, err)
+	assert.Nil(t, got)
+}

spinifex/domains/ec2/awsapi/instance/ModifyInstanceAttribute.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/nats-io/nats.go"
 )
@@ -65,6 +66,20 @@ func ModifyInstanceAttribute(ctx context.Context, input *ec2.ModifyInstanceAttri
 
 	slog.InfoContext(ctx, "ModifyInstanceAttribute: Processing request", "instance_id", *input.InstanceId)
 
+	// Termination protection on a running instance lives on its owner, so it
+	// goes there first. A nil value is not a value and keeps the queue path.
+	if input.DisableApiTermination != nil && input.DisableApiTermination.Value != nil {
+		cmd := ec2v1.EC2InstanceCommand{
+			ID:                        *input.InstanceId,
+			Attributes:                ec2v1.EC2CommandAttributes{SetDisableApiTermination: true},
+			DisableApiTerminationData: &ec2v1.DisableApiTerminationData{Value: *input.DisableApiTermination.Value},
+		}
+		if _, err := routeInstanceMutation[ec2.ModifyInstanceAttributeOutput](ctx, natsConn, *input.InstanceId, accountID, cmd, "ec2.ModifyInstanceAttribute", input); err != nil {
+			return ec2.ModifyInstanceAttributeOutput{}, err
+		}
+		return ec2.ModifyInstanceAttributeOutput{}, nil
+	}
+
 	jsonData, err := json.Marshal(input)
 	if err != nil {
 		slog.ErrorContext(ctx, "ModifyInstanceAttribute: Failed to marshal request", "instance_id", *input.InstanceId, "err", err)

spinifex/domains/ec2/awsapi/instance/ModifyInstanceAttribute_test.go

@@ -3,10 +3,13 @@ package instance
 import (
 	"context"
 	"encoding/json"
+	"sync"
+	"sync/atomic"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
@@ -226,3 +229,53 @@ func TestModifyInstanceAttribute_ValidationFailure(t *testing.T) {
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorInvalidParameterValue, err.Error())
 }
+
+func TestModifyInstanceAttribute_DisableApiTerminationRouting(t *testing.T) {
+	const id = "i-0123456789abcdef0"
+	_, nc := startTestNATSServer(t)
+
+	var gotValues []bool
+	var mu sync.Mutex
+	_, err := nc.Subscribe(ec2v1.InstanceCommandSubject(id), func(msg *nats.Msg) {
+		var cmd ec2v1.EC2InstanceCommand
+		if json.Unmarshal(msg.Data, &cmd) == nil && cmd.Attributes.SetDisableApiTermination && cmd.DisableApiTerminationData != nil {
+			mu.Lock()
+			gotValues = append(gotValues, cmd.DisableApiTerminationData.Value)
+			mu.Unlock()
+		}
+		reply := nats.NewMsg(msg.Reply)
+		reply.Data = []byte(`{}`)
+		reply.Header.Set(ec2v1.HandledHeader, "1")
+		_ = msg.RespondMsg(reply)
+	})
+	require.NoError(t, err)
+	var queue atomic.Int32
+	_, err = nc.Subscribe("ec2.ModifyInstanceAttribute", func(msg *nats.Msg) {
+		queue.Add(1)
+		_ = msg.Respond([]byte(`{}`))
+	})
+	require.NoError(t, err)
+
+	for _, v := range []bool{true, false} {
+		_, err := ModifyInstanceAttribute(context.Background(), &ec2.ModifyInstanceAttributeInput{
+			InstanceId:            aws.String(id),
+			DisableApiTermination: &ec2.AttributeBooleanValue{Value: aws.Bool(v)},
+		}, nc, "acct")
+		require.NoError(t, err)
+	}
+	mu.Lock()
+	assert.Equal(t, []bool{true, false}, gotValues, "explicit true and false both reach the owner")
+	mu.Unlock()
+	assert.Zero(t, queue.Load())
+
+	// A nil value is not a value: queue path, no owner command.
+	_, err = ModifyInstanceAttribute(context.Background(), &ec2.ModifyInstanceAttributeInput{
+		InstanceId:            aws.String(id),
+		DisableApiTermination: &ec2.AttributeBooleanValue{},
+	}, nc, "acct")
+	require.NoError(t, err)
+	assert.Equal(t, int32(1), queue.Load())
+	mu.Lock()
+	assert.Len(t, gotValues, 2, "no owner command for a nil value")
+	mu.Unlock()
+}

spinifex/domains/ec2/awsapi/instance/ModifyInstanceMetadataOptions.go

@@ -3,11 +3,11 @@ package instance
 import (
 	"context"
 	"errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"strings"
-	"time"
 
+	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/nats-io/nats.go"
 )
@@ -22,5 +22,16 @@ func ModifyInstanceMetadataOptions(ctx context.Context, input *ec2.ModifyInstanc
 	if !strings.HasPrefix(*input.InstanceId, "i-") {
 		return nil, errors.New(awserrors.ErrorInvalidInstanceIDMalformed)
 	}
-	return natsmsg.NATSRequest[ec2.ModifyInstanceMetadataOptionsOutput](ctx, natsConn, "ec2.ModifyInstanceMetadataOptions", input, 30*time.Second, accountID)
+	cmd := ec2v1.EC2InstanceCommand{
+		ID:         *input.InstanceId,
+		Attributes: ec2v1.EC2CommandAttributes{ModifyInstanceMetadataOptions: true},
+		InstanceMetadataOptionsData: &ec2v1.InstanceMetadataOptionsData{
+			HttpTokens:              aws.StringValue(input.HttpTokens),
+			HttpEndpoint:            aws.StringValue(input.HttpEndpoint),
+			HttpProtocolIpv6:        aws.StringValue(input.HttpProtocolIpv6),
+			InstanceMetadataTags:    aws.StringValue(input.InstanceMetadataTags),
+			HttpPutResponseHopLimit: input.HttpPutResponseHopLimit,
+		},
+	}
+	return routeInstanceMutation[ec2.ModifyInstanceMetadataOptionsOutput](ctx, natsConn, *input.InstanceId, accountID, cmd, "ec2.ModifyInstanceMetadataOptions", input)
 }

spinifex/domains/ec2/awsapi/instance/ModifyInstanceMetadataOptions_test.go

@@ -2,11 +2,16 @@ package instance
 
 import (
 	"context"
+	"encoding/json"
+	"sync/atomic"
 	"testing"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -30,3 +35,179 @@ func TestModifyInstanceMetadataOptions_RejectsMalformedID(t *testing.T) {
 		})
 	}
 }
+
+const metaTestID = "i-0123456789abcdef0"
+
+func metaInput(hop int64) *ec2.ModifyInstanceMetadataOptionsInput {
+	return &ec2.ModifyInstanceMetadataOptionsInput{
+		InstanceId:              aws.String(metaTestID),
+		HttpPutResponseHopLimit: aws.Int64(hop),
+	}
+}
+
+// ownerReply answers one ec2.cmd request, optionally carrying the handled marker.
+func ownerReply(t *testing.T, nc *nats.Conn, handled bool, body []byte, calls *atomic.Int32) {
+	t.Helper()
+	_, err := nc.Subscribe(ec2v1.InstanceCommandSubject(metaTestID), func(msg *nats.Msg) {
+		calls.Add(1)
+		reply := nats.NewMsg(msg.Reply)
+		reply.Data = body
+		if handled {
+			reply.Header.Set(ec2v1.HandledHeader, "1")
+		}
+		_ = msg.RespondMsg(reply)
+	})
+	require.NoError(t, err)
+}
+
+func queueReply(t *testing.T, nc *nats.Conn, body []byte, calls *atomic.Int32) {
+	t.Helper()
+	_, err := nc.QueueSubscribe("ec2.ModifyInstanceMetadataOptions", "spinifex-workers", func(msg *nats.Msg) {
+		calls.Add(1)
+		_ = msg.Respond(body)
+	})
+	require.NoError(t, err)
+}
+
+func metaOutputJSON(t *testing.T, hop int64) []byte {
+	t.Helper()
+	b, err := json.Marshal(&ec2.ModifyInstanceMetadataOptionsOutput{
+		InstanceId:              aws.String(metaTestID),
+		InstanceMetadataOptions: &ec2.InstanceMetadataOptionsResponse{HttpPutResponseHopLimit: aws.Int64(hop)},
+	})
+	require.NoError(t, err)
+	return b
+}
+
+func TestModifyInstanceMetadataOptions_OwnerAnswerReturnedAsIs(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+	var owner, queue atomic.Int32
+	ownerReply(t, nc, true, metaOutputJSON(t, 3), &owner)
+	queueReply(t, nc, []byte(`{}`), &queue)
+
+	out, err := ModifyInstanceMetadataOptions(context.Background(), metaInput(3), nc, "acct")
+	require.NoError(t, err)
+	assert.Equal(t, int64(3), aws.Int64Value(out.InstanceMetadataOptions.HttpPutResponseHopLimit))
+	assert.Equal(t, int32(1), owner.Load())
+	assert.Zero(t, queue.Load())
+}
+
+func TestModifyInstanceMetadataOptions_OwnerErrorWithHeaderIsNotRetried(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+	var owner, queue atomic.Int32
+	ownerReply(t, nc, true, awserrors.GenerateErrorPayload(awserrors.ErrorServerInternal), &owner)
+	queueReply(t, nc, metaOutputJSON(t, 3), &queue)
+
+	_, err := ModifyInstanceMetadataOptions(context.Background(), metaInput(3), nc, "acct")
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+	assert.Zero(t, queue.Load())
+}
+
+func TestModifyInstanceMetadataOptions_OwnerValidationErrorRelayed(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+	var owner, queue atomic.Int32
+	ownerReply(t, nc, true, awserrors.GenerateErrorPayload(awserrors.ErrorUnsupportedOperation), &owner)
+	queueReply(t, nc, metaOutputJSON(t, 3), &queue)
+
+	_, err := ModifyInstanceMetadataOptions(context.Background(), metaInput(3), nc, "acct")
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorUnsupportedOperation, err.Error())
+	assert.Zero(t, queue.Load())
+}
+
+func TestModifyInstanceMetadataOptions_HeaderlessOtherErrorReturnedAsIs(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+	var owner, queue atomic.Int32
+	ownerReply(t, nc, false, awserrors.GenerateErrorPayload(awserrors.ErrorInvalidInstanceIDNotFound), &owner)
+	queueReply(t, nc, metaOutputJSON(t, 3), &queue)
+
+	_, err := ModifyInstanceMetadataOptions(context.Background(), metaInput(3), nc, "acct")
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
+	assert.Zero(t, queue.Load())
+}
+
+func TestS9_N1OwnerRejectionFallsBackOnceToQueueGroup(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+	var owner, queue atomic.Int32
+	ownerReply(t, nc, false, awserrors.GenerateErrorPayload(awserrors.ErrorServerInternal), &owner)
+	queueReply(t, nc, metaOutputJSON(t, 3), &queue)
+
+	out, err := ModifyInstanceMetadataOptions(context.Background(), metaInput(3), nc, "acct")
+	require.NoError(t, err)
+	assert.Equal(t, int64(3), aws.Int64Value(out.InstanceMetadataOptions.HttpPutResponseHopLimit))
+	assert.Equal(t, int32(1), owner.Load(), "exactly one owner request")
+	assert.Equal(t, int32(1), queue.Load(), "exactly one queue-group request")
+}
+
+func TestModifyInstanceMetadataOptions_NoOwnerUsesQueueGroupAfterThreeAttempts(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+	var sleeps atomic.Int32
+	prev := cmdRetrySleep
+	cmdRetrySleep = func(time.Duration) { sleeps.Add(1) }
+	t.Cleanup(func() { cmdRetrySleep = prev })
+
+	var queue atomic.Int32
+	queueReply(t, nc, metaOutputJSON(t, 2), &queue)
+
+	out, err := ModifyInstanceMetadataOptions(context.Background(), metaInput(2), nc, "acct")
+	require.NoError(t, err)
+	assert.Equal(t, int64(2), aws.Int64Value(out.InstanceMetadataOptions.HttpPutResponseHopLimit))
+	assert.Equal(t, int32(2), sleeps.Load(), "three attempts means two backoffs")
+	assert.Equal(t, int32(1), queue.Load())
+}
+
+func TestModifyInstanceMetadataOptions_AbsentIsNotFound(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+	var queue atomic.Int32
+	queueReply(t, nc, awserrors.GenerateErrorPayload(awserrors.ErrorInvalidInstanceIDNotFound), &queue)
+
+	_, err := ModifyInstanceMetadataOptions(context.Background(), metaInput(2), nc, "acct")
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
+}
+
+func TestModifyInstanceMetadataOptions_OwnerTimeoutIsServerInternalWithoutFallback(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+	prev := instanceCmdTimeout
+	instanceCmdTimeout = 50 * time.Millisecond
+	t.Cleanup(func() { instanceCmdTimeout = prev })
+
+	// The owner applies the change, then withholds its reply.
+	var applied, queue atomic.Int32
+	_, err := nc.Subscribe(ec2v1.InstanceCommandSubject(metaTestID), func(*nats.Msg) { applied.Add(1) })
+	require.NoError(t, err)
+	queueReply(t, nc, metaOutputJSON(t, 3), &queue)
+
+	_, err = ModifyInstanceMetadataOptions(context.Background(), metaInput(3), nc, "acct")
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+	assert.Equal(t, int32(1), applied.Load(), "the owner saw the command, so the outcome is unknown")
+	assert.Zero(t, queue.Load(), "a timeout never falls back to the queue group")
+}
+
+func TestModifyInstanceMetadataOptions_TimeoutAfterApplyIsUnknownNotRolledBack(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+	prev := instanceCmdTimeout
+	instanceCmdTimeout = 50 * time.Millisecond
+	t.Cleanup(func() { instanceCmdTimeout = prev })
+
+	// The owner persists the change, then the reply is lost.
+	var persisted atomic.Int64
+	_, err := nc.Subscribe(ec2v1.InstanceCommandSubject(metaTestID), func(msg *nats.Msg) {
+		var cmd ec2v1.EC2InstanceCommand
+		if json.Unmarshal(msg.Data, &cmd) == nil && cmd.InstanceMetadataOptionsData != nil {
+			persisted.Store(aws.Int64Value(cmd.InstanceMetadataOptionsData.HttpPutResponseHopLimit))
+		}
+	})
+	require.NoError(t, err)
+	var queue atomic.Int32
+	queueReply(t, nc, metaOutputJSON(t, 1), &queue)
+
+	_, err = ModifyInstanceMetadataOptions(context.Background(), metaInput(3), nc, "acct")
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+	assert.Zero(t, queue.Load(), "no fallback that could race a live owner")
+	assert.Equal(t, int64(3), persisted.Load(), "the gateway error does not undo what the owner stored")
+}

spinifex/domains/ec2/awsapi/instance/instanceid.go

@@ -99,3 +99,46 @@ func classifyUnownedInstance(ctx context.Context, natsConn *nats.Conn, instanceI
 	}
 	return bucketStateAbsent, allOK
 }
+
+// routeInstanceMutation sends cmd to the instance's owner first and falls back
+// to the queue-group subject only when that is safe: no daemon runs the
+// instance (ErrNoResponders after every retry), or the owner is N-1 and
+// rejected the command without writing (ServerInternal with no HandledHeader).
+// A timeout or other transport error is an unknown outcome and never falls back.
+func routeInstanceMutation[Out any](ctx context.Context, natsConn *nats.Conn, instanceID, accountID string, cmd ec2v1.EC2InstanceCommand, queueSubject string, queueInput any) (*Out, error) {
+	queue := func() (*Out, error) {
+		return natsmsg.NATSRequest[Out](ctx, natsConn, queueSubject, queueInput, 30*time.Second, accountID)
+	}
+
+	data, err := json.Marshal(cmd)
+	if err != nil {
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	msg, err := requestInstanceCmd(ctx, natsConn, instanceID, accountID, data)
+	if err != nil {
+		if errors.Is(err, nats.ErrNoResponders) {
+			return queue()
+		}
+		slog.ErrorContext(ctx, "Owner request failed, outcome unknown", "instance_id", instanceID, "subject", queueSubject, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+
+	responseError, parseErr := awserrors.ValidateErrorPayload(msg.Data)
+	if parseErr != nil {
+		handled := msg.Header.Get(ec2v1.HandledHeader) == "1"
+		if !handled && *responseError.Code == awserrors.ErrorServerInternal {
+			slog.WarnContext(ctx, "Owner did not recognise the command, using the queue group", "instance_id", instanceID, "subject", queueSubject)
+			return queue()
+		}
+		if responseError.Message != nil && *responseError.Message != "" {
+			return nil, awserrors.Errorf(*responseError.Code, "%s", *responseError.Message)
+		}
+		return nil, errors.New(*responseError.Code)
+	}
+
+	var out Out
+	if err := json.Unmarshal(msg.Data, &out); err != nil {
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	return &out, nil
+}

spinifex/domains/ec2/instance/teardown_outstanding.go

@@ -1,12 +1,33 @@
 package instance
 
+import "context"
+
 // OutstandingInstance is an instance of an account that is not yet fully torn
 // down, with the state its record carries.
 type OutstandingInstance struct {
-	InstanceID string
+	InstanceID string `json:"instance_id"`
 	// State is the record's lifecycle state, such as running or terminated.
-	State string
+	State string `json:"state"`
 	// Teardown is the per-dependent progress (done, pending or failed) the
 	// record carries. It is empty when no dependent has been recorded.
-	Teardown map[string]string
+	Teardown map[string]string `json:"teardown,omitempty"`
+}
+
+// TeardownOutstandingInput is the empty request of the outstanding-teardown
+// read; the account comes from the message header.
+type TeardownOutstandingInput struct{}
+
+// TeardownOutstandingOutput carries every instance not yet fully torn down.
+type TeardownOutstandingOutput struct {
+	Instances []OutstandingInstance `json:"instances"`
+}
+
+// AccountTeardownOutstandingRPC serves AccountTeardownOutstanding over the
+// message bus, so a process without the instance record store can read it.
+func (s *InstanceServiceImpl) AccountTeardownOutstandingRPC(_ context.Context, _ *TeardownOutstandingInput, accountID string) (*TeardownOutstandingOutput, error) {
+	outstanding, err := s.AccountTeardownOutstanding(accountID)
+	if err != nil {
+		return nil, err
+	}
+	return &TeardownOutstandingOutput{Instances: outstanding}, nil
 }

spinifex/domains/ec2/inventory/inventory.go

@@ -0,0 +1,67 @@
+// Package inventory is the EC2-owned read capability for an account's complete
+// instance list, for callers outside the EC2 domain.
+package inventory
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	ec2instanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/instance"
+	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// Instances lists every non-terminated instance an account owns, or returns an
+// error. It never returns a shorter list than the truth.
+type Instances interface {
+	ListAccountInstances(ctx context.Context, accountID string) (*ec2.DescribeInstancesOutput, error)
+}
+
+// NATSInstances reads the complete list across nodeIDs over NATS.
+type NATSInstances struct {
+	nc      *nats.Conn
+	nodeIDs []string
+}
+
+var _ Instances = (*NATSInstances)(nil)
+
+// NewNATSInstances returns a reader over the configured node set.
+func NewNATSInstances(nc *nats.Conn, nodeIDs []string) *NATSInstances {
+	return &NATSInstances{nc: nc, nodeIDs: nodeIDs}
+}
+
+// ListAccountInstances passes through the strict listing unchanged: any node
+// that did not answer, or any record that could not be read, is an error.
+func (n *NATSInstances) ListAccountInstances(ctx context.Context, accountID string) (*ec2.DescribeInstancesOutput, error) {
+	return ec2instanceapi.DescribeAccountInstancesStrict(ctx, n.nc, n.nodeIDs, accountID)
+}
+
+// NewENITagReader returns a reader of one ENI's tags straight from its stored
+// record, so an ENI that is absent and one that cannot be read are told apart:
+// a missing key is found=false, a failed read or an undecodable record is an
+// error. accountID is the account the ENI is stored under.
+func NewENITagReader(js jetstream.JetStream, accountID string) func(ctx context.Context, eniID string) (map[string]string, bool, error) {
+	return func(ctx context.Context, eniID string) (map[string]string, bool, error) {
+		kv, err := js.KeyValue(ctx, ec2vpc.KVBucketENIs)
+		if err != nil {
+			return nil, false, fmt.Errorf("open ENI bucket: %w", err)
+		}
+		entry, err := kv.Get(ctx, kvutil.AccountKey(accountID, eniID))
+		if errors.Is(err, jetstream.ErrKeyNotFound) {
+			return nil, false, nil
+		}
+		if err != nil {
+			return nil, false, fmt.Errorf("read ENI record %s: %w", eniID, err)
+		}
+		var record ec2vpc.ENIRecord
+		if err := json.Unmarshal(entry.Value(), &record); err != nil {
+			return nil, false, fmt.Errorf("decode ENI record %s: %w", eniID, err)
+		}
+		return record.Tags, true, nil
+	}
+}

spinifex/domains/ec2/inventory/inventory_test.go

@@ -0,0 +1,49 @@
+package inventory
+
+import (
+	"encoding/json"
+	"testing"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestListAccountInstancesRefusesAnUnknownNodeSet(t *testing.T) {
+	out, err := NewNATSInstances(nil, nil).ListAccountInstances(t.Context(), "000000000000")
+	require.Error(t, err)
+	assert.Nil(t, out, "an unknown node set must not yield a list")
+}
+
+func TestENITagReaderSeparatesGoneFromUnreadable(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	js := testutil.NewJetStream(t, nc)
+	read := NewENITagReader(js, "000000000000")
+
+	_, _, err := read(t.Context(), "eni-1")
+	require.Error(t, err, "a missing ENI bucket is a read failure, not an absent ENI")
+
+	enis, err := kvutil.GetOrCreateBucket(t.Context(), js, ec2vpc.KVBucketENIs, 10)
+	require.NoError(t, err)
+	good, err := json.Marshal(ec2vpc.ENIRecord{NetworkInterfaceId: "eni-good", Tags: map[string]string{"k": "v"}})
+	require.NoError(t, err)
+	_, err = enis.Put(t.Context(), kvutil.AccountKey("000000000000", "eni-good"), good)
+	require.NoError(t, err)
+	_, err = enis.Put(t.Context(), kvutil.AccountKey("000000000000", "eni-corrupt"), []byte("{not json"))
+	require.NoError(t, err)
+
+	tags, found, err := read(t.Context(), "eni-good")
+	require.NoError(t, err)
+	assert.True(t, found)
+	assert.Equal(t, map[string]string{"k": "v"}, tags)
+
+	_, found, err = read(t.Context(), "eni-absent")
+	require.NoError(t, err)
+	assert.False(t, found, "a missing key is the ENI gone")
+
+	_, found, err = read(t.Context(), "eni-corrupt")
+	require.Error(t, err, "an undecodable record is a read failure")
+	assert.False(t, found)
+}

spinifex/domains/ec2/systeminstance/launcher.go

@@ -57,7 +57,7 @@ const (
 	BootAMI BootMode = "ami"
 )
 
-// SystemInstanceLauncher launches and terminates system-managed VMs.
+// SystemInstanceLauncher launches, reboots and terminates system-managed VMs.
 // Defined here to avoid a circular import between elbv2/eks and daemon.
 type SystemInstanceLauncher interface {
 	// LaunchSystemInstance creates and starts a system-managed VM, returning its ID and private IP.
@@ -65,6 +65,10 @@ type SystemInstanceLauncher interface {
 
 	// TerminateSystemInstance stops and cleans up a system-managed VM.
 	TerminateSystemInstance(instanceID string) error
+
+	// RebootSystemInstance restarts a system-managed load balancer VM into a
+	// fresh QEMU on its owning node, keeping its ID, ENIs and addresses.
+	RebootSystemInstance(instanceID string) error
 }
 
 // SystemInstanceInput describes a system VM to launch.

spinifex/domains/eks/cluster/cp_orphans.go

@@ -0,0 +1,158 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"sort"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// ClusterRecordState is what a read of a cluster's record established.
+type ClusterRecordState int
+
+const (
+	ClusterRecordPresent ClusterRecordState = iota
+	ClusterRecordMissing
+	ClusterRecordUnreadable
+)
+
+// ClusterRecordVerdict is the outcome of looking a cluster record up without
+// creating anything. Reason names a missing or unreadable record; Err is set
+// only when the record could not be read.
+type ClusterRecordVerdict struct {
+	State  ClusterRecordState
+	Reason string
+	Err    error
+}
+
+// JudgeClusterRecord reads the cluster record read-only. A missing bucket or a
+// missing key is Missing, any other failure is Unreadable, and neither is ever
+// repaired. The billable reaper and the orphan report both use it, so they
+// judge a record the same way.
+func JudgeClusterRecord(ctx context.Context, js jetstream.JetStream, account, cluster string) ClusterRecordVerdict {
+	kv, err := OpenAccountBucket(ctx, js, account)
+	switch {
+	case errors.Is(err, jetstream.ErrBucketNotFound):
+		return ClusterRecordVerdict{State: ClusterRecordMissing, Reason: "account bucket missing"}
+	case err != nil:
+		return ClusterRecordVerdict{State: ClusterRecordUnreadable, Reason: "account bucket lookup failed", Err: err}
+	}
+	_, err = GetClusterMeta(ctx, kv, cluster)
+	switch {
+	case err == nil:
+		return ClusterRecordVerdict{State: ClusterRecordPresent}
+	case errors.Is(err, ErrClusterNotFound):
+		return ClusterRecordVerdict{State: ClusterRecordMissing, Reason: "cluster record missing"}
+	}
+	return ClusterRecordVerdict{State: ClusterRecordUnreadable, Reason: "cluster record read failed", Err: err}
+}
+
+// ENITagReader returns the tags on a control-plane ENI. found is false when
+// the ENI is gone.
+type ENITagReader func(ctx context.Context, eniID string) (tagSet map[string]string, found bool, err error)
+
+// ControlPlaneRow is one running EKS control-plane VM the report could not tie
+// to a live cluster record. Unreadable rows carry Err and are never omitted.
+type ControlPlaneRow struct {
+	InstanceID       string
+	Cluster          string
+	Account          string
+	Reason           string
+	Age              time.Duration
+	AvailabilityZone string
+	Unreadable       bool
+	Err              error
+}
+
+// FindControlPlaneOrphans classifies the instances in reservations. An instance
+// with no ENI, or whose ENI is gone or not tagged as an EKS control plane, is
+// skipped. A failed ENI read is an unreadable row, never a silent drop. A
+// control plane whose cluster record is present is omitted; one whose bucket or
+// record is missing is an orphan row; any other read failure is unreadable. It
+// only reads.
+func FindControlPlaneOrphans(ctx context.Context, js jetstream.JetStream, reservations []*ec2.Reservation, eni ENITagReader, now time.Time) []ControlPlaneRow {
+	var rows []ControlPlaneRow
+	for _, res := range reservations {
+		if res == nil {
+			continue
+		}
+		for _, inst := range res.Instances {
+			if inst == nil || !instanceRunning(inst) || len(inst.NetworkInterfaces) == 0 || inst.NetworkInterfaces[0] == nil {
+				continue
+			}
+			eniID := aws.StringValue(inst.NetworkInterfaces[0].NetworkInterfaceId)
+			if eniID == "" {
+				continue
+			}
+			row := ControlPlaneRow{InstanceID: aws.StringValue(inst.InstanceId)}
+			if inst.Placement != nil {
+				row.AvailabilityZone = aws.StringValue(inst.Placement.AvailabilityZone)
+			}
+			if inst.LaunchTime != nil {
+				row.Age = now.Sub(*inst.LaunchTime).Truncate(time.Second)
+			}
+
+			tagSet, found, err := eni(ctx, eniID)
+			if err != nil {
+				row.Unreadable, row.Err, row.Reason = true, err, "control-plane ENI read failed"
+				rows = append(rows, row)
+				continue
+			}
+			if !found || tagSet[tags.ManagedByKey] != tags.ManagedByEKS || tagSet[clusterEKSRoleTagKey] != clusterEKSRoleControlPlane {
+				continue
+			}
+			row.Cluster, row.Account = tagSet[clusterEKSClusterTagKey], tagSet[clusterEKSAccountTagKey]
+			if row.Cluster == "" || row.Account == "" {
+				continue
+			}
+
+			verdict := JudgeClusterRecord(ctx, js, row.Account, row.Cluster)
+			switch verdict.State {
+			case ClusterRecordPresent:
+				continue
+			case ClusterRecordUnreadable:
+				row.Unreadable, row.Err = true, verdict.Err
+			}
+			row.Reason = verdict.Reason
+			rows = append(rows, row)
+		}
+	}
+	sort.Slice(rows, func(i, j int) bool { return rows[i].InstanceID < rows[j].InstanceID })
+	return rows
+}
+
+// ReportControlPlaneOrphans writes the orphan report. list must return every
+// non-terminated system instance or an error: a failed or partial listing is
+// returned as an error and nothing is printed, so a short list is never taken
+// for a complete one. Unreadable rows are printed and counted separately.
+func ReportControlPlaneOrphans(ctx context.Context, w io.Writer, js jetstream.JetStream, list func(context.Context) (*ec2.DescribeInstancesOutput, error), eni ENITagReader, now time.Time) (orphans, unreadable int, err error) {
+	out, err := list(ctx)
+	if err != nil {
+		return 0, 0, fmt.Errorf("listing incomplete: %w", err)
+	}
+	if out == nil {
+		return 0, 0, errors.New("listing incomplete: no result")
+	}
+	rows := FindControlPlaneOrphans(ctx, js, out.Reservations, eni, now)
+	for _, r := range rows {
+		if r.Unreadable {
+			unreadable++
+			fmt.Fprintf(w, "UNREADABLE  %s  cluster=%s account=%s az=%s age=%s  %s: %v\n", r.InstanceID, r.Cluster, r.Account, r.AvailabilityZone, r.Age, r.Reason, r.Err)
+			continue
+		}
+		orphans++
+		fmt.Fprintf(w, "ORPHAN      %s  cluster=%s account=%s az=%s age=%s  %s\n", r.InstanceID, r.Cluster, r.Account, r.AvailabilityZone, r.Age, r.Reason)
+	}
+	return orphans, unreadable, nil
+}
+
+func instanceRunning(inst *ec2.Instance) bool {
+	return inst.State != nil && aws.StringValue(inst.State.Name) == ec2.InstanceStateNameRunning
+}

spinifex/domains/eks/cluster/cp_orphans_test.go

@@ -0,0 +1,198 @@
+package cluster
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	ec2inventory "github.com/mulgadc/spinifex/spinifex/domains/ec2/inventory"
+	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func orphanInstance(id, eni string) *ec2.Instance {
+	return &ec2.Instance{
+		InstanceId:        aws.String(id),
+		State:             &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)},
+		LaunchTime:        aws.Time(time.Unix(0, 0)),
+		Placement:         &ec2.Placement{AvailabilityZone: aws.String("ap-southeast-2a")},
+		NetworkInterfaces: []*ec2.InstanceNetworkInterface{{NetworkInterfaceId: aws.String(eni)}},
+	}
+}
+
+func cpTags(cluster, account string) map[string]string {
+	return map[string]string{
+		tags.ManagedByKey:       tags.ManagedByEKS,
+		clusterEKSClusterTagKey: cluster,
+		clusterEKSAccountTagKey: account,
+		clusterEKSRoleTagKey:    clusterEKSRoleControlPlane,
+	}
+}
+
+func workerTags(cluster, account string) map[string]string {
+	m := cpTags(cluster, account)
+	m[clusterEKSRoleTagKey] = "worker"
+	return m
+}
+
+func TestFindControlPlaneOrphansUnreadableRecordIsNotAnOrphan(t *testing.T) {
+	f := newEKSServiceFixture(t)
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+	ctx, cancel := context.WithCancel(t.Context())
+	cancel()
+	eni := func(context.Context, string) (map[string]string, bool, error) {
+		return cpTags("gone", testAccountID), true, nil
+	}
+	res := []*ec2.Reservation{{Instances: []*ec2.Instance{orphanInstance("i-1", "eni-1")}}}
+
+	rows := FindControlPlaneOrphans(ctx, js, res, eni, time.Unix(60, 0))
+
+	require.Len(t, rows, 1)
+	assert.True(t, rows[0].Unreadable, "a read that failed is not a missing record")
+	assert.Error(t, rows[0].Err)
+}
+
+func TestFindControlPlaneOrphans(t *testing.T) {
+	const lost = "999999999999"
+	f := newEKSServiceFixture(t)
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+	require.NoError(t, PutClusterMeta(t.Context(), f.kv, sampleClusterMeta("alive")))
+
+	eniTags := map[string]map[string]string{
+		"eni-orphan":   cpTags("gone", testAccountID),
+		"eni-live":     cpTags("alive", testAccountID),
+		"eni-nobucket": cpTags("lost", lost),
+		"eni-worker":   workerTags("gone", testAccountID),
+	}
+	eni := func(_ context.Context, id string) (map[string]string, bool, error) {
+		if id == "eni-broken" {
+			return nil, false, errors.New("vpc service down")
+		}
+		m, ok := eniTags[id]
+		return m, ok, nil
+	}
+	res := []*ec2.Reservation{{Instances: []*ec2.Instance{
+		orphanInstance("i-orphan", "eni-orphan"),
+		orphanInstance("i-live", "eni-live"),
+		orphanInstance("i-nobucket", "eni-nobucket"),
+		orphanInstance("i-worker", "eni-worker"),
+		orphanInstance("i-ghost", "eni-ghost"),
+		orphanInstance("i-broken", "eni-broken"),
+		{InstanceId: aws.String("i-noeni")},
+	}}}
+
+	rows := FindControlPlaneOrphans(t.Context(), js, res, eni, time.Unix(60, 0))
+
+	got := map[string]ControlPlaneRow{}
+	for _, r := range rows {
+		got[r.InstanceID] = r
+	}
+	assert.Len(t, got, 3, "only orphan, nobucket and the unreadable ENI are reported")
+	assert.Equal(t, "cluster record missing", got["i-orphan"].Reason)
+	assert.False(t, got["i-orphan"].Unreadable)
+	assert.Equal(t, "account bucket missing", got["i-nobucket"].Reason)
+	assert.True(t, got["i-broken"].Unreadable, "a failed ENI read is never dropped")
+	assert.Error(t, got["i-broken"].Err)
+	assert.NotContains(t, got, "i-live", "a present record is omitted")
+	assert.NotContains(t, got, "i-worker")
+	assert.NotContains(t, got, "i-ghost")
+	assert.NotContains(t, got, "i-noeni")
+}
+
+func TestReportControlPlaneOrphansIncompleteListingPrintsNothing(t *testing.T) {
+	f := newEKSServiceFixture(t)
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+	var buf bytes.Buffer
+	list := func(context.Context) (*ec2.DescribeInstancesOutput, error) {
+		return nil, errors.New("ServiceUnavailable")
+	}
+	orphans, unreadable, err := ReportControlPlaneOrphans(t.Context(), &buf, js, list, nil, time.Now())
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "listing incomplete")
+	assert.Zero(t, orphans+unreadable)
+	assert.Empty(t, buf.String(), "no partial list on a failed listing")
+}
+
+func TestReportControlPlaneOrphansPrintsRows(t *testing.T) {
+	f := newEKSServiceFixture(t)
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+	var buf bytes.Buffer
+	list := func(context.Context) (*ec2.DescribeInstancesOutput, error) {
+		return &ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{{Instances: []*ec2.Instance{
+			orphanInstance("i-orphan", "eni-orphan"), orphanInstance("i-broken", "eni-broken"),
+		}}}}, nil
+	}
+	eni := func(_ context.Context, id string) (map[string]string, bool, error) {
+		if id == "eni-broken" {
+			return nil, false, errors.New("boom")
+		}
+		return cpTags("gone", testAccountID), true, nil
+	}
+	orphans, unreadable, err := ReportControlPlaneOrphans(t.Context(), &buf, js, list, eni, time.Unix(60, 0))
+	require.NoError(t, err)
+	assert.Equal(t, 1, orphans)
+	assert.Equal(t, 1, unreadable)
+	assert.Contains(t, buf.String(), "ORPHAN      i-orphan")
+	assert.Contains(t, buf.String(), "UNREADABLE  i-broken")
+	assert.Equal(t, 2, strings.Count(buf.String(), "az=ap-southeast-2a"), "both row kinds print the zone")
+}
+
+func TestFindControlPlaneOrphansCorruptENIRecordIsUnreadable(t *testing.T) {
+	f := newEKSServiceFixture(t)
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+	enis, err := kvutil.GetOrCreateBucket(t.Context(), js, ec2vpc.KVBucketENIs, 10)
+	require.NoError(t, err)
+	good, err := json.Marshal(ec2vpc.ENIRecord{NetworkInterfaceId: "eni-good", Tags: cpTags("gone", testAccountID)})
+	require.NoError(t, err)
+	_, err = enis.Put(t.Context(), kvutil.AccountKey("000000000000", "eni-good"), good)
+	require.NoError(t, err)
+	_, err = enis.Put(t.Context(), kvutil.AccountKey("000000000000", "eni-corrupt"), []byte("{not json"))
+	require.NoError(t, err)
+	res := []*ec2.Reservation{{Instances: []*ec2.Instance{
+		orphanInstance("i-good", "eni-good"), orphanInstance("i-corrupt", "eni-corrupt"), orphanInstance("i-absent", "eni-absent"),
+	}}}
+
+	rows := FindControlPlaneOrphans(t.Context(), js, res, ec2inventory.NewENITagReader(js, "000000000000"), time.Unix(60, 0))
+
+	require.Len(t, rows, 2, "an absent ENI is skipped, the other two are reported")
+	assert.Equal(t, "i-corrupt", rows[0].InstanceID)
+	assert.True(t, rows[0].Unreadable, "a corrupt ENI record is unreadable, never gone")
+	assert.Equal(t, "i-good", rows[1].InstanceID)
+	assert.False(t, rows[1].Unreadable)
+}
+
+func TestFindControlPlaneOrphansOnlyReportsRunningInstances(t *testing.T) {
+	f := newEKSServiceFixture(t)
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+	eni := func(context.Context, string) (map[string]string, bool, error) {
+		return cpTags("gone", testAccountID), true, nil
+	}
+	mk := func(id, state string) *ec2.Instance {
+		i := orphanInstance(id, "eni-"+id)
+		i.State = &ec2.InstanceState{Name: aws.String(state)}
+		return i
+	}
+	noState := orphanInstance("i-nostate", "eni-nostate")
+	noState.State = nil
+	res := []*ec2.Reservation{{Instances: []*ec2.Instance{
+		mk("i-running", ec2.InstanceStateNameRunning),
+		mk("i-stopped", ec2.InstanceStateNameStopped),
+		mk("i-pending", ec2.InstanceStateNamePending),
+		noState,
+	}}}
+
+	rows := FindControlPlaneOrphans(t.Context(), js, res, eni, time.Unix(60, 0))
+
+	require.Len(t, rows, 1)
+	assert.Equal(t, "i-running", rows[0].InstanceID)
+}

spinifex/domains/eks/cluster/eks_billable_reaper.go

@@ -2,7 +2,6 @@ package cluster
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"log/slog"
 
@@ -13,13 +12,10 @@ import (
 	"github.com/nats-io/nats.go/jetstream"
 )
 
-// EKSBillableReaper is the meta-independent billable cleanup from historic PROP-LIFECYCLE-006 §5 (behaviour per ADR-0003:S2), built on
+// EKSBillableReaper raises the data-safety alarm for a running EKS control-plane
-// the resource lifecycle GC backstop. A normal DeleteCluster can leave a
+// VM whose cluster record cannot be found. It never terminates anything: a
-// running control-plane VM behind (terminate completes async; a daemon restart
+// missing record cannot tell a completed DeleteCluster from a lost bucket or an
-// re-launches it from saved running-state) after the cluster meta is already
+// out-of-band delete, so removal is an operator decision. It is node-local.
-// swept — a billable orphan with no owner to drive a retry. This reaper finds a
-// running EKS control-plane VM whose cluster meta is DEFINITIVELY GONE and
-// terminates it. It is node-local: each node reaps the orphans it runs.
 type EKSBillableReaper struct {
 	svc *EKSServiceImpl
 	// running returns this node's running VMs. The reaper filters to the EKS
@@ -38,10 +34,11 @@ func (s *EKSServiceImpl) NewBillableReaper(running func() ([]*vm.VM, error)) *EK
 func (r *EKSBillableReaper) Class() string         { return "eks-billable" }
 func (r *EKSBillableReaper) Scope() vm.ReaperScope { return vm.ScopeNodeLocal }
 
-// Sweep terminates every running EKS control-plane VM whose cluster meta is
+// Sweep alarms on every running EKS control-plane VM whose cluster record, or
-// absent. A VM whose cluster still exists (CREATING or DELETING) is left to the
+// whole account bucket, is missing, and removes nothing. A VM whose cluster
-// cluster's own teardown/retry; an ENI that is gone or untagged is skipped, so
+// exists is left to the cluster's own teardown; an ENI that is gone or untagged
-// the reaper never reaps on uncertainty.
+// is skipped, and a read that fails is logged and skipped. It returns 0 because
+// it never removes a VM.
 func (r *EKSBillableReaper) Sweep(ctx context.Context) (int, error) {
 	vms, err := r.running()
 	if err != nil {
@@ -53,11 +50,10 @@ func (r *EKSBillableReaper) Sweep(ctx context.Context) (int, error) {
 		return 0, fmt.Errorf("jetstream: %w", err)
 	}
 
-	reaped := 0
 	for _, v := range vms {
 		select {
 		case <-ctx.Done():
-			return reaped, ctx.Err()
+			return 0, ctx.Err()
 		default:
 		}
 		if v == nil || v.ManagedBy != tags.ManagedByEKS || v.ENIId == "" {
@@ -66,47 +62,23 @@ func (r *EKSBillableReaper) Sweep(ctx context.Context) (int, error) {
 
 		clusterName, clusterAccount, ok := r.clusterRefFromENI(ctx, v)
 		if !ok {
-			continue // ENI gone or untagged: cannot confirm orphan-hood, never reap
+			continue // ENI gone or untagged: cannot tie the VM to a cluster
 		}
 
-		acctKV, err := GetOrCreateAccountBucket(ctx, js, clusterAccount)
+		verdict := JudgeClusterRecord(ctx, js, clusterAccount, clusterName)
-		if err != nil {
+		switch verdict.State {
-			slog.WarnContext(ctx, "eks-billable: account bucket lookup failed", "account", clusterAccount, "err", err)
+		case ClusterRecordPresent:
-			continue
+			continue // cluster exists (live, CREATING or DELETING)
-		}
+		case ClusterRecordUnreadable:
-		if _, err := GetClusterMeta(ctx, acctKV, clusterName); !errors.Is(err, ErrClusterNotFound) {
+			slog.WarnContext(ctx, "eks-billable: "+verdict.Reason, "cluster", clusterName, "account", clusterAccount, "err", verdict.Err)
-			// Meta present (live/CREATING/DELETING) or unreadable: not a definitive
-			// orphan — leave it to the cluster's own teardown/retry.
 			continue
 		}
+		reason := verdict.Reason
 
-		// ALARM + reap: the cluster is gone but its control-plane VM still runs.
+		slog.WarnContext(ctx, "DATA-SAFETY ALARM: EKS control-plane VM has no cluster record; retained for an operator",
-		slog.WarnContext(ctx, "DATA-SAFETY ALARM: orphaned EKS control-plane VM reaped (cluster meta gone)",
+			"reason", reason, "instanceId", v.ID, "cluster", clusterName, "account", clusterAccount, "node", v.LastNode)
-			"instanceId", v.ID, "cluster", clusterName, "account", clusterAccount, "node", v.LastNode)
-		if err := TerminateK3sServerVM(ctx, r.svc.deps.VPCK3s, r.svc.deps.Instance, v.AccountID, v.ID, v.ENIId); err != nil {
-			slog.WarnContext(ctx, "eks-billable: failed to terminate orphan CP VM", "instanceId", v.ID, "err", err)
-			continue
-		}
-		reaped++
-		r.reclaimOrphanInfra(ctx, v.AccountID, clusterName)
-	}
-	return reaped, nil
-}
-
-// reclaimOrphanInfra tears down the billable infra a reaped orphan's swept meta
-// no longer anchors: the NLB front-end NAT and the managed CP VPC's NAT-GW EIP.
-// Both are keyed by cluster name and idempotent, so a clean prior delete is a
-// no-op. Best-effort — the costliest resource (the VM) is already gone, and
-// DeleteClusterCPVPC releases the NAT-GW EIP before the VPC delete, so the
-// billable address is reclaimed even if the now-empty VPC delete trails. account
-// is the infra account (system account for the managed CP VPC topology).
-func (r *EKSBillableReaper) reclaimOrphanInfra(ctx context.Context, account, clusterName string) {
-	if err := DeleteClusterNLB(ctx, r.svc.deps.NLB, account, clusterName); err != nil {
-		slog.WarnContext(ctx, "eks-billable: reclaim orphan NLB failed", "cluster", clusterName, "err", err)
-	}
-	if err := DeleteClusterCPVPC(ctx, r.svc.cpVPCDeps(), account, clusterName, nil); err != nil {
-		slog.WarnContext(ctx, "eks-billable: reclaim orphan CP VPC (NAT-GW EIP) failed", "cluster", clusterName, "err", err)
 	}
+	return 0, nil
 }
 
 // clusterRefFromENI reads the cluster name + customer account from the VM's

spinifex/domains/eks/cluster/eks_billable_reaper_test.go

@@ -1,14 +1,19 @@
 package cluster
 
 import (
+	"bytes"
 	"context"
+	"log/slog"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/elbv2"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -33,53 +38,119 @@ func cpVM(id, eniID string) *vm.VM {
 	return &vm.VM{ID: id, ManagedBy: tags.ManagedByEKS, ENIId: eniID, AccountID: reaperSysAccount, LastNode: "node-1"}
 }
 
-// TestRLC5_EKSBillableReaperTerminatesOrphanCPVM exercises the historic PROP-LIFECYCLE-006 §5 (behaviour per ADR-0003:S2)
+// ADR-0003:S7. A running control-plane VM whose cluster record cannot be found
-// meta-independent billable cleanup: a running EKS control-plane VM whose
+// is retained and alarmed, never terminated and never has its NLB or VPC torn
-// cluster meta is DEFINITIVELY GONE is a billable orphan and must be terminated
+// down: a missing record cannot tell a completed DeleteCluster from a lost
-// by the GC backstop — the real fix for the orphan CP VM surviving a
+// bucket or an out-of-band delete.
-// daemon restart after DeleteCluster swept the meta.
+func TestS7_EKSBillableReaperNeverTerminatesAControlPlane(t *testing.T) {
-func TestRLC5_EKSBillableReaperTerminatesOrphanCPVM(t *testing.T) {
+	const lostAccount = "999999999999"
-	t.Parallel()
+
-	f := newEKSServiceFixture(t)
+	cases := []struct {
-	f.vpc.describeByENI = map[string]*ec2.NetworkInterface{
+		name      string
-		"eni-orphan": cpENI("eni-orphan", "gone-cluster", testAccountID),
+		account   string
+		prepare   func(t *testing.T, f *eksServiceFixture)
+		wantAlarm string
+		wantLog   string
+	}{
+		{
+			name:      "account bucket missing",
+			account:   lostAccount,
+			prepare:   func(*testing.T, *eksServiceFixture) {},
+			wantAlarm: "account bucket missing",
+		},
+		{
+			name:      "bucket has no meta key",
+			account:   testAccountID,
+			prepare:   func(*testing.T, *eksServiceFixture) {},
+			wantAlarm: "cluster record missing",
+		},
+		{
+			name:    "meta removed by the teardown revision-checked delete",
+			account: testAccountID,
+			prepare: func(t *testing.T, f *eksServiceFixture) {
+				require.NoError(t, PutClusterMeta(t.Context(), f.kv, sampleClusterMeta("orphan")))
+				_, err := kvutil.DeleteIf(t.Context(), f.kv, ClusterMetaKey("orphan"), kvutil.CASConfig{
+					Attempts: maxClusterStateCASRetries,
+					NotFound: ErrClusterNotFound,
+				}, func(*ClusterMeta) error { return nil })
+				require.NoError(t, err)
+			},
+			wantAlarm: "cluster record missing",
+		},
+		{
+			name:    "meta removed out of band",
+			account: testAccountID,
+			prepare: func(t *testing.T, f *eksServiceFixture) {
+				require.NoError(t, PutClusterMeta(t.Context(), f.kv, sampleClusterMeta("orphan")))
+				require.NoError(t, f.kv.Delete(t.Context(), ClusterMetaKey("orphan")))
+			},
+			wantAlarm: "cluster record missing",
+		},
+		{
+			name:    "meta unreadable",
+			account: testAccountID,
+			prepare: func(t *testing.T, f *eksServiceFixture) {
+				_, err := f.kv.Put(t.Context(), ClusterMetaKey("orphan"), []byte("{not json"))
+				require.NoError(t, err)
+			},
+			wantLog: "cluster record read failed",
+		},
 	}
-	orphan := cpVM("i-orphan", "eni-orphan")
 
-	// Seed the orphan cluster's name/tag-driven billable infra the swept meta no
+	for _, tc := range cases {
-	// longer anchors: an NLB front-end and a tagged NAT gateway holding a billable
+		t.Run(tc.name, func(t *testing.T) {
-	// EIP. The reaper's post-terminate reclaim must sweep both.
+			// slog.SetDefault is process-wide, so this test is not parallel.
-	nlbName := ClusterNLBName("gone-cluster")
+			var buf bytes.Buffer
-	f.nlb.lbByName[nlbName] = &elbv2.LoadBalancer{
+			restore := slog.Default()
-		LoadBalancerArn:  aws.String("arn:lb-orphan"),
+			slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo})))
-		LoadBalancerName: aws.String(nlbName),
+			t.Cleanup(func() { slog.SetDefault(restore) })
+
+			f := newEKSServiceFixture(t)
+			f.vpc.describeByENI = map[string]*ec2.NetworkInterface{
+				"eni-cp": cpENI("eni-cp", "orphan", tc.account),
+			}
+			nlbName := ClusterNLBName("orphan")
+			f.nlb.lbByName[nlbName] = &elbv2.LoadBalancer{
+				LoadBalancerArn:  aws.String("arn:lb-orphan"),
+				LoadBalancerName: aws.String(nlbName),
+			}
+			f.ngw.gws = []*fakeCPNatGateway{{
+				id:    "nat-orphan",
+				tags:  map[string]string{clusterEKSClusterTagKey: "orphan", clusterEKSRoleTagKey: cpVPCRoles.NatGW},
+				state: "available",
+				addrs: []*ec2.NatGatewayAddress{{AllocationId: aws.String("eipalloc-orphan")}},
+			}}
+			tc.prepare(t, f)
+
+			reaper := f.svc.NewBillableReaper(func() ([]*vm.VM, error) { return []*vm.VM{cpVM("i-cp", "eni-cp")}, nil })
+			for range 2 {
+				removed, err := reaper.Sweep(context.Background())
+				require.NoError(t, err)
+				assert.Zero(t, removed, "ADR-0003:S7: the reaper removes nothing")
+			}
+
+			assert.Empty(t, f.inst.terminateCalls, "ADR-0003:S7: a reaper's judgement is never the decision to terminate")
+			assert.Empty(t, f.vpc.deleteCalls, "the control-plane ENI must stay")
+			assert.Contains(t, f.nlb.lbByName, nlbName, "the NLB must stay")
+			assert.Empty(t, f.eip.releaseCalls, "no EIP may be released")
+
+			js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+			if tc.account == lostAccount {
+				_, err := OpenAccountBucket(t.Context(), js, lostAccount)
+				assert.ErrorIs(t, err, jetstream.ErrBucketNotFound, "a sweep must not create the bucket it found missing")
+			}
+
+			out := buf.String()
+			if tc.wantAlarm != "" {
+				assert.Contains(t, out, "DATA-SAFETY ALARM")
+				assert.Contains(t, out, tc.wantAlarm)
+				assert.Contains(t, out, "i-cp")
+			} else {
+				assert.NotContains(t, out, "DATA-SAFETY ALARM", "an unreadable record is not a missing one")
+				assert.Contains(t, out, tc.wantLog)
+			}
+		})
 	}
-	f.ngw.gws = []*fakeCPNatGateway{{
-		id:    "nat-orphan",
-		tags:  map[string]string{clusterEKSClusterTagKey: "gone-cluster", clusterEKSRoleTagKey: cpVPCRoles.NatGW},
-		state: "available",
-		addrs: []*ec2.NatGatewayAddress{{AllocationId: aws.String("eipalloc-orphan")}},
-	}}
-
-	reaper := f.svc.NewBillableReaper(func() ([]*vm.VM, error) { return []*vm.VM{orphan}, nil })
-
-	reaped, err := reaper.Sweep(context.Background())
-	require.NoError(t, err)
-	assert.Equal(t, 1, reaped, "ADR-0003:S2 behaviour; historic PROP-LIFECYCLE-006 §5 mechanism: a CP VM whose cluster meta is gone must be reaped")
-	assert.Contains(t, f.inst.terminateCalls, "i-orphan", "the orphan CP VM must be terminated")
-	require.Len(t, f.vpc.deleteCalls, 1, "the orphan CP ENI must be deleted")
-	assert.Equal(t, "eni-orphan", aws.StringValue(f.vpc.deleteCalls[0].NetworkInterfaceId))
-
-	// The orphan's billable NLB + NAT-GW EIP must also be reclaimed.
-	assert.NotContains(t, f.nlb.lbByName, nlbName, "the orphan NLB front-end must be reclaimed")
-	require.Len(t, f.eip.releaseCalls, 1, "the orphan NAT-GW EIP must be released")
-	assert.Equal(t, "eipalloc-orphan", aws.StringValue(f.eip.releaseCalls[0].AllocationId))
-
-	// Idempotent: a repeat sweep over the same (still meta-absent) VM is safe —
-	// the terminate + reclaim paths tolerate already-gone infra (no second release).
-	_, err = reaper.Sweep(context.Background())
-	require.NoError(t, err)
-	assert.Len(t, f.eip.releaseCalls, 1, "reclaim must not double-release an already-freed EIP")
 }
 
 // TestRLC5_EKSBillableReaperSpareLiveAndUncertain enforces the reaper's no-false-

spinifex/domains/eks/cluster/store.go

@@ -185,7 +185,8 @@ func AccountWatchBuckets(ctx context.Context, nc *nats.Conn) ([]*kvstore.Bucket,
 	}
 	buckets := make([]*kvstore.Bucket, 0, len(names))
 	for _, name := range names {
-		buckets = append(buckets, kvstore.NewBucket(js, kvstore.Config{Name: name, History: 1}))
+		// Attach only: a watch on a deleted bucket must fail and retry, not recreate it.
+		buckets = append(buckets, kvstore.NewBucket(js, kvstore.Config{Name: name, History: 1, AttachOnly: true}))
 	}
 	return buckets, nil
 }
@@ -206,6 +207,13 @@ func GetOrCreateAccountBucket(ctx context.Context, js jetstream.JetStream, accou
 	return kv, nil
 }
 
+// OpenAccountBucket returns the per-account KV bucket for accountID without
+// creating it. A bucket that does not exist is jetstream.ErrBucketNotFound, so a
+// read-only caller never turns a lost bucket into an empty one.
+func OpenAccountBucket(ctx context.Context, js jetstream.JetStream, accountID string) (jetstream.KeyValue, error) {
+	return js.KeyValue(ctx, AccountBucketName(accountID))
+}
+
 // InitLeaderBucket creates (or attaches to) the shared spinifex-eks-leader
 // bucket used for per-cluster reconciler leader-lease CAS locks.
 func InitLeaderBucket(ctx context.Context, js jetstream.JetStream) (jetstream.KeyValue, error) {

spinifex/domains/eks/cluster/watch_source_test.go

@@ -7,6 +7,7 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit"
 	ekscluster "github.com/mulgadc/spinifex/spinifex/domains/eks/cluster"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -79,3 +80,26 @@ func TestAccountWatchBuckets_EnumerationErrorPropagates(t *testing.T) {
 	_, err := ekscluster.AccountWatchBuckets(ctx, nc)
 	require.Error(t, err)
 }
+
+// TestAccountWatchBuckets_WatchOnDeletedBucketDoesNotRecreateIt pins attach-only
+// handles: a DNS watch that outlives its bucket must fail and retry, because
+// recreating it hides a deleted account bucket behind an empty one.
+func TestAccountWatchBuckets_WatchOnDeletedBucketDoesNotRecreateIt(t *testing.T) {
+	t.Parallel()
+	_, nc, _ := testutil.StartTestJetStream(t)
+	js := testutil.NewJetStream(t, nc)
+	_, err := ekscluster.GetOrCreateAccountBucket(t.Context(), js, "111111111111")
+	require.NoError(t, err)
+
+	buckets, err := ekscluster.AccountWatchBuckets(t.Context(), nc)
+	require.NoError(t, err)
+	require.Len(t, buckets, 1)
+
+	require.NoError(t, js.DeleteKeyValue(t.Context(), ekscluster.AccountBucketName("111111111111")))
+
+	_, err = buckets[0].Watch(t.Context(), ">")
+	require.Error(t, err, "a watch on a deleted bucket must fail")
+
+	_, err = ekscluster.OpenAccountBucket(t.Context(), js, "111111111111")
+	assert.ErrorIs(t, err, jetstream.ErrBucketNotFound, "the watch must not have recreated the bucket")
+}

spinifex/domains/elbv2/agent_heartbeat_wire_test.go

@@ -0,0 +1,73 @@
+package elbv2
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/query"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLBAgentHeartbeatInput_OldAgentDecodesUnchanged(t *testing.T) {
+	t.Parallel()
+	// Exactly the parameters an agent without the data-path probe sends.
+	params := map[string]string{
+		"Action":                   "LBAgentHeartbeat",
+		"Version":                  "2015-12-01",
+		"LBID":                     "lb-abc123",
+		"Servers.member.1.Backend": "bk_tg1",
+		"Servers.member.1.Server":  "srv_i-web1",
+		"Servers.member.1.Status":  "UP",
+		"Servers.member.2.Backend": "bk_tg1",
+		"Servers.member.2.Server":  "srv_i-web2",
+		"Servers.member.2.Status":  "DOWN",
+	}
+
+	var in LBAgentHeartbeatInput
+	require.NoError(t, query.QueryParamsToStruct(params, &in))
+
+	require.NotNil(t, in.LBID)
+	assert.Equal(t, "lb-abc123", *in.LBID)
+	require.Len(t, in.Servers, 2)
+	assert.Equal(t, "DOWN", *in.Servers[1].Status)
+	assert.Nil(t, in.InstanceID, "an old agent reports no instance ID")
+	assert.Empty(t, in.NICs, "an old agent reports no NICs")
+	assert.Len(t, in.toHealthReport().Servers, 2)
+}
+
+func TestLBAgentHeartbeatInput_DecodesDataPathEvidence(t *testing.T) {
+	t.Parallel()
+	params := map[string]string{
+		"LBID":                              "lb-abc123",
+		"InstanceID":                        "i-0123456789abcdef0",
+		"NICs.member.1.MAC":                 "02:00:00:aa:00:01",
+		"NICs.member.1.ProbeOK":             "false",
+		"NICs.member.1.ConsecutiveFailures": "4",
+		"NICs.member.1.RxPackets":           "100",
+		"NICs.member.1.TxPackets":           "200",
+		"NICs.member.1.RxDropped":           "3",
+		"NICs.member.1.TxDropped":           "9",
+		"NICs.member.2.MAC":                 "02:00:00:aa:00:03",
+		"NICs.member.2.ProbeOK":             "true",
+		"NICs.member.2.ConsecutiveFailures": "0",
+	}
+
+	var in LBAgentHeartbeatInput
+	require.NoError(t, query.QueryParamsToStruct(params, &in))
+
+	require.NotNil(t, in.InstanceID)
+	assert.Equal(t, "i-0123456789abcdef0", *in.InstanceID)
+	require.Len(t, in.NICs, 2)
+
+	n := in.NICs[0]
+	assert.Equal(t, "02:00:00:aa:00:01", *n.MAC)
+	assert.False(t, *n.ProbeOK)
+	assert.EqualValues(t, 4, *n.ConsecutiveFailures)
+	assert.EqualValues(t, 100, *n.RxPackets)
+	assert.EqualValues(t, 200, *n.TxPackets)
+	assert.EqualValues(t, 3, *n.RxDropped)
+	assert.EqualValues(t, 9, *n.TxDropped)
+
+	assert.True(t, *in.NICs[1].ProbeOK)
+	assert.Nil(t, in.NICs[1].RxPackets, "an omitted counter stays unset")
+}

spinifex/domains/elbv2/agent_types.go

@@ -7,9 +7,25 @@ import (
 
 // LBAgentHeartbeatInput is sent by the LB agent on each heartbeat tick,
 // including its health report so the daemon can update target health.
+//
+// InstanceID and NICs are optional: an older agent sends neither.
 type LBAgentHeartbeatInput struct {
-	LBID    *string                `locationName:"LBID" type:"string"`
+	LBID       *string                `locationName:"LBID" type:"string"`
-	Servers []*LBAgentServerStatus `locationName:"Servers" type:"list"`
+	Servers    []*LBAgentServerStatus `locationName:"Servers" type:"list"`
+	InstanceID *string                `locationName:"InstanceID" type:"string"`
+	NICs       []*LBAgentNICStatus    `locationName:"NICs" type:"list"`
+}
+
+// LBAgentNICStatus is the gateway probe result and interface counters of one
+// data NIC, as reported by the LB agent.
+type LBAgentNICStatus struct {
+	MAC                 *string `locationName:"MAC" type:"string"`
+	ProbeOK             *bool   `locationName:"ProbeOK" type:"boolean"`
+	ConsecutiveFailures *int64  `locationName:"ConsecutiveFailures" type:"long"`
+	RxPackets           *int64  `locationName:"RxPackets" type:"long"`
+	TxPackets           *int64  `locationName:"TxPackets" type:"long"`
+	RxDropped           *int64  `locationName:"RxDropped" type:"long"`
+	TxDropped           *int64  `locationName:"TxDropped" type:"long"`
 }
 
 // LBAgentServerStatus represents a single backend server's health status as reported by the LB agent.

spinifex/domains/elbv2/launcher_test.go

@@ -14,6 +14,9 @@ type mockSystemInstanceLauncher struct {
 	launchErr      error
 	terminateErr   error
 	terminateDone  chan struct{} // closed after TerminateSystemInstance completes
+	onTerminate    func(instanceID string)
+	rebootCalls    []string
+	rebootErr      error
 }
 
 func (m *mockSystemInstanceLauncher) LaunchSystemInstance(input *SystemInstanceInput) (*SystemInstanceOutput, error) {
@@ -25,12 +28,22 @@ func (m *mockSystemInstanceLauncher) TerminateSystemInstance(instanceID string)
 	m.mu.Lock()
 	m.terminateCalls = append(m.terminateCalls, instanceID)
 	m.mu.Unlock()
+	if m.onTerminate != nil {
+		m.onTerminate(instanceID)
+	}
 	if m.terminateDone != nil {
 		close(m.terminateDone)
 	}
 	return m.terminateErr
 }
 
+func (m *mockSystemInstanceLauncher) RebootSystemInstance(instanceID string) error {
+	m.mu.Lock()
+	m.rebootCalls = append(m.rebootCalls, instanceID)
+	m.mu.Unlock()
+	return m.rebootErr
+}
+
 // waitTerminate blocks until TerminateSystemInstance has been called.
 func (m *mockSystemInstanceLauncher) waitTerminate() {
 	if m.terminateDone != nil {

spinifex/domains/elbv2/lb_converge_test.go

@@ -0,0 +1,192 @@
+package elbv2
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLBRepair_AnotherReservationWinsTheFirstCAS(t *testing.T) {
+	t.Parallel()
+	e := newRepairEnv(t)
+	lb := e.seed("lbr20", true)
+	cands := e.candidates()
+	require.Len(t, cands, 1)
+	require.True(t, e.rec.eligible(context.Background(), cands[0], e.clock))
+
+	_, err := e.svc.store.UpdateLBHealth(context.Background(), lb.LoadBalancerID, func(rec *LBDataPathRecord) (bool, error) {
+		rec.Repairs = append(rec.Repairs, LBRepair{At: e.clock, InstanceID: lb.InstanceID, Outcome: RepairReserved})
+		return true, nil
+	})
+	require.NoError(t, err)
+
+	assert.False(t, e.rec.repair(context.Background(), cands[0]))
+	assert.Empty(t, e.reboots())
+	assert.Len(t, e.health(lb.LoadBalancerID).Repairs, 1, "the losing reservation must not add an entry")
+}
+
+func TestLBRepair_LeaseLostAfterReservationKeepsItCountedAndDoesNotDispatch(t *testing.T) {
+	t.Parallel()
+	e := newRepairEnv(t)
+	e.seed("lbr21", true)
+	cands := e.candidates()
+	require.Len(t, cands, 1)
+
+	calls := 0
+	e.rec.isLeader = func() bool {
+		calls++
+		return calls == 1
+	}
+	assert.True(t, e.rec.repair(context.Background(), cands[0]), "a written reservation is reported as spent")
+	assert.Empty(t, e.reboots())
+
+	repairs := e.health("lbr21").Repairs
+	require.Len(t, repairs, 1)
+	assert.Equal(t, RepairReserved, repairs[0].Outcome, "the entry stays counted with no outcome")
+}
+
+func TestLBRepair_NonLeaderNeverReserves(t *testing.T) {
+	t.Parallel()
+	e := newRepairEnv(t)
+	e.seed("lbr22", true)
+	cands := e.candidates()
+	require.Len(t, cands, 1)
+
+	e.rec.isLeader = func() bool { return false }
+	assert.False(t, e.rec.repair(context.Background(), cands[0]))
+	assert.Empty(t, e.health("lbr22").Repairs)
+}
+
+func TestLBHealth_FailedResumeThenNewVMHeartbeatsConverge(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+	lb := seedHealthLB(t, svc)
+	ctx := context.Background()
+
+	// The old VM's evidence was suspended by SetSubnets, whose resume write then failed.
+	_, err := svc.store.UpdateLBHealth(ctx, lb.LoadBalancerID, func(rec *LBDataPathRecord) (bool, error) {
+		rec.InstanceID = "i-oldvm"
+		rec.Suspended = true
+		rec.SuspendedAt = time.Now().UTC()
+		return true, nil
+	})
+	require.NoError(t, err)
+
+	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, true, 0, 10))
+	rec, err := svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
+	require.NoError(t, err)
+	assert.False(t, rec.Suspended, "a heartbeat from the VM the LB names lifts the stale suspension")
+	assert.Equal(t, healthInst, rec.InstanceID)
+
+	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, 3, 11))
+	rec, err = svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
+	require.NoError(t, err)
+	require.Contains(t, rec.NICs, healthMACData, "a later NIC failure is recorded and repairable")
+}
+
+func TestLBHealth_SuspensionIsNotLiftedByAHeartbeatFromAVMTheLBDoesNotName(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+	lb := seedHealthLB(t, svc)
+	ctx := context.Background()
+
+	_, err := svc.store.UpdateLBHealth(ctx, lb.LoadBalancerID, func(rec *LBDataPathRecord) (bool, error) {
+		rec.InstanceID = "i-oldvm"
+		rec.Suspended = true
+		rec.SuspendedAt = time.Now().UTC()
+		return true, nil
+	})
+	require.NoError(t, err)
+
+	healthBeat(t, svc, aws.String("i-oldvm"), nicStatus(healthMACData, false, 3, 1))
+	rec, err := svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
+	require.NoError(t, err)
+	assert.True(t, rec.Suspended)
+	assert.Equal(t, "i-oldvm", rec.InstanceID)
+}
+
+func TestLBHealth_SuspensionBackstopLiftsOnlyAfterThirtyMinutes(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+	lb := seedHealthLB(t, svc)
+	ctx := context.Background()
+
+	suspend := func(age time.Duration) {
+		_, err := svc.store.UpdateLBHealth(ctx, lb.LoadBalancerID, func(rec *LBDataPathRecord) (bool, error) {
+			rec.InstanceID = healthInst
+			rec.Suspended = true
+			rec.SuspendedAt = time.Now().UTC().Add(-age)
+			return true, nil
+		})
+		require.NoError(t, err)
+	}
+
+	suspend(29 * time.Minute)
+	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, 3, 1))
+	rec, err := svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
+	require.NoError(t, err)
+	assert.True(t, rec.Suspended, "inside the backstop the suspension holds")
+
+	suspend(31 * time.Minute)
+	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, 3, 2))
+	rec, err = svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
+	require.NoError(t, err)
+	assert.False(t, rec.Suspended, "past the backstop the current VM's heartbeat lifts it")
+	assert.Contains(t, rec.NICs, healthMACData)
+}
+
+func TestLBHealth_EstablishedFailureSurvivesAnAgentRestart(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+	lb := seedHealthLB(t, svc)
+	ctx := context.Background()
+
+	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, 3, 1))
+	first, err := svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
+	require.NoError(t, err)
+	since := first.NICs[healthMACData].FailingSince
+
+	// A restarted agent reports healthy until it has missed three probes again.
+	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, true, 1, 2))
+	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, true, 2, 3))
+	rec, err := svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
+	require.NoError(t, err)
+	require.Contains(t, rec.NICs, healthMACData)
+	assert.True(t, since.Equal(rec.NICs[healthMACData].FailingSince), "FailingSince is kept")
+
+	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, true, 0, 4))
+	rec, err = svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
+	require.NoError(t, err)
+	assert.Empty(t, rec.NICs, "only a clean probe clears the failure")
+}
+
+func TestLBHealth_OldVMHeartbeatPausedAcrossPublicationLeavesTheNewBinding(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+	lb := seedHealthLB(t, svc)
+	ctx := context.Background()
+
+	// SetSubnets has already published the new VM and rebound the evidence.
+	lb.InstanceID = "i-newvm"
+	require.NoError(t, svc.store.PutLoadBalancer(ctx, lb))
+	require.NoError(t, svc.store.ResumeLBHealth(ctx, lb.LoadBalancerID, "i-newvm"))
+
+	// The old VM's heartbeat read the LB before that and runs now.
+	stale := *lb
+	stale.InstanceID = healthInst
+	svc.recordDataPathEvidence(ctx, &stale, &LBAgentHeartbeatInput{
+		LBID:       aws.String(lb.LoadBalancerID),
+		InstanceID: aws.String(healthInst),
+		NICs:       []*LBAgentNICStatus{nicStatus(healthMACData, false, 5, 1)},
+	})
+
+	rec, err := svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
+	require.NoError(t, err)
+	require.NotNil(t, rec)
+	assert.Equal(t, "i-newvm", rec.InstanceID)
+	assert.Empty(t, rec.NICs, "the old VM's failure must not land on the new binding")
+}

spinifex/domains/elbv2/lb_health.go

@@ -0,0 +1,333 @@
+package elbv2
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+const (
+	// KeyPrefixLBHealth holds the data-path evidence of one load balancer. It is
+	// a separate key from the LB record so whole-record writes cannot drop it.
+	KeyPrefixLBHealth = "lbhealth."
+
+	// StateActiveImpaired is what DescribeLoadBalancers reports while a data NIC
+	// of the LB's current VM is failing its gateway probe.
+	StateActiveImpaired = "active_impaired"
+
+	// lbSuspendBackstop is how long a suspension may outlive its mutation before
+	// a heartbeat from the current VM lifts it.
+	lbSuspendBackstop = 30 * time.Minute
+)
+
+// Outcomes of a recorded repair. Reserved is written before the repair is
+// dispatched, so an entry whose outcome was never updated still counts.
+const (
+	RepairReserved    = "reserved"
+	RepairDispatched  = "dispatched"
+	RepairRefused     = "refused"
+	RepairUnreachable = "unreachable"
+)
+
+// LBNICFailure is one data NIC that was failing its gateway probe when the
+// transition was recorded.
+type LBNICFailure struct {
+	ENIID               string    `json:"eni_id,omitempty"`
+	FailingSince        time.Time `json:"failing_since"`
+	ConsecutiveFailures int64     `json:"consecutive_failures"`
+	RxPackets           int64     `json:"rx_packets"`
+	TxPackets           int64     `json:"tx_packets"`
+	RxDropped           int64     `json:"rx_dropped"`
+	TxDropped           int64     `json:"tx_dropped"`
+}
+
+// LBRepair is one repair of an LB's VM and how it ended.
+type LBRepair struct {
+	At         time.Time `json:"at"`
+	InstanceID string    `json:"instance_id"`
+	ENIID      string    `json:"eni_id,omitempty"`
+	Reason     string    `json:"reason,omitempty"`
+	Outcome    string    `json:"outcome"`
+}
+
+// LBDataPathRecord is the data-path evidence of one load balancer VM. NICs holds
+// only failing NICs, keyed by lower-case MAC; an absent NIC is healthy.
+type LBDataPathRecord struct {
+	InstanceID  string                   `json:"instance_id"`
+	Suspended   bool                     `json:"suspended,omitempty"`
+	SuspendedAt time.Time                `json:"suspended_at,omitzero"`
+	BoundAt     time.Time                `json:"bound_at,omitzero"`
+	NICs        map[string]*LBNICFailure `json:"nics,omitempty"`
+	Repairs     []LBRepair               `json:"repairs,omitempty"`
+}
+
+// failing reports whether any NIC of the record is failing.
+func (r *LBDataPathRecord) failing() bool {
+	return r != nil && len(r.NICs) > 0
+}
+
+// GetLBHealth returns the evidence record of a load balancer, or nil when none
+// has been written.
+func (s *Store) GetLBHealth(ctx context.Context, lbID string) (*LBDataPathRecord, error) {
+	entry, err := s.kv.Get(ctx, KeyPrefixLBHealth+lbID)
+	if err != nil {
+		if errors.Is(err, jetstream.ErrKeyNotFound) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	var rec LBDataPathRecord
+	if err := json.Unmarshal(entry.Value(), &rec); err != nil {
+		return nil, fmt.Errorf("unmarshal lb health: %w", err)
+	}
+	return &rec, nil
+}
+
+// UpdateLBHealth applies mutate to the evidence record under optimistic
+// concurrency, creating it when absent. mutate returns false to write nothing.
+func (s *Store) UpdateLBHealth(ctx context.Context, lbID string, mutate func(*LBDataPathRecord) (bool, error)) (*LBDataPathRecord, error) {
+	return kvutil.Update(ctx, s.kv, KeyPrefixLBHealth+lbID, kvutil.CASConfig{
+		Attempts:       targetGroupCASAttempts,
+		CreateIfAbsent: true,
+		Exhausted: func(_ string, attempts int) error {
+			return fmt.Errorf("elbv2: lb health %s contended, CAS exhausted after %d attempts", lbID, attempts)
+		},
+	}, mutate)
+}
+
+// DeleteLBHealth removes the evidence record of a load balancer.
+func (s *Store) DeleteLBHealth(ctx context.Context, lbID string) error {
+	err := s.kv.Delete(ctx, KeyPrefixLBHealth+lbID)
+	if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
+		return err
+	}
+	return nil
+}
+
+// SuspendLBHealth marks the load balancer as being torn down or reshaped, so
+// no repair is admitted against its VM until the mutation finishes.
+func (s *Store) SuspendLBHealth(ctx context.Context, lbID string) error {
+	_, err := s.UpdateLBHealth(ctx, lbID, func(rec *LBDataPathRecord) (bool, error) {
+		if rec.Suspended {
+			return false, nil
+		}
+		rec.Suspended = true
+		rec.SuspendedAt = time.Now().UTC()
+		return true, nil
+	})
+	return err
+}
+
+// ResumeLBHealth lifts a suspension. A non-empty instanceID also binds the
+// record to that VM and drops the old VM's failing NICs.
+func (s *Store) ResumeLBHealth(ctx context.Context, lbID, instanceID string) error {
+	_, err := s.UpdateLBHealth(ctx, lbID, func(rec *LBDataPathRecord) (bool, error) {
+		changed := rec.Suspended
+		rec.Suspended = false
+		rec.SuspendedAt = time.Time{}
+		if instanceID != "" && rec.InstanceID != instanceID {
+			rec.InstanceID = instanceID
+			rec.BoundAt = time.Now().UTC()
+			rec.NICs = nil
+			changed = true
+		}
+		return changed, nil
+	})
+	return err
+}
+
+// failingNICMACs returns the lower-case MACs that are failing. A probe that
+// failed starts a failure. An established failure ends only when the latest
+// probe succeeded, because an agent restarted by a repair reports healthy for
+// its first misses.
+func failingNICMACs(nics []*LBAgentNICStatus, established map[string]*LBNICFailure) map[string]*LBAgentNICStatus {
+	out := make(map[string]*LBAgentNICStatus)
+	for _, n := range nics {
+		if n == nil || n.MAC == nil || n.ProbeOK == nil {
+			continue
+		}
+		mac := strings.ToLower(*n.MAC)
+		if !*n.ProbeOK || (established[mac] != nil && aws.Int64Value(n.ConsecutiveFailures) != 0) {
+			out[mac] = n
+		}
+	}
+	return out
+}
+
+// recordDataPathEvidence stores the data-path evidence of one heartbeat. It
+// writes only when the set of failing NICs changes, and ignores evidence that
+// does not come from the VM the LB record currently names.
+func (s *ELBv2ServiceImpl) recordDataPathEvidence(ctx context.Context, lb *LoadBalancerRecord, input *LBAgentHeartbeatInput) {
+	if lb.InstanceID == "" || input.InstanceID == nil || *input.InstanceID != lb.InstanceID || len(input.NICs) == 0 {
+		return
+	}
+
+	rec, err := s.store.GetLBHealth(ctx, lb.LoadBalancerID)
+	if err != nil {
+		slog.WarnContext(ctx, "LBAgentHeartbeat: failed to read data-path evidence", "lbId", lb.LoadBalancerID, "err", err)
+		return
+	}
+	converge := false
+	if rec != nil && rec.Suspended {
+		if !s.suspensionConverges(ctx, lb, rec, *input.InstanceID) {
+			return
+		}
+		converge = true
+	}
+	var known map[string]*LBNICFailure
+	if rec != nil && rec.InstanceID == lb.InstanceID {
+		known = rec.NICs
+	}
+	failing := failingNICMACs(input.NICs, known)
+	if !converge && sameMACs(known, failing) && (len(failing) == 0 || rec != nil) {
+		return
+	}
+
+	eniByMAC := s.resolveLBENIs(ctx, lb, failing)
+	now := time.Now().UTC()
+	_, err = s.store.UpdateLBHealth(ctx, lb.LoadBalancerID, func(cur *LBDataPathRecord) (bool, error) {
+		changed := false
+		if cur.Suspended {
+			if !converge || !suspensionLifts(cur, lb.InstanceID, now) {
+				return false, nil
+			}
+			cur.Suspended = false
+			cur.SuspendedAt = time.Time{}
+			changed = true
+			if cur.InstanceID != lb.InstanceID {
+				cur.InstanceID = lb.InstanceID
+				cur.BoundAt = now
+				cur.NICs = nil
+			}
+		}
+		if cur.InstanceID != "" && cur.InstanceID != lb.InstanceID {
+			return changed, nil
+		}
+		prev := cur.NICs
+		failing := failingNICMACs(input.NICs, prev)
+		if cur.InstanceID == lb.InstanceID && sameMACs(prev, failing) {
+			return changed, nil
+		}
+		next := make(map[string]*LBNICFailure, len(failing))
+		for mac, n := range failing {
+			f := &LBNICFailure{
+				ENIID:               eniByMAC[mac],
+				FailingSince:        now,
+				ConsecutiveFailures: aws.Int64Value(n.ConsecutiveFailures),
+				RxPackets:           aws.Int64Value(n.RxPackets),
+				TxPackets:           aws.Int64Value(n.TxPackets),
+				RxDropped:           aws.Int64Value(n.RxDropped),
+				TxDropped:           aws.Int64Value(n.TxDropped),
+			}
+			if old, ok := prev[mac]; ok {
+				f.FailingSince = old.FailingSince
+			}
+			next[mac] = f
+		}
+		cur.InstanceID = lb.InstanceID
+		cur.NICs = next
+		return true, nil
+	})
+	if err != nil {
+		slog.WarnContext(ctx, "LBAgentHeartbeat: failed to store data-path evidence", "lbId", lb.LoadBalancerID, "err", err)
+		return
+	}
+	for mac, n := range failing {
+		if _, ok := known[mac]; !ok {
+			slog.WarnContext(ctx, "ELBv2 data-path probe failing",
+				"lbArn", lb.LoadBalancerArn, "lbId", lb.LoadBalancerID, "instanceId", lb.InstanceID,
+				"eniId", eniByMAC[mac], "mac", mac, "consecutive_failures", aws.Int64Value(n.ConsecutiveFailures),
+				"rx_packets", aws.Int64Value(n.RxPackets), "tx_packets", aws.Int64Value(n.TxPackets),
+				"rx_dropped", aws.Int64Value(n.RxDropped), "tx_dropped", aws.Int64Value(n.TxDropped))
+		}
+	}
+	for mac := range known {
+		if _, ok := failing[mac]; !ok {
+			slog.InfoContext(ctx, "ELBv2 data-path probe recovered",
+				"lbArn", lb.LoadBalancerArn, "lbId", lb.LoadBalancerID, "instanceId", lb.InstanceID, "mac", mac)
+		}
+	}
+}
+
+// suspensionLifts reports whether a suspended record may be lifted by a
+// heartbeat from instanceID: it is bound to an older VM, or it has outlived the
+// backstop while bound to the current one.
+func suspensionLifts(rec *LBDataPathRecord, instanceID string, now time.Time) bool {
+	if rec.InstanceID != "" && rec.InstanceID != instanceID {
+		return true
+	}
+	return !rec.SuspendedAt.IsZero() && now.Sub(rec.SuspendedAt) >= lbSuspendBackstop
+}
+
+// suspensionConverges confirms against a fresh read of the LB record that the
+// VM the heartbeat reports is the one the LB currently names.
+func (s *ELBv2ServiceImpl) suspensionConverges(ctx context.Context, lb *LoadBalancerRecord, rec *LBDataPathRecord, heartbeatInstance string) bool {
+	if !suspensionLifts(rec, heartbeatInstance, time.Now().UTC()) {
+		return false
+	}
+	fresh, err := s.store.GetLoadBalancer(ctx, lb.LoadBalancerID)
+	return err == nil && fresh != nil && fresh.InstanceID == heartbeatInstance
+}
+
+// sameMACs reports whether two failing sets name the same NICs.
+func sameMACs[A, B any](a map[string]A, b map[string]B) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for mac := range a {
+		if _, ok := b[mac]; !ok {
+			return false
+		}
+	}
+	return true
+}
+
+// resolveLBENIs maps failing MACs to ENI IDs from the LB's cross-account ENIs
+// and one describe of its own ENIs. A MAC it cannot resolve maps to nothing.
+func (s *ELBv2ServiceImpl) resolveLBENIs(ctx context.Context, lb *LoadBalancerRecord, failing map[string]*LBAgentNICStatus) map[string]string {
+	out := make(map[string]string, len(failing))
+	for _, x := range lb.CrossAccountENIs {
+		mac := strings.ToLower(x.ENIMac)
+		if _, ok := failing[mac]; ok {
+			out[mac] = x.ENIID
+		}
+	}
+	if len(out) == len(failing) {
+		return out
+	}
+	for id, eni := range s.describeENIs(ctx, lb.ENIs, lb.AccountID) {
+		mac := strings.ToLower(aws.StringValue(eni.MacAddress))
+		if _, ok := failing[mac]; ok && out[mac] == "" {
+			out[mac] = id
+		}
+	}
+	return out
+}
+
+// impairedByEvidence reports whether the LB's current VM has a failing data NIC.
+func (s *ELBv2ServiceImpl) impairedByEvidence(ctx context.Context, r *LoadBalancerRecord) bool {
+	if r.State != StateActive || r.InstanceID == "" {
+		return false
+	}
+	rec, err := s.store.GetLBHealth(ctx, r.LoadBalancerID)
+	if err != nil {
+		slog.DebugContext(ctx, "lbRecordToSDK: failed to read data-path evidence", "lbId", r.LoadBalancerID, "err", err)
+		return false
+	}
+	return rec != nil && !rec.Suspended && rec.InstanceID == r.InstanceID && rec.failing()
+}
+
+// resumeLBHealth lifts a suspension after a failure that left the VM untouched.
+func (s *ELBv2ServiceImpl) resumeLBHealth(ctx context.Context, lbID string) {
+	if err := s.store.ResumeLBHealth(ctx, lbID, ""); err != nil {
+		slog.WarnContext(ctx, "failed to lift data-path suspension", "lbId", lbID, "err", err)
+	}
+}

spinifex/domains/elbv2/lb_health_test.go

@@ -0,0 +1,370 @@
+package elbv2
+
+import (
+	"context"
+	"errors"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/elbv2"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	healthLBID    = "lbhealth01"
+	healthInst    = "i-lbvm01"
+	healthMACData = "02:00:00:aa:00:01"
+	healthMACX    = "02:00:00:aa:00:02"
+)
+
+func seedHealthLB(t *testing.T, svc *ELBv2ServiceImpl) *LoadBalancerRecord {
+	t.Helper()
+	lb := &LoadBalancerRecord{
+		LoadBalancerArn: "arn:aws:elasticloadbalancing:us-east-1:" + testAccountID + ":loadbalancer/app/health-lb/" + healthLBID,
+		LoadBalancerID:  healthLBID,
+		Name:            "health-lb",
+		State:           StateActive,
+		InstanceID:      healthInst,
+		AccountID:       testAccountID,
+		CrossAccountENIs: []ExtraENIInput{
+			{ENIID: "eni-cross01", ENIMac: healthMACX},
+		},
+		CreatedAt: time.Now().UTC(),
+	}
+	require.NoError(t, svc.store.PutLoadBalancer(context.Background(), lb))
+	return lb
+}
+
+func nicStatus(mac string, ok bool, failures, rx int64) *LBAgentNICStatus {
+	return &LBAgentNICStatus{
+		MAC:                 aws.String(mac),
+		ProbeOK:             aws.Bool(ok),
+		ConsecutiveFailures: aws.Int64(failures),
+		RxPackets:           aws.Int64(rx),
+		TxPackets:           aws.Int64(rx + 5),
+		RxDropped:           aws.Int64(0),
+		TxDropped:           aws.Int64(2),
+	}
+}
+
+func healthBeat(t *testing.T, svc *ELBv2ServiceImpl, instanceID *string, nics ...*LBAgentNICStatus) {
+	t.Helper()
+	_, err := svc.LBAgentHeartbeat(context.Background(), &LBAgentHeartbeatInput{
+		LBID:       aws.String(healthLBID),
+		InstanceID: instanceID,
+		NICs:       nics,
+	}, testAccountID)
+	require.NoError(t, err)
+}
+
+func healthRevision(t *testing.T, svc *ELBv2ServiceImpl) uint64 {
+	t.Helper()
+	entry, err := svc.store.kv.Get(context.Background(), KeyPrefixLBHealth+healthLBID)
+	require.NoError(t, err)
+	return entry.Revision()
+}
+
+func TestLBHealth_SteadyHeartbeatsWriteNothing(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+	seedHealthLB(t, svc)
+
+	for range 5 {
+		healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, true, 0, 100))
+	}
+	rec, err := svc.store.GetLBHealth(context.Background(), healthLBID)
+	require.NoError(t, err)
+	assert.Nil(t, rec, "healthy heartbeats must not create an evidence record")
+
+	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, 3, 100))
+	rev := healthRevision(t, svc)
+	for i := range 5 {
+		healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, int64(4+i), int64(100+i)))
+	}
+	assert.Equal(t, rev, healthRevision(t, svc), "a NIC that stays failing must not rewrite the record")
+}
+
+func TestLBHealth_FailingNICWritesFailingSinceOnceAndRecoveryClears(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+	seedHealthLB(t, svc)
+
+	before := time.Now().UTC().Add(-time.Second)
+	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, 3, 777), nicStatus(healthMACX, true, 0, 1))
+
+	rec, err := svc.store.GetLBHealth(context.Background(), healthLBID)
+	require.NoError(t, err)
+	require.NotNil(t, rec)
+	assert.Equal(t, healthInst, rec.InstanceID)
+	require.Len(t, rec.NICs, 1)
+	f := rec.NICs[healthMACData]
+	require.NotNil(t, f)
+	assert.True(t, f.FailingSince.After(before))
+	assert.EqualValues(t, 777, f.RxPackets)
+	assert.EqualValues(t, 782, f.TxPackets)
+	assert.EqualValues(t, 2, f.TxDropped)
+	assert.EqualValues(t, 3, f.ConsecutiveFailures)
+
+	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, true, 0, 900))
+	rec, err = svc.store.GetLBHealth(context.Background(), healthLBID)
+	require.NoError(t, err)
+	require.NotNil(t, rec)
+	assert.Empty(t, rec.NICs, "recovery clears the failing NIC")
+}
+
+func TestLBHealth_SecondNICFailingKeepsFirstFailingSince(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+	seedHealthLB(t, svc)
+
+	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, 3, 1))
+	first, err := svc.store.GetLBHealth(context.Background(), healthLBID)
+	require.NoError(t, err)
+	since := first.NICs[healthMACData].FailingSince
+
+	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, 9, 2), nicStatus(healthMACX, false, 3, 2))
+	rec, err := svc.store.GetLBHealth(context.Background(), healthLBID)
+	require.NoError(t, err)
+	require.Len(t, rec.NICs, 2)
+	assert.True(t, since.Equal(rec.NICs[healthMACData].FailingSince))
+	assert.Equal(t, "eni-cross01", rec.NICs[healthMACX].ENIID, "a cross-account NIC resolves to its ENI without a describe")
+}
+
+func TestLBHealth_WrongOrMissingInstanceIDIsIgnored(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+	seedHealthLB(t, svc)
+
+	failing := nicStatus(healthMACData, false, 5, 1)
+	healthBeat(t, svc, aws.String("i-someoneelse"), failing)
+	healthBeat(t, svc, aws.String(""), failing)
+	healthBeat(t, svc, nil, failing)
+
+	rec, err := svc.store.GetLBHealth(context.Background(), healthLBID)
+	require.NoError(t, err)
+	assert.Nil(t, rec, "evidence from another VM, or with no instance ID, must be ignored")
+}
+
+func TestLBHealth_WholeRecordPutLeavesEvidenceIntact(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+	lb := seedHealthLB(t, svc)
+
+	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, 3, 1))
+	want, err := svc.store.GetLBHealth(context.Background(), healthLBID)
+	require.NoError(t, err)
+	rev := healthRevision(t, svc)
+
+	lb.StateReason = "rewritten by an older daemon"
+	require.NoError(t, svc.store.PutLoadBalancer(context.Background(), lb))
+
+	got, err := svc.store.GetLBHealth(context.Background(), healthLBID)
+	require.NoError(t, err)
+	assert.Equal(t, want, got)
+	assert.Equal(t, rev, healthRevision(t, svc))
+}
+
+func TestLBHealth_DeleteLoadBalancerRemovesTheKey(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+	lb := seedHealthLB(t, svc)
+	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, 3, 1))
+
+	_, err := svc.DeleteLoadBalancer(context.Background(), &elbv2.DeleteLoadBalancerInput{
+		LoadBalancerArn: aws.String(lb.LoadBalancerArn),
+	}, testAccountID)
+	require.NoError(t, err)
+
+	rec, err := svc.store.GetLBHealth(context.Background(), healthLBID)
+	require.NoError(t, err)
+	assert.Nil(t, rec)
+}
+
+func TestLBHealth_DeleteLoadBalancerSuspendsBeforeTerminatingTheVM(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+	lb := seedHealthLB(t, svc)
+	seen := make(chan bool, 1)
+	svc.InstanceLauncher = &mockSystemInstanceLauncher{
+		onTerminate: func(string) {
+			rec, err := svc.store.GetLBHealth(context.Background(), healthLBID)
+			seen <- err == nil && rec != nil && rec.Suspended
+		},
+	}
+
+	_, err := svc.DeleteLoadBalancer(context.Background(), &elbv2.DeleteLoadBalancerInput{
+		LoadBalancerArn: aws.String(lb.LoadBalancerArn),
+	}, testAccountID)
+	require.NoError(t, err)
+
+	select {
+	case suspended := <-seen:
+		assert.True(t, suspended, "repair must be excluded before the VM is terminated")
+	case <-time.After(5 * time.Second):
+		t.Fatal("the LB VM was never terminated")
+	}
+}
+
+func TestLBHealth_SetSubnetsSuspendsFirstAndResumesOnTheNewVM(t *testing.T) {
+	t.Parallel()
+	svc, vpcSvc, mock := setupSubnetTestService(t)
+	vid := vpcID(t, vpcSvc)
+	sub1 := getTestSubnetID(t, vpcSvc, vid, "10.0.40.0/24", "us-east-1a")
+	sub2 := getTestSubnetID(t, vpcSvc, vid, "10.0.41.0/24", "us-east-1b")
+
+	out, err := svc.CreateLoadBalancer(context.Background(), &elbv2.CreateLoadBalancerInput{
+		Name:    aws.String("health-sub-lb"),
+		Subnets: []*string{aws.String(sub1)},
+	}, testAccountID)
+	require.NoError(t, err)
+	svc.WaitLaunches()
+	lbID := aws.StringValue(out.LoadBalancers[0].LoadBalancerArn)
+	rec0, err := svc.store.GetLoadBalancerByArn(context.Background(), lbID)
+	require.NoError(t, err)
+	require.Equal(t, "i-lb", rec0.InstanceID)
+
+	_, err = svc.LBAgentHeartbeat(context.Background(), &LBAgentHeartbeatInput{
+		LBID:       aws.String(rec0.LoadBalancerID),
+		InstanceID: aws.String("i-lb"),
+		NICs:       []*LBAgentNICStatus{nicStatus(healthMACData, false, 3, 1)},
+	}, testAccountID)
+	require.NoError(t, err)
+
+	suspendedAtTerminate := false
+	mock.onTerminate = func(string) {
+		rec, gerr := svc.store.GetLBHealth(context.Background(), rec0.LoadBalancerID)
+		suspendedAtTerminate = gerr == nil && rec != nil && rec.Suspended
+	}
+	mock.launchResult = &SystemInstanceOutput{InstanceID: "i-new", PrivateIP: "10.0.1.6"}
+
+	_, err = svc.SetSubnets(context.Background(), &elbv2.SetSubnetsInput{
+		LoadBalancerArn: aws.String(lbID),
+		Subnets:         []*string{aws.String(sub1), aws.String(sub2)},
+	}, testAccountID)
+	require.NoError(t, err)
+	assert.True(t, suspendedAtTerminate, "repair must be excluded before the VM is terminated")
+
+	rec, err := svc.store.GetLBHealth(context.Background(), rec0.LoadBalancerID)
+	require.NoError(t, err)
+	require.NotNil(t, rec)
+	assert.False(t, rec.Suspended, "the suspension is lifted once the new VM is published")
+	assert.Equal(t, "i-new", rec.InstanceID)
+	assert.Empty(t, rec.NICs, "the old VM's failing NIC does not carry over")
+}
+
+func TestLBHealth_SetSubnetsFailureBeforeTheVMIsTouchedResumes(t *testing.T) {
+	t.Parallel()
+	svc, vpcSvc, mock := setupSubnetTestService(t)
+	vid := vpcID(t, vpcSvc)
+	sub1 := getTestSubnetID(t, vpcSvc, vid, "10.0.42.0/24", "us-east-1a")
+	sub2 := getTestSubnetID(t, vpcSvc, vid, "10.0.43.0/24", "us-east-1b")
+
+	out, err := svc.CreateLoadBalancer(context.Background(), &elbv2.CreateLoadBalancerInput{
+		Name:    aws.String("health-resume-lb"),
+		Subnets: []*string{aws.String(sub1)},
+	}, testAccountID)
+	require.NoError(t, err)
+	svc.WaitLaunches()
+	arn := aws.StringValue(out.LoadBalancers[0].LoadBalancerArn)
+	rec0, err := svc.store.GetLoadBalancerByArn(context.Background(), arn)
+	require.NoError(t, err)
+
+	mock.terminateErr = assert.AnError
+	_, err = svc.SetSubnets(context.Background(), &elbv2.SetSubnetsInput{
+		LoadBalancerArn: aws.String(arn),
+		Subnets:         []*string{aws.String(sub1), aws.String(sub2)},
+	}, testAccountID)
+	require.Error(t, err)
+
+	rec, err := svc.store.GetLBHealth(context.Background(), rec0.LoadBalancerID)
+	require.NoError(t, err)
+	require.NotNil(t, rec)
+	assert.False(t, rec.Suspended, "a failed reshape that left the VM running must not leave repair excluded")
+}
+
+func TestLBHealth_DescribeShowsImpairedOnlyForFailingNICOfCurrentVM(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+	lb := seedHealthLB(t, svc)
+	code := func() string {
+		out, err := svc.DescribeLoadBalancers(context.Background(), &elbv2.DescribeLoadBalancersInput{
+			LoadBalancerArns: []*string{aws.String(lb.LoadBalancerArn)},
+		}, testAccountID)
+		require.NoError(t, err)
+		require.Len(t, out.LoadBalancers, 1)
+		return aws.StringValue(out.LoadBalancers[0].State.Code)
+	}
+
+	assert.Equal(t, StateActive, code())
+
+	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, false, 3, 1))
+	assert.Equal(t, StateActiveImpaired, code())
+
+	healthBeat(t, svc, aws.String(healthInst), nicStatus(healthMACData, true, 0, 2))
+	assert.Equal(t, StateActive, code(), "active again once the probe recovers")
+
+	// Evidence recorded against a VM the LB no longer names never impairs it.
+	_, err := svc.store.UpdateLBHealth(context.Background(), healthLBID, func(r *LBDataPathRecord) (bool, error) {
+		r.InstanceID = "i-oldvm"
+		r.NICs = map[string]*LBNICFailure{healthMACData: {FailingSince: time.Now().UTC()}}
+		return true, nil
+	})
+	require.NoError(t, err)
+	assert.Equal(t, StateActive, code())
+
+	// A LB that is not active is reported as it is.
+	_, err = svc.store.UpdateLBHealth(context.Background(), healthLBID, func(r *LBDataPathRecord) (bool, error) {
+		r.InstanceID = healthInst
+		return true, nil
+	})
+	require.NoError(t, err)
+	require.Equal(t, StateActiveImpaired, code())
+	_, err = svc.store.UpdateLoadBalancer(context.Background(), healthLBID, func(r *LoadBalancerRecord) (bool, error) {
+		r.State = StateFailed
+		return true, nil
+	})
+	require.NoError(t, err)
+	assert.Equal(t, StateFailed, code())
+}
+
+// failKeysKV fails key listing once armed, which is what the listener cascade
+// of DeleteLoadBalancer reads first.
+type failKeysKV struct {
+	jetstream.KeyValue
+
+	armed atomic.Bool
+}
+
+func (k *failKeysKV) Keys(ctx context.Context, opts ...jetstream.WatchOpt) ([]string, error) {
+	if k.armed.Load() {
+		return nil, errors.New("injected key listing failure")
+	}
+	return k.KeyValue.Keys(ctx, opts...)
+}
+
+func TestLBHealth_DeleteLoadBalancerListenerCascadeFailureResumes(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+	lb := seedHealthLB(t, svc)
+	kv := &failKeysKV{KeyValue: svc.store.kv}
+	svc.store.kv = kv
+	mock := &mockSystemInstanceLauncher{}
+	svc.InstanceLauncher = mock
+	kv.armed.Store(true)
+
+	_, err := svc.DeleteLoadBalancer(context.Background(), &elbv2.DeleteLoadBalancerInput{
+		LoadBalancerArn: aws.String(lb.LoadBalancerArn),
+	}, testAccountID)
+	require.Error(t, err)
+	kv.armed.Store(false)
+
+	rec, err := svc.store.GetLBHealth(context.Background(), healthLBID)
+	require.NoError(t, err)
+	require.NotNil(t, rec)
+	assert.False(t, rec.Suspended, "a delete that failed before touching the VM must not leave repair excluded")
+	assert.Empty(t, mock.terminateCalls)
+}

spinifex/domains/elbv2/lb_repair.go

@@ -0,0 +1,377 @@
+package elbv2
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/systeminstance"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvlease"
+	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+const (
+	// KVBucketELBv2Leader holds the lease key of the data-path reconciler.
+	KVBucketELBv2Leader        = "spinifex-elbv2-leader"
+	KVBucketELBv2LeaderVersion = 1
+	KVBucketELBv2LeaderTTL     = 60 * time.Second
+
+	lbRepairLeaderKey     = "datapath"
+	lbRepairLeaseRenew    = 20 * time.Second
+	lbRepairPassInterval  = 30 * time.Second
+	lbRepairMinFailing    = 60 * time.Second
+	lbRepairMinVMAge      = 5 * time.Minute
+	lbRepairCooldown      = 10 * time.Minute
+	lbRepairBudgetWindow  = 24 * time.Hour
+	lbRepairBudget        = 3
+	lbRepairFleetMinCount = 3
+
+	// A heartbeat that persists only every heartbeatPersistInterval is still
+	// fresh for twice that long.
+	lbRepairHeartbeatMaxAge = 2 * heartbeatPersistInterval
+)
+
+// SystemInstanceRebooter is the only capability the repair path has over a VM.
+type SystemInstanceRebooter interface {
+	RebootSystemInstance(instanceID string) error
+}
+
+var _ SystemInstanceRebooter = SystemInstanceLauncher(nil)
+
+// counts reports whether a repair entry uses up cooldown and budget. Only a
+// repair no node received is free.
+func (r LBRepair) counts() bool {
+	return r.Outcome != RepairUnreachable
+}
+
+// repairWindow counts the entries inside window ending at now, and returns the
+// newest of them.
+func repairWindow(repairs []LBRepair, now time.Time, window time.Duration) (n int, newest time.Time) {
+	for _, r := range repairs {
+		if !r.counts() || now.Sub(r.At) >= window {
+			continue
+		}
+		n++
+		if r.At.After(newest) {
+			newest = r.At
+		}
+	}
+	return n, newest
+}
+
+// lbRepairReconciler restarts the VM of a load balancer whose data NIC has been
+// failing its gateway probe. Only the lease holder acts.
+type lbRepairReconciler struct {
+	svc      *ELBv2ServiceImpl
+	rebooter SystemInstanceRebooter
+	isLeader func() bool
+	now      func() time.Time
+
+	mu             sync.Mutex
+	budgetReported map[string]bool
+}
+
+func newLBRepairReconciler(svc *ELBv2ServiceImpl, rebooter SystemInstanceRebooter, isLeader func() bool) *lbRepairReconciler {
+	return &lbRepairReconciler{
+		svc:            svc,
+		rebooter:       rebooter,
+		isLeader:       isLeader,
+		now:            func() time.Time { return time.Now().UTC() },
+		budgetReported: make(map[string]bool),
+	}
+}
+
+// StartDataPathReconciler elects a leader among the daemons and runs the repair
+// pass on it every lbRepairPassInterval until ctx or the service is cancelled.
+func (s *ELBv2ServiceImpl) StartDataPathReconciler(ctx context.Context, holder string) {
+	if s.InstanceLauncher == nil || s.store == nil || s.store.js == nil {
+		return
+	}
+	lease, err := kvlease.New(kvlease.Config{
+		Name:   "elbv2/datapath",
+		Bucket: s.leaderBucket,
+		Key:    lbRepairLeaderKey,
+		Holder: holder,
+		TTL:    KVBucketELBv2LeaderTTL,
+		Renew:  lbRepairLeaseRenew,
+		Retry:  lbRepairLeaseRenew,
+	})
+	if err != nil {
+		slog.ErrorContext(ctx, "ELBv2 data-path reconciler: lease config invalid", "err", err)
+		return
+	}
+	rec := newLBRepairReconciler(s, s.InstanceLauncher, lease.Held)
+	go lease.Run(ctx)
+	go func() {
+		ticker := time.NewTicker(lbRepairPassInterval)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-s.ctx.Done():
+				return
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				rec.pass(ctx)
+			}
+		}
+	}()
+}
+
+func (s *ELBv2ServiceImpl) leaderBucket(ctx context.Context) (jetstream.KeyValue, error) {
+	return kvlease.OpenBucket(ctx, s.store.js, kvlease.BucketConfig{
+		Name:    KVBucketELBv2Leader,
+		TTL:     KVBucketELBv2LeaderTTL,
+		Version: KVBucketELBv2LeaderVersion,
+	})
+}
+
+// candidate is a load balancer that is failing and may be repaired.
+type lbRepairCandidate struct {
+	lb    *LoadBalancerRecord
+	rec   *LBDataPathRecord
+	mac   string
+	nic   *LBNICFailure
+	since time.Time
+}
+
+// pass runs one repair pass. It starts at most one repair.
+func (r *lbRepairReconciler) pass(ctx context.Context) {
+	if !r.isLeader() {
+		return
+	}
+	lbs, err := r.svc.store.ListLoadBalancersStrict(ctx)
+	if err != nil {
+		slog.WarnContext(ctx, "ELBv2 data-path reconciler: failed to list load balancers", "err", err)
+		return
+	}
+	now := r.now()
+
+	var heartbeating, failing int
+	var cands []lbRepairCandidate
+	live := make(map[string]bool, len(lbs))
+	for _, lb := range lbs {
+		live[lb.LoadBalancerID] = true
+		if lb.State != StateActive || lb.InstanceID == "" || now.Sub(lb.LastHeartbeat) >= lbRepairHeartbeatMaxAge {
+			continue
+		}
+		heartbeating++
+		rec, err := r.svc.store.GetLBHealth(ctx, lb.LoadBalancerID)
+		if err != nil || rec == nil || rec.Suspended || rec.InstanceID != lb.InstanceID || !rec.failing() {
+			continue
+		}
+		failing++
+		c := lbRepairCandidate{lb: lb, rec: rec}
+		for mac, nic := range rec.NICs {
+			if c.nic == nil || nic.FailingSince.Before(c.since) {
+				c.mac, c.nic, c.since = mac, nic, nic.FailingSince
+			}
+		}
+		cands = append(cands, c)
+	}
+	r.forgetBudgetReports(live)
+
+	if failing >= lbRepairFleetMinCount && failing*2 > heartbeating {
+		slog.WarnContext(ctx, "ELBv2 data-path repair suppressed: fleet-wide probe failure",
+			"failing_lbs", failing, "heartbeating_lbs", heartbeating)
+		return
+	}
+
+	sort.Slice(cands, func(i, j int) bool { return cands[i].since.Before(cands[j].since) })
+	for _, c := range cands {
+		if r.eligible(ctx, c, now) && r.repair(ctx, c) {
+			return
+		}
+	}
+}
+
+// eligible applies the per-LB conditions that need no write.
+func (r *lbRepairReconciler) eligible(ctx context.Context, c lbRepairCandidate, now time.Time) bool {
+	if now.Sub(c.since) < lbRepairMinFailing {
+		return false
+	}
+	if now.Sub(vmRunningSince(c.lb, c.rec)) < lbRepairMinVMAge {
+		return false
+	}
+	n, newest := repairWindow(c.rec.Repairs, now, lbRepairBudgetWindow)
+	if n >= lbRepairBudget {
+		r.reportBudgetSpent(ctx, c, n)
+		return false
+	}
+	r.clearBudgetReport(c.lb.LoadBalancerID)
+	return now.Sub(newest) >= lbRepairCooldown || newest.IsZero()
+}
+
+// vmRunningSince is when the LB's current VM last came up as far as the control
+// plane can tell: its activation, a relaunch, or the last repair.
+func vmRunningSince(lb *LoadBalancerRecord, rec *LBDataPathRecord) time.Time {
+	since := lb.ActiveSince
+	if since.IsZero() {
+		since = lb.CreatedAt
+	}
+	if rec.BoundAt.After(since) {
+		since = rec.BoundAt
+	}
+	for _, e := range rec.Repairs {
+		if e.counts() && e.At.After(since) {
+			since = e.At
+		}
+	}
+	return since
+}
+
+// repair reserves the attempt durably, then dispatches it. It reports whether a
+// repair was started, which ends the pass.
+func (r *lbRepairReconciler) repair(ctx context.Context, c lbRepairCandidate) bool {
+	if !r.isLeader() {
+		return false
+	}
+	lb, err := r.svc.store.GetLoadBalancer(ctx, c.lb.LoadBalancerID)
+	if err != nil || lb == nil || lb.State != StateActive || lb.InstanceID != c.lb.InstanceID {
+		return false
+	}
+	now := r.now()
+	reason := "data NIC failing gateway probe"
+	var attempt int
+	reserved := false
+	_, err = r.svc.store.UpdateLBHealth(ctx, lb.LoadBalancerID, func(cur *LBDataPathRecord) (bool, error) {
+		// The callback reruns after a conflict, so admission is judged by the run
+		// that wrote the reservation and nothing earlier.
+		reserved, attempt = false, 0
+		nic := cur.NICs[c.mac]
+		if cur.Suspended || cur.InstanceID != lb.InstanceID || nic == nil || now.Sub(nic.FailingSince) < lbRepairMinFailing {
+			return false, nil
+		}
+		n, newest := repairWindow(cur.Repairs, now, lbRepairBudgetWindow)
+		if n >= lbRepairBudget || (!newest.IsZero() && now.Sub(newest) < lbRepairCooldown) {
+			return false, nil
+		}
+		cur.Repairs = append(pruneRepairs(cur.Repairs, now), LBRepair{
+			At: now, InstanceID: lb.InstanceID, ENIID: nic.ENIID, Reason: reason, Outcome: RepairReserved,
+		})
+		attempt = n + 1
+		reserved = true
+		return true, nil
+	})
+	if err != nil {
+		slog.WarnContext(ctx, "ELBv2 data-path repair: reservation failed", "lbId", lb.LoadBalancerID, "err", err)
+		return false
+	}
+	if !reserved {
+		return false
+	}
+
+	attrs := repairLogAttrs(lb, c, attempt, reason, now)
+	if !r.isLeader() {
+		slog.WarnContext(ctx, "ELBv2 data-path repair not dispatched: leadership lost after reservation", attrs...)
+		return true
+	}
+	slog.WarnContext(ctx, "ELBv2 data-path repair started", attrs...)
+	outcome := r.dispatch(lb.InstanceID)
+	if outcome != "" {
+		r.recordOutcome(ctx, lb.LoadBalancerID, now, lb.InstanceID, outcome)
+	}
+	switch outcome {
+	case RepairDispatched:
+		slog.InfoContext(ctx, "ELBv2 data-path repair dispatched", attrs...)
+	case RepairRefused:
+		slog.ErrorContext(ctx, "ELBv2 data-path repair refused by the owning node", attrs...)
+	case RepairUnreachable:
+		slog.WarnContext(ctx, "ELBv2 data-path repair: owning node unreachable, will retry", attrs...)
+	default:
+		slog.WarnContext(ctx, "ELBv2 data-path repair: outcome unknown, counted", attrs...)
+	}
+	return true
+}
+
+// dispatch asks the owning node to restart exactly the LB's own VM. It returns
+// an empty outcome when the reply was lost, which leaves the entry reserved.
+func (r *lbRepairReconciler) dispatch(instanceID string) string {
+	err := r.rebooter.RebootSystemInstance(instanceID)
+	switch {
+	case err == nil:
+		return RepairDispatched
+	case errors.Is(err, nats.ErrNoResponders), strings.Contains(err.Error(), "no responders"):
+		return RepairUnreachable
+	case errors.Is(err, systeminstance.ErrSystemInstanceNotFound),
+		errors.Is(err, systeminstance.ErrSystemInstanceNotSystem),
+		errors.Is(err, systeminstance.ErrSystemInstanceOwnershipUnknown):
+		return RepairRefused
+	default:
+		return ""
+	}
+}
+
+func (r *lbRepairReconciler) recordOutcome(ctx context.Context, lbID string, at time.Time, instanceID, outcome string) {
+	_, err := r.svc.store.UpdateLBHealth(ctx, lbID, func(cur *LBDataPathRecord) (bool, error) {
+		for i := range cur.Repairs {
+			e := &cur.Repairs[i]
+			if e.At.Equal(at) && e.InstanceID == instanceID && e.Outcome == RepairReserved {
+				e.Outcome = outcome
+				return true, nil
+			}
+		}
+		return false, nil
+	})
+	if err != nil {
+		slog.WarnContext(ctx, "ELBv2 data-path repair: failed to record outcome", "lbId", lbID, "outcome", outcome, "err", err)
+	}
+}
+
+// pruneRepairs drops entries older than the budget window.
+func pruneRepairs(repairs []LBRepair, now time.Time) []LBRepair {
+	kept := repairs[:0:0]
+	for _, e := range repairs {
+		if now.Sub(e.At) < lbRepairBudgetWindow {
+			kept = append(kept, e)
+		}
+	}
+	return kept
+}
+
+func repairLogAttrs(lb *LoadBalancerRecord, c lbRepairCandidate, attempt int, reason string, now time.Time) []any {
+	return []any{
+		"lbArn", lb.LoadBalancerArn, "lbId", lb.LoadBalancerID, "instanceId", lb.InstanceID, "nodeId", lb.NodeID,
+		"eniId", c.nic.ENIID, "mac", c.mac, "failing_since", c.nic.FailingSince,
+		"failing_ms", otelsetup.Millis(now.Sub(c.nic.FailingSince)),
+		"consecutive_failures", c.nic.ConsecutiveFailures,
+		"rx_packets", c.nic.RxPackets, "tx_packets", c.nic.TxPackets,
+		"rx_dropped", c.nic.RxDropped, "tx_dropped", c.nic.TxDropped,
+		"repair_attempt", attempt, "reason", reason,
+	}
+}
+
+func (r *lbRepairReconciler) reportBudgetSpent(ctx context.Context, c lbRepairCandidate, n int) {
+	r.mu.Lock()
+	already := r.budgetReported[c.lb.LoadBalancerID]
+	r.budgetReported[c.lb.LoadBalancerID] = true
+	r.mu.Unlock()
+	if already {
+		return
+	}
+	slog.ErrorContext(ctx, "ELBv2 data-path repair budget exhausted",
+		"lbArn", c.lb.LoadBalancerArn, "lbId", c.lb.LoadBalancerID, "instanceId", c.lb.InstanceID,
+		"eniId", c.nic.ENIID, "mac", c.mac, "repairs_24h", n)
+}
+
+func (r *lbRepairReconciler) clearBudgetReport(lbID string) {
+	r.mu.Lock()
+	delete(r.budgetReported, lbID)
+	r.mu.Unlock()
+}
+
+func (r *lbRepairReconciler) forgetBudgetReports(live map[string]bool) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	for id := range r.budgetReported {
+		if !live[id] {
+			delete(r.budgetReported, id)
+		}
+	}
+}

spinifex/domains/elbv2/lb_repair_test.go

@@ -0,0 +1,418 @@
+package elbv2
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/systeminstance"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const repairMAC = "02:00:00:bb:00:01"
+
+type repairEnv struct {
+	t        *testing.T
+	svc      *ELBv2ServiceImpl
+	rec      *lbRepairReconciler
+	launcher *mockSystemInstanceLauncher
+	clock    time.Time
+	leader   bool
+}
+
+func newRepairEnv(t *testing.T) *repairEnv {
+	t.Helper()
+	e := &repairEnv{
+		t:        t,
+		svc:      setupTestService(t),
+		launcher: &mockSystemInstanceLauncher{},
+		clock:    time.Date(2026, 11, 5, 12, 0, 0, 0, time.UTC),
+		leader:   true,
+	}
+	e.rec = newLBRepairReconciler(e.svc, e.launcher, func() bool { return e.leader })
+	e.rec.now = func() time.Time { return e.clock }
+	return e
+}
+
+// seed stores an active LB whose VM has run for ten minutes and whose data NIC
+// has been failing for two, with a heartbeat from this instant.
+func (e *repairEnv) seed(id string, failing bool) *LoadBalancerRecord {
+	e.t.Helper()
+	lb := &LoadBalancerRecord{
+		LoadBalancerArn: "arn:aws:elasticloadbalancing:us-east-1:" + testAccountID + ":loadbalancer/app/" + id + "/" + id,
+		LoadBalancerID:  id,
+		Name:            id,
+		State:           StateActive,
+		InstanceID:      "i-" + id,
+		AccountID:       testAccountID,
+		NodeID:          "node-1",
+		CreatedAt:       e.clock.Add(-time.Hour),
+		ActiveSince:     e.clock.Add(-10 * time.Minute),
+		LastHeartbeat:   e.clock,
+	}
+	require.NoError(e.t, e.svc.store.PutLoadBalancer(context.Background(), lb))
+	if failing {
+		e.fail(id, 2*time.Minute)
+	}
+	return lb
+}
+
+func (e *repairEnv) fail(id string, age time.Duration) {
+	e.t.Helper()
+	_, err := e.svc.store.UpdateLoadBalancer(context.Background(), id, func(lb *LoadBalancerRecord) (bool, error) {
+		lb.LastHeartbeat = e.clock
+		return true, nil
+	})
+	require.NoError(e.t, err)
+	_, err = e.svc.store.UpdateLBHealth(context.Background(), id, func(rec *LBDataPathRecord) (bool, error) {
+		rec.InstanceID = "i-" + id
+		rec.NICs = map[string]*LBNICFailure{
+			repairMAC: {ENIID: "eni-" + id, FailingSince: e.clock.Add(-age), ConsecutiveFailures: 9, RxPackets: 4, TxPackets: 8},
+		}
+		return true, nil
+	})
+	require.NoError(e.t, err)
+}
+
+func (e *repairEnv) health(id string) *LBDataPathRecord {
+	e.t.Helper()
+	rec, err := e.svc.store.GetLBHealth(context.Background(), id)
+	require.NoError(e.t, err)
+	require.NotNil(e.t, rec)
+	return rec
+}
+
+func (e *repairEnv) pass() { e.rec.pass(context.Background()) }
+
+func (e *repairEnv) reboots() []string {
+	e.launcher.mu.Lock()
+	defer e.launcher.mu.Unlock()
+	return append([]string(nil), e.launcher.rebootCalls...)
+}
+
+func captureLogs(t *testing.T) *bytes.Buffer {
+	t.Helper()
+	var buf bytes.Buffer
+	prev := slog.Default()
+	slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})))
+	t.Cleanup(func() { slog.SetDefault(prev) })
+	return &buf
+}
+
+func TestLBRepair_RepairsOnlyAfterSixtySecondsOfFailure(t *testing.T) {
+	t.Parallel()
+	e := newRepairEnv(t)
+	e.seed("lbr1", false)
+	e.fail("lbr1", 59*time.Second)
+	e.pass()
+	assert.Empty(t, e.reboots())
+
+	e.clock = e.clock.Add(2 * time.Second)
+	e.pass()
+	assert.Equal(t, []string{"i-lbr1"}, e.reboots())
+	h := e.health("lbr1")
+	require.Len(t, h.Repairs, 1)
+	assert.Equal(t, RepairDispatched, h.Repairs[0].Outcome)
+	assert.Equal(t, "eni-lbr1", h.Repairs[0].ENIID)
+}
+
+func TestLBRepair_ContinuouslyHeartbeatingLBIsNeverStale(t *testing.T) {
+	t.Parallel()
+	// A healthy agent persists its heartbeat only every minute, so the stored
+	// timestamp can be any age below that plus one pass at the reconciler.
+	for age := time.Duration(0); age < lbRepairHeartbeatMaxAge; age += 15 * time.Second {
+		e := newRepairEnv(t)
+		lb := e.seed("lbr2", true)
+		lb.LastHeartbeat = e.clock.Add(-age)
+		require.NoError(t, e.svc.store.PutLoadBalancer(context.Background(), lb))
+		e.pass()
+		assert.Len(t, e.reboots(), 1, "heartbeat %s old", age)
+	}
+
+	e := newRepairEnv(t)
+	lb := e.seed("lbr2", true)
+	lb.LastHeartbeat = e.clock.Add(-lbRepairHeartbeatMaxAge)
+	require.NoError(t, e.svc.store.PutLoadBalancer(context.Background(), lb))
+	e.pass()
+	assert.Empty(t, e.reboots(), "a heartbeat older than the bound is stale")
+}
+
+func TestLBRepair_NeverWithoutCurrentEvidence(t *testing.T) {
+	t.Parallel()
+	mutate := map[string]func(*LoadBalancerRecord){
+		"non-active LB":      func(lb *LoadBalancerRecord) { lb.State = StateProvisioning },
+		"failed LB":          func(lb *LoadBalancerRecord) { lb.State = StateFailed },
+		"empty instance ID":  func(lb *LoadBalancerRecord) { lb.InstanceID = "" },
+		"changed instance":   func(lb *LoadBalancerRecord) { lb.InstanceID = "i-other" },
+		"stale heartbeat":    func(lb *LoadBalancerRecord) { lb.LastHeartbeat = lb.LastHeartbeat.Add(-time.Hour) },
+		"VM under 5 minutes": func(lb *LoadBalancerRecord) { lb.ActiveSince = lb.LastHeartbeat.Add(-4 * time.Minute) },
+	}
+	for name, fn := range mutate {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			e := newRepairEnv(t)
+			lb := e.seed("lbr3", true)
+			fn(lb)
+			require.NoError(t, e.svc.store.PutLoadBalancer(context.Background(), lb))
+			e.pass()
+			assert.Empty(t, e.reboots())
+		})
+	}
+}
+
+func TestLBRepair_NonHolderNeverRepairs(t *testing.T) {
+	t.Parallel()
+	e := newRepairEnv(t)
+	e.seed("lbr4", true)
+	e.leader = false
+	e.pass()
+	assert.Empty(t, e.reboots())
+	e.leader = true
+	e.pass()
+	assert.Len(t, e.reboots(), 1)
+}
+
+func TestLBRepair_OneRepairPerPass(t *testing.T) {
+	t.Parallel()
+	e := newRepairEnv(t)
+	e.seed("lbr5a", true)
+	e.seed("lbr5b", true)
+	for i := range 3 {
+		e.seed(fmt.Sprintf("lbr5h%d", i), false)
+	}
+	e.fail("lbr5b", 3*time.Minute)
+
+	e.pass()
+	assert.Equal(t, []string{"i-lbr5b"}, e.reboots(), "the longest-failing LB goes first")
+	e.pass()
+	assert.Equal(t, []string{"i-lbr5b", "i-lbr5a"}, e.reboots())
+}
+
+func TestLBRepair_FleetWideFailureRepairsNothing(t *testing.T) {
+	e := newRepairEnv(t)
+	logs := captureLogs(t)
+	e.seed("lbr6a", true)
+	e.seed("lbr6b", true)
+	e.seed("lbr6c", true)
+	e.seed("lbr6d", false)
+	e.pass()
+	assert.Empty(t, e.reboots())
+	assert.Contains(t, logs.String(), "ELBv2 data-path repair suppressed: fleet-wide probe failure")
+
+	// Three failing out of seven is not more than half.
+	for i := range 3 {
+		e.seed(fmt.Sprintf("lbr6h%d", i), false)
+	}
+	e.pass()
+	assert.Len(t, e.reboots(), 1)
+}
+
+func TestLBRepair_CooldownAndDailyBudget(t *testing.T) {
+	e := newRepairEnv(t)
+	logs := captureLogs(t)
+	e.seed("lbr7", true)
+
+	e.pass()
+	require.Len(t, e.reboots(), 1)
+
+	e.clock = e.clock.Add(9 * time.Minute)
+	e.fail("lbr7", 2*time.Minute)
+	e.pass()
+	assert.Len(t, e.reboots(), 1, "inside the 10 minute cooldown")
+
+	for want := 2; want <= 3; want++ {
+		e.clock = e.clock.Add(11 * time.Minute)
+		e.fail("lbr7", 2*time.Minute)
+		e.pass()
+		assert.Len(t, e.reboots(), want)
+	}
+
+	e.clock = e.clock.Add(11 * time.Minute)
+	e.fail("lbr7", 2*time.Minute)
+	e.pass()
+	e.pass()
+	assert.Len(t, e.reboots(), 3, "the fourth repair in 24 hours is refused")
+	assert.Equal(t, 1, strings.Count(logs.String(), "ELBv2 data-path repair budget exhausted"))
+
+	e.clock = e.clock.Add(24 * time.Hour)
+	e.fail("lbr7", 2*time.Minute)
+	e.pass()
+	assert.Len(t, e.reboots(), 4, "the budget frees after 24 hours")
+}
+
+func TestLBRepair_UnreachableOwnerIsRetriedAndNotCounted(t *testing.T) {
+	t.Parallel()
+	e := newRepairEnv(t)
+	e.launcher.rebootErr = fmt.Errorf("nats request: %w", nats.ErrNoResponders)
+	e.seed("lbr8", true)
+	for range 4 {
+		e.pass()
+		e.clock = e.clock.Add(30 * time.Second)
+		e.fail("lbr8", 2*time.Minute)
+	}
+	assert.Len(t, e.reboots(), 4, "no node received it, so cooldown and budget do not apply")
+	for _, r := range e.health("lbr8").Repairs {
+		assert.Equal(t, RepairUnreachable, r.Outcome)
+	}
+
+	e.launcher.rebootErr = nil
+	e.pass()
+	assert.Len(t, e.reboots(), 5)
+}
+
+func TestLBRepair_RefusedAndAmbiguousRepairsStillCount(t *testing.T) {
+	t.Parallel()
+	for name, err := range map[string]error{
+		"refused": fmt.Errorf("%w: i-lbr9", systeminstance.ErrSystemInstanceNotSystem),
+		"lost":    errors.New("timeout waiting for reply"),
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			e := newRepairEnv(t)
+			e.launcher.rebootErr = err
+			e.seed("lbr9", true)
+			e.pass()
+			e.clock = e.clock.Add(30 * time.Second)
+			e.fail("lbr9", 2*time.Minute)
+			e.pass()
+			assert.Len(t, e.reboots(), 1, "inside the cooldown")
+			want := RepairRefused
+			if name == "lost" {
+				want = RepairReserved
+			}
+			assert.Equal(t, want, e.health("lbr9").Repairs[0].Outcome)
+		})
+	}
+}
+
+func TestLBRepair_ReservationWithoutOutcomeCountsAfterCrashOrLostLease(t *testing.T) {
+	t.Parallel()
+	e := newRepairEnv(t)
+	e.seed("lbr10", true)
+	// A leader reserved and died before dispatching.
+	_, err := e.svc.store.UpdateLBHealth(context.Background(), "lbr10", func(rec *LBDataPathRecord) (bool, error) {
+		rec.Repairs = append(rec.Repairs, LBRepair{At: e.clock.Add(-time.Minute), InstanceID: "i-lbr10", Outcome: RepairReserved})
+		return true, nil
+	})
+	require.NoError(t, err)
+
+	e.pass()
+	assert.Empty(t, e.reboots(), "the new leader sees the counted reservation")
+
+	e.leader = false
+	e.pass()
+	e.leader = true
+	assert.Empty(t, e.reboots())
+}
+
+func TestLBRepair_PausedBetweenEligibilityAndReservationRepairsNothing(t *testing.T) {
+	t.Parallel()
+	cases := map[string]func(e *repairEnv, lb *LoadBalancerRecord){
+		"DeleteLoadBalancer starts": func(e *repairEnv, lb *LoadBalancerRecord) {
+			require.NoError(t, e.svc.store.SuspendLBHealth(context.Background(), lb.LoadBalancerID))
+		},
+		"SetSubnets relaunches the VM": func(e *repairEnv, lb *LoadBalancerRecord) {
+			lb.InstanceID = "i-new"
+			require.NoError(t, e.svc.store.PutLoadBalancer(context.Background(), lb))
+		},
+		"SetSubnets rebinds the evidence": func(e *repairEnv, lb *LoadBalancerRecord) {
+			require.NoError(t, e.svc.store.ResumeLBHealth(context.Background(), lb.LoadBalancerID, "i-new"))
+		},
+	}
+	for name, interleave := range cases {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			e := newRepairEnv(t)
+			lb := e.seed("lbr11", true)
+			cands := e.candidates()
+			require.Len(t, cands, 1)
+			require.True(t, e.rec.eligible(context.Background(), cands[0], e.clock))
+
+			interleave(e, lb)
+
+			assert.False(t, e.rec.repair(context.Background(), cands[0]))
+			assert.Empty(t, e.reboots())
+		})
+	}
+}
+
+func (e *repairEnv) candidates() []lbRepairCandidate {
+	e.t.Helper()
+	var out []lbRepairCandidate
+	lbs, err := e.svc.store.ListLoadBalancersStrict(context.Background())
+	require.NoError(e.t, err)
+	for _, lb := range lbs {
+		rec, err := e.svc.store.GetLBHealth(context.Background(), lb.LoadBalancerID)
+		require.NoError(e.t, err)
+		if rec == nil {
+			continue
+		}
+		c := lbRepairCandidate{lb: lb, rec: rec}
+		for mac, nic := range rec.NICs {
+			c.mac, c.nic, c.since = mac, nic, nic.FailingSince
+		}
+		out = append(out, c)
+	}
+	return out
+}
+
+// Verifies: ADR-0003:S7.
+func TestS7_LBRepairActsOnlyOnTheLBsOwnVM(t *testing.T) {
+	t.Parallel()
+	e := newRepairEnv(t)
+	e.seed("lbr12", true)
+	e.seed("lbr12ok", false)
+	e.pass()
+	e.pass()
+	assert.Equal(t, []string{"i-lbr12"}, e.reboots(), "only the failing LB's own instance ID reaches the launcher")
+	e.launcher.mu.Lock()
+	defer e.launcher.mu.Unlock()
+	assert.Empty(t, e.launcher.terminateCalls)
+	assert.Empty(t, e.launcher.launchCalls)
+}
+
+func TestLBRepair_TeardownNeverTriggersRepair(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	t.Run("DeleteLoadBalancer", func(t *testing.T) {
+		t.Parallel()
+		e := newRepairEnv(t)
+		lb := e.seed("lbr13a", true)
+		require.NoError(t, e.svc.store.SuspendLBHealth(ctx, lb.LoadBalancerID))
+		e.pass()
+		assert.Empty(t, e.reboots())
+	})
+	t.Run("SetSubnets window", func(t *testing.T) {
+		t.Parallel()
+		e := newRepairEnv(t)
+		lb := e.seed("lbr13b", true)
+		require.NoError(t, e.svc.store.SuspendLBHealth(ctx, lb.LoadBalancerID))
+		lb.InstanceID = "i-relaunched"
+		require.NoError(t, e.svc.store.PutLoadBalancer(ctx, lb))
+		e.pass()
+		assert.Empty(t, e.reboots(), "suspended, and the old VM's evidence names another instance")
+	})
+	t.Run("transient probe failure on the relaunched VM", func(t *testing.T) {
+		t.Parallel()
+		e := newRepairEnv(t)
+		lb := e.seed("lbr13c", false)
+		lb.InstanceID = "i-relaunched"
+		require.NoError(t, e.svc.store.PutLoadBalancer(ctx, lb))
+		require.NoError(t, e.svc.store.ResumeLBHealth(ctx, lb.LoadBalancerID, "i-relaunched"))
+		_, err := e.svc.store.UpdateLBHealth(ctx, lb.LoadBalancerID, func(rec *LBDataPathRecord) (bool, error) {
+			rec.BoundAt = e.clock.Add(-90 * time.Second)
+			rec.NICs = map[string]*LBNICFailure{repairMAC: {FailingSince: e.clock.Add(-80 * time.Second)}}
+			return true, nil
+		})
+		require.NoError(t, err)
+		e.pass()
+		assert.Empty(t, e.reboots(), "a freshly relaunched VM gets five minutes")
+	})
+}

spinifex/domains/elbv2/service_impl.go

@@ -1138,6 +1138,7 @@ func (s *ELBv2ServiceImpl) LBAgentHeartbeat(ctx context.Context, input *LBAgentH
 			if rec.State == StateProvisioning {
 				rec.State = StateActive
 				rec.StateReason = ""
+				rec.ActiveSince = now
 			}
 			rec.LastHeartbeat = now
 			return true, nil
@@ -1162,6 +1163,8 @@ func (s *ELBv2ServiceImpl) LBAgentHeartbeat(ctx context.Context, input *LBAgentH
 		}
 	}
 
+	s.recordDataPathEvidence(ctx, lb, input)
+
 	// Process health report directly — no JSON round-trip needed.
 	if len(input.Servers) > 0 {
 		s.hc.handleHealthReportDirect(ctx, input.toHealthReport())
@@ -1853,15 +1856,23 @@ func (s *ELBv2ServiceImpl) DeleteLoadBalancer(ctx context.Context, input *elbv2.
 		return &elbv2.DeleteLoadBalancerOutput{}, nil
 	}
 
+	// Exclude data-path repair before anything touches the VM or its ENIs.
+	if err := s.store.SuspendLBHealth(ctx, lb.LoadBalancerID); err != nil {
+		slog.ErrorContext(ctx, "DeleteLoadBalancer: failed to suspend data-path repair", "lbId", lb.LoadBalancerID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+
 	// Cascade-delete all listeners and their rules so no orphan pins a TG as ResourceInUse.
 	listeners, err := s.store.ListListenersByLB(ctx, lb.LoadBalancerArn)
 	if err != nil {
 		slog.ErrorContext(ctx, "DeleteLoadBalancer: failed to list listeners for cascade", "lbArn", lb.LoadBalancerArn, "err", err)
+		s.resumeLBHealth(ctx, lb.LoadBalancerID)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 	for _, l := range listeners {
 		if err := s.deleteListenerCascade(ctx, l); err != nil {
 			slog.ErrorContext(ctx, "DeleteLoadBalancer: failed to cascade listener delete", "listenerID", l.ListenerID, "err", err)
+			s.resumeLBHealth(ctx, lb.LoadBalancerID)
 			return nil, errors.New(awserrors.ErrorServerInternal)
 		}
 	}
@@ -1910,6 +1921,9 @@ func (s *ELBv2ServiceImpl) DeleteLoadBalancer(ctx context.Context, input *elbv2.
 		slog.ErrorContext(ctx, "DeleteLoadBalancer: failed to delete record", "lbId", lb.LoadBalancerID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
+	if err := s.store.DeleteLBHealth(ctx, lb.LoadBalancerID); err != nil {
+		slog.WarnContext(ctx, "DeleteLoadBalancer: failed to delete data-path evidence", "lbId", lb.LoadBalancerID, "err", err)
+	}
 
 	// Release the name claim so the name is reusable. Idempotent on a missing
 	// key, so a delete that races the record removal still converges.
@@ -3456,6 +3470,9 @@ func (s *ELBv2ServiceImpl) lbRecordToSDK(ctx context.Context, r *LoadBalancerRec
 	if r.StateReason != "" {
 		lb.State.Reason = aws.String(r.StateReason)
 	}
+	if s.impairedByEvidence(ctx, r) {
+		lb.State.Code = aws.String(StateActiveImpaired)
+	}
 
 	for _, sg := range r.SecurityGroups {
 		lb.SecurityGroups = append(lb.SecurityGroups, aws.String(sg))

spinifex/domains/elbv2/service_impl_lb_network.go

@@ -172,6 +172,18 @@ func (s *ELBv2ServiceImpl) SetSubnets(ctx context.Context, input *elbv2.SetSubne
 		return s.setSubnetsOutput(ctx, lb), nil
 	}
 
+	// Exclude data-path repair before any ENI or VM call. It stays excluded once the
+	// VM is terminated and is lifted when the new VM is published.
+	if err := s.store.SuspendLBHealth(ctx, lb.LoadBalancerID); err != nil {
+		slog.ErrorContext(ctx, "SetSubnets: failed to suspend data-path repair", "arn", *input.LoadBalancerArn, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	resume := func() {
+		if err := s.store.ResumeLBHealth(ctx, lb.LoadBalancerID, ""); err != nil {
+			slog.WarnContext(ctx, "SetSubnets: failed to lift data-path suspension", "lbId", lb.LoadBalancerID, "err", err)
+		}
+	}
+
 	// Create ENIs for added subnets; roll back on failure to avoid leaks.
 	newENIBySubnet := make(map[string]string, len(toAdd))
 	newAZBySubnet := make(map[string]string, len(toAdd))
@@ -188,6 +200,7 @@ func (s *ELBv2ServiceImpl) SetSubnets(ctx context.Context, input *elbv2.SetSubne
 		eniID, az, eniErr := s.createLBENI(ctx, subnetID, lb, accountID)
 		if eniErr != nil {
 			rollbackNewENIs()
+			resume()
 			slog.ErrorContext(ctx, "SetSubnets: failed to create ENI", "subnet", subnetID, "err", eniErr)
 			return nil, errors.New(awserrors.ErrorELBv2SubnetNotFound)
 		}
@@ -209,6 +222,7 @@ func (s *ELBv2ServiceImpl) SetSubnets(ctx context.Context, input *elbv2.SetSubne
 	if lb.InstanceID != "" && s.InstanceLauncher != nil {
 		if err := s.InstanceLauncher.TerminateSystemInstance(lb.InstanceID); err != nil {
 			rollbackNewENIs()
+			resume()
 			slog.ErrorContext(ctx, "SetSubnets: failed to terminate LB VM for relaunch", "arn", *input.LoadBalancerArn, "instanceId", lb.InstanceID, "err", err)
 			return nil, errors.New(awserrors.ErrorServerInternal)
 		}
@@ -268,6 +282,9 @@ func (s *ELBv2ServiceImpl) SetSubnets(ctx context.Context, input *elbv2.SetSubne
 		slog.ErrorContext(ctx, "SetSubnets: failed to persist LB", "arn", *input.LoadBalancerArn, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
+	if err := s.store.ResumeLBHealth(ctx, lb.LoadBalancerID, lb.InstanceID); err != nil {
+		slog.WarnContext(ctx, "SetSubnets: failed to lift data-path suspension", "lbId", lb.LoadBalancerID, "err", err)
+	}
 
 	slog.InfoContext(ctx, "SetSubnets completed", "arn", *input.LoadBalancerArn, "subnets", len(desired), "added", len(toAdd), "removed", len(toRemove), "state", lb.State)
 	return s.setSubnetsOutput(ctx, lb), nil

spinifex/domains/elbv2/service_impl_test.go

@@ -2108,6 +2108,8 @@ func (m *mockTerminateLauncher) TerminateSystemInstance(instanceID string) error
 	return nil
 }
 
+func (m *mockTerminateLauncher) RebootSystemInstance(_ string) error { return nil }
+
 // --- Scheme unit tests ---
 
 func TestCreateLoadBalancer_InternetFacingScheme_DNSName(t *testing.T) {

spinifex/domains/elbv2/system_instance_launcher_nats.go

@@ -96,4 +96,25 @@ func (l *natsSystemInstanceLauncher) TerminateSystemInstance(instanceID string)
 	return nil
 }
 
+func (l *natsSystemInstanceLauncher) RebootSystemInstance(instanceID string) error {
+	if instanceID == "" {
+		return fmt.Errorf("system instance reboot: empty instance ID")
+	}
+
+	subject := fmt.Sprintf("system.RebootInstance.%s", instanceID)
+	reply, err := l.nc.Request(subject, nil, l.timeout)
+	if err != nil {
+		return fmt.Errorf("nats request %s: %w", subject, err)
+	}
+
+	var env natsSystemInstanceTerminateEnvelope
+	if err := json.Unmarshal(reply.Data, &env); err != nil {
+		return fmt.Errorf("decode reboot reply: %w", err)
+	}
+	if env.Error != "" {
+		return fmt.Errorf("%s", env.Error)
+	}
+	return nil
+}
+
 var _ SystemInstanceLauncher = (*natsSystemInstanceLauncher)(nil)

spinifex/domains/elbv2/system_instance_launcher_nats_test.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -311,3 +312,69 @@ func TestNATSSystemInstanceLauncher_RefusesNonLoadBalancerKindBeforeSending(t *t
 	defer mu.Unlock()
 	assert.Equal(t, 1, received, "only the load balancer launch may be sent")
 }
+
+func TestNATSSystemInstanceLauncher_RebootEmptyID(t *testing.T) {
+	t.Parallel()
+	_, nc, _ := testutil.StartTestJetStream(t)
+	defer nc.Close()
+
+	err := NewNATSSystemInstanceLauncher(nc, time.Second).RebootSystemInstance("")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "empty instance ID")
+}
+
+func TestNATSSystemInstanceLauncher_RebootRoundTrip(t *testing.T) {
+	t.Parallel()
+	_, nc, _ := testutil.StartTestJetStream(t)
+	defer nc.Close()
+
+	var asked atomic.Int32
+	sub, err := nc.Subscribe("system.RebootInstance.i-lb", func(msg *nats.Msg) {
+		asked.Add(1)
+		_ = msg.Respond([]byte(`{}`))
+	})
+	require.NoError(t, err)
+	defer func() { _ = sub.Unsubscribe() }()
+
+	require.NoError(t, NewNATSSystemInstanceLauncher(nc, 2*time.Second).RebootSystemInstance("i-lb"))
+	assert.EqualValues(t, 1, asked.Load())
+}
+
+func TestNATSSystemInstanceLauncher_RebootNoResponderKeepsTransportError(t *testing.T) {
+	t.Parallel()
+	_, nc, _ := testutil.StartTestJetStream(t)
+	defer nc.Close()
+
+	err := NewNATSSystemInstanceLauncher(nc, 250*time.Millisecond).RebootSystemInstance("i-no-one-listens")
+	require.ErrorIs(t, err, nats.ErrNoResponders)
+}
+
+func TestNATSSystemInstanceLauncher_RebootDecodeFailureAndRemoteError(t *testing.T) {
+	t.Parallel()
+	_, nc, _ := testutil.StartTestJetStream(t)
+	defer nc.Close()
+
+	bad, err := nc.Subscribe("system.RebootInstance.i-decode", func(msg *nats.Msg) {
+		_ = msg.Respond([]byte("{not valid json"))
+	})
+	require.NoError(t, err)
+	defer func() { _ = bad.Unsubscribe() }()
+
+	refused, err := nc.Subscribe("system.RebootInstance.i-refused", func(msg *nats.Msg) {
+		payload, _ := json.Marshal(struct {
+			Error string `json:"error,omitempty"`
+		}{Error: "not a system instance"})
+		_ = msg.Respond(payload)
+	})
+	require.NoError(t, err)
+	defer func() { _ = refused.Unsubscribe() }()
+
+	l := NewNATSSystemInstanceLauncher(nc, 2*time.Second)
+	err = l.RebootSystemInstance("i-decode")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "decode reboot reply")
+
+	err = l.RebootSystemInstance("i-refused")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "not a system instance")
+}

spinifex/domains/elbv2/types.go

@@ -155,6 +155,7 @@ type LoadBalancerRecord struct {
 	Tags             map[string]string  `json:"tags,omitempty"`
 	AccountID        string             `json:"account_id"`
 	CreatedAt        time.Time          `json:"created_at"`
+	ActiveSince      time.Time          `json:"active_since,omitzero"` // First heartbeat of the current VM generation
 }
 
 // HealthTargetSpec is a backend the nginx agent actively health-checks, computed

spinifex/domains/lambda/awsapi/eventsource.go

@@ -0,0 +1,277 @@
+package awsapi
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/lambda"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/eventsource"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// MappingStore is what the handlers need of the event-source mapping store.
+type MappingStore interface {
+	Create(ctx context.Context, m eventsource.Mapping) error
+	Get(ctx context.Context, uuid string) (eventsource.Mapping, uint64, error)
+	List(ctx context.Context) ([]eventsource.Entry, error)
+	Update(ctx context.Context, m eventsource.Mapping, revision uint64) (uint64, error)
+}
+
+var _ MappingStore = (*eventsource.Store)(nil)
+
+// SourceChecker reads an SQS queue's visibility timeout as the function's
+// execution role, so a queue the role cannot read is refused at create.
+type SourceChecker interface {
+	QueueVisibilityTimeout(ctx context.Context, rec function.Record, queueARN string) (int64, error)
+}
+
+// Batch size bounds for an SQS standard queue without a batching window.
+const (
+	defaultBatchSize = 10
+	maxBatchSize     = 10
+)
+
+// mappingCreateFields are the CreateEventSourceMapping members this build
+// accepts; every other member is refused rather than ignored.
+var mappingCreateFields = map[string]bool{
+	"FunctionName": true, "EventSourceArn": true, "BatchSize": true, "Enabled": true,
+	"MaximumBatchingWindowInSeconds": true,
+}
+
+var mappingUUIDRE = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`)
+
+func mappingNotFound() error {
+	return awserrors.Errorf(awserrors.ErrorResourceNotFoundException, "The resource you requested does not exist.")
+}
+
+// mappingARN is the IAM resource of one mapping, or "*" for a malformed UUID.
+func mappingARN(region, accountID, uuid string) string {
+	if region == "" || accountID == "" || !mappingUUIDRE.MatchString(uuid) {
+		return "*"
+	}
+	return fmt.Sprintf("arn:aws:lambda:%s:%s:event-source-mapping:%s", region, accountID, uuid)
+}
+
+type mappingCreate struct {
+	FunctionName, QueueARN string
+	BatchSize              int64
+}
+
+// decodeMappingCreate reads the body strictly and refuses every option this
+// build does not implement.
+func decodeMappingCreate(body []byte) (*mappingCreate, error) {
+	var raw map[string]json.RawMessage
+	dec := json.NewDecoder(bytes.NewReader(body))
+	if err := dec.Decode(&raw); err != nil {
+		return nil, invalid("The request body could not be parsed as JSON.")
+	}
+	for k := range raw {
+		if !mappingCreateFields[k] {
+			return nil, unsupported(k)
+		}
+	}
+	var in struct {
+		FunctionName                   *string `json:"FunctionName"`
+		EventSourceArn                 *string `json:"EventSourceArn"`
+		BatchSize                      *int64  `json:"BatchSize"`
+		Enabled                        *bool   `json:"Enabled"`
+		MaximumBatchingWindowInSeconds *int64  `json:"MaximumBatchingWindowInSeconds"`
+	}
+	if err := json.Unmarshal(body, &in); err != nil {
+		return nil, invalid("The request body could not be parsed as JSON.")
+	}
+	if in.FunctionName == nil || *in.FunctionName == "" {
+		return nil, invalid("1 validation error detected: Value null at 'functionName' failed to satisfy constraint: Member must not be null")
+	}
+	if in.EventSourceArn == nil || *in.EventSourceArn == "" {
+		return nil, invalid("Unrecognized event source, must be kinesis, dynamodb stream or sqs.")
+	}
+	if in.Enabled != nil && !*in.Enabled {
+		return nil, unsupported("Enabled false")
+	}
+	if in.MaximumBatchingWindowInSeconds != nil && *in.MaximumBatchingWindowInSeconds != 0 {
+		return nil, unsupported("MaximumBatchingWindowInSeconds")
+	}
+	m := &mappingCreate{FunctionName: *in.FunctionName, QueueARN: *in.EventSourceArn, BatchSize: defaultBatchSize}
+	if in.BatchSize != nil {
+		if *in.BatchSize < 1 || *in.BatchSize > maxBatchSize {
+			return nil, invalid("BatchSize must be between 1 and %d for an SQS queue without a batching window.", maxBatchSize)
+		}
+		m.BatchSize = *in.BatchSize
+	}
+	return m, nil
+}
+
+// checkQueueARN refuses anything but a standard SQS queue in the caller's
+// own account and Region.
+func checkQueueARN(region, accountID, queueARN string) error {
+	parts := strings.Split(queueARN, ":")
+	if len(parts) != 6 || parts[0] != "arn" || parts[1] != "aws" {
+		return invalid("Unrecognized event source, must be kinesis, dynamodb stream or sqs.")
+	}
+	if parts[2] != "sqs" {
+		return unsupported("an event source other than SQS")
+	}
+	if parts[3] != region || parts[4] != accountID {
+		return unsupported("a queue in another account or Region")
+	}
+	if parts[5] == "" {
+		return invalid("Unrecognized event source, must be kinesis, dynamodb stream or sqs.")
+	}
+	if strings.HasSuffix(parts[5], ".fifo") {
+		return unsupported("a FIFO queue")
+	}
+	return nil
+}
+
+// mappingUUID derives one UUID per account, Region, function and queue, so the
+// keyed create itself refuses a duplicate with no inventory scan or second record.
+func mappingUUID(accountID, region, function, queueARN string) string {
+	b := sha256.Sum256([]byte(accountID + "\x00" + region + "\x00" + function + "\x00" + queueARN))
+	b[6] = b[6]&0x0f | 0x50
+	b[8] = b[8]&0x3f | 0x80
+	return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:16])
+}
+
+func mappingShape(m eventsource.Mapping, state string) *lambda.EventSourceMappingConfiguration {
+	return &lambda.EventSourceMappingConfiguration{
+		UUID:                           aws.String(m.UUID),
+		BatchSize:                      aws.Int64(m.BatchSize),
+		MaximumBatchingWindowInSeconds: aws.Int64(0),
+		EventSourceArn:                 aws.String(m.EventSourceARN),
+		FunctionArn:                    aws.String(m.FunctionARN),
+		LastModified:                   aws.Time(m.LastModified),
+		State:                          aws.String(state),
+		StateTransitionReason:          aws.String(m.StateTransitionReason),
+	}
+}
+
+func (d Deps) mappings(ctx context.Context) (MappingStore, error) {
+	if d.Mappings == nil {
+		return nil, serviceError("event-source mappings are unavailable")
+	}
+	return d.Mappings(ctx)
+}
+
+// createEventSourceMapping resolves the function, reads the queue as its
+// execution role, and records an Enabled mapping its poller picks up.
+func (d Deps) createEventSourceMapping(ctx context.Context, c *call) (reply, error) {
+	in, err := decodeMappingCreate(c.Body)
+	if err != nil {
+		return reply{}, err
+	}
+	if err := checkQueueARN(c.Region, c.AccountID, in.QueueARN); err != nil {
+		return reply{}, err
+	}
+	name, q, err := resolveFunctionName(c.Region, c.AccountID, in.FunctionName)
+	if err != nil {
+		return reply{}, err
+	}
+	if q != "" {
+		return reply{}, unsupported("a qualified FunctionName")
+	}
+	rec, _, err := d.Meta.Get(ctx, c.AccountID, c.Region, name)
+	if errors.Is(err, function.ErrNotFound) {
+		return reply{}, notFound(c.Region, c.AccountID, name)
+	}
+	if err != nil {
+		return reply{}, err
+	}
+	if d.Sources == nil {
+		return reply{}, serviceError("event-source mappings are unavailable")
+	}
+	visibility, err := d.Sources.QueueVisibilityTimeout(ctx, rec, in.QueueARN)
+	if err != nil {
+		return reply{}, err
+	}
+	if rec.Timeout > visibility {
+		return reply{}, invalid("Queue visibility timeout: %d seconds is less than Function timeout: %d seconds", visibility, rec.Timeout)
+	}
+	store, err := d.mappings(ctx)
+	if err != nil {
+		return reply{}, err
+	}
+	uuid := mappingUUID(c.AccountID, c.Region, name, in.QueueARN)
+	m := eventsource.Mapping{
+		FormatVersion: eventsource.FormatVersion, UUID: uuid, AccountID: c.AccountID, Region: c.Region,
+		FunctionName: name, FunctionARN: arn.FormatLambdaFunction(c.Region, c.AccountID, name),
+		EventSourceARN: in.QueueARN, BatchSize: in.BatchSize, State: eventsource.StateEnabled,
+		StateTransitionReason: eventsource.ReasonUserInitiated, LastModified: time.Now().UTC().Truncate(time.Millisecond),
+	}
+	if err := store.Create(ctx, m); err != nil {
+		if errors.Is(err, eventsource.ErrConflict) {
+			return reply{}, awserrors.Errorf(awserrors.ErrorResourceConflictException,
+				"An event source mapping with SQS arn (\" %s \") and function (\" %s \") already exists. Please update or delete the existing mapping with UUID %s",
+				in.QueueARN, name, uuid)
+		}
+		return reply{}, err
+	}
+	return reply{Status: http.StatusAccepted, Body: mappingShape(m, "Creating")}, nil
+}
+
+// ownMapping reads a mapping of the caller's account and Region.
+func (d Deps) ownMapping(ctx context.Context, c *call) (MappingStore, eventsource.Mapping, uint64, error) {
+	if !mappingUUIDRE.MatchString(c.Params[0]) {
+		return nil, eventsource.Mapping{}, 0, mappingNotFound()
+	}
+	store, err := d.mappings(ctx)
+	if err != nil {
+		return nil, eventsource.Mapping{}, 0, err
+	}
+	m, rev, err := store.Get(ctx, c.Params[0])
+	if errors.Is(err, eventsource.ErrNotFound) || (err == nil && (m.AccountID != c.AccountID || m.Region != c.Region)) {
+		return nil, eventsource.Mapping{}, 0, mappingNotFound()
+	}
+	return store, m, rev, err
+}
+
+func (d Deps) getEventSourceMapping(ctx context.Context, c *call) (reply, error) {
+	_, m, _, err := d.ownMapping(ctx, c)
+	if err != nil {
+		return reply{}, err
+	}
+	return reply{Status: http.StatusOK, Body: mappingShape(m, m.State)}, nil
+}
+
+// deleteEventSourceMapping records Deleting and returns; the poller stops its
+// worker and only then removes the record.
+func (d Deps) deleteEventSourceMapping(ctx context.Context, c *call) (reply, error) {
+	for range 3 {
+		store, m, rev, err := d.ownMapping(ctx, c)
+		if err != nil {
+			return reply{}, err
+		}
+		if m.State == eventsource.StateDeleting {
+			return reply{Status: http.StatusAccepted, Body: mappingShape(m, m.State)}, nil
+		}
+		m.State = eventsource.StateDeleting
+		m.StateTransitionReason = eventsource.ReasonUserInitiated
+		m.LastModified = time.Now().UTC().Truncate(time.Millisecond)
+		_, err = store.Update(ctx, m, rev)
+		if errors.Is(err, eventsource.ErrConflict) {
+			continue
+		}
+		if err != nil {
+			return reply{}, err
+		}
+		return reply{Status: http.StatusAccepted, Body: mappingShape(m, m.State)}, nil
+	}
+	return reply{}, serviceError("the mapping changed concurrently; retry the request")
+}
+
+// listEventSourceMappings is refused: the mapping store can only be listed
+// whole, and a request must not wait on the whole inventory.
+func (d Deps) listEventSourceMappings(_ context.Context, _ *call) (reply, error) {
+	return reply{}, invalid("ListEventSourceMappings is not supported in this build; use GetEventSourceMapping with the mapping UUID")
+}

spinifex/domains/lambda/awsapi/eventsource_test.go

@@ -0,0 +1,230 @@
+package awsapi_test
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"sync"
+	"testing"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/internal/testkit/gwtest"
+	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/eventsource"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type fakeSources struct {
+	mu         sync.Mutex
+	visibility int64
+	err        error
+	asRole     []string
+}
+
+func (f *fakeSources) QueueVisibilityTimeout(_ context.Context, rec function.Record, _ string) (int64, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.asRole = append(f.asRole, rec.Role)
+	return f.visibility, f.err
+}
+
+const mappingsPath = "/2015-03-31/event-source-mappings/"
+
+var queueARN = "arn:aws:sqs:" + gwtest.Region + ":" + gwtest.AccountID + ":orders"
+
+func newMappingEnv(t *testing.T, statements ...iamdomain.Statement) (*env, *eventsource.Store, *fakeSources) {
+	t.Helper()
+	e := newEnv(t, statements...)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) { b["Timeout"] = 30 }))
+	require.NoError(t, err)
+	_, _, js := testutil.StartTestJetStream(t)
+	store, err := eventsource.Open(t.Context(), js)
+	require.NoError(t, err)
+	src := &fakeSources{visibility: 30}
+	e.deps.Mappings = func(context.Context) (awsapi.MappingStore, error) { return store, nil }
+	e.deps.Sources = src
+	return e, store, src
+}
+
+func mappingBody(mutate func(map[string]any)) map[string]any {
+	b := map[string]any{"FunctionName": "hello", "EventSourceArn": queueARN}
+	if mutate != nil {
+		mutate(b)
+	}
+	return b
+}
+
+func decodeMapping(t *testing.T, raw []byte) map[string]any {
+	t.Helper()
+	var out map[string]any
+	require.NoError(t, json.Unmarshal(raw, &out))
+	return out
+}
+
+func TestEventSourceMapping_CreateGetDelete(t *testing.T) {
+	e, store, src := newMappingEnv(t)
+	rec, err := e.do(http.MethodPost, mappingsPath, mappingBody(func(b map[string]any) { b["BatchSize"] = 4 }))
+	require.NoError(t, err)
+	require.Equal(t, http.StatusAccepted, rec.Code)
+	created := decodeMapping(t, rec.Body.Bytes())
+	assert.Equal(t, "Creating", created["State"])
+	assert.EqualValues(t, 4, created["BatchSize"])
+	assert.Equal(t, queueARN, created["EventSourceArn"])
+	assert.Equal(t, []string{roleARN}, src.asRole, "the queue is read as the function's execution role")
+	uuid, _ := created["UUID"].(string)
+
+	stored, _, err := store.Get(t.Context(), uuid)
+	require.NoError(t, err)
+	assert.Equal(t, eventsource.StateEnabled, stored.State)
+
+	rec, err = e.do(http.MethodGet, mappingsPath+uuid, nil)
+	require.NoError(t, err)
+	assert.Equal(t, "Enabled", decodeMapping(t, rec.Body.Bytes())["State"])
+
+	_, err = e.do(http.MethodPost, mappingsPath, mappingBody(nil))
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
+
+	for range 2 {
+		rec, err = e.do(http.MethodDelete, mappingsPath+uuid, nil)
+		require.NoError(t, err)
+		require.Equal(t, http.StatusAccepted, rec.Code)
+		assert.Equal(t, "Deleting", decodeMapping(t, rec.Body.Bytes())["State"])
+	}
+	stored, _, err = store.Get(t.Context(), uuid)
+	require.NoError(t, err, "delete leaves the record for the poller to remove")
+	assert.Equal(t, eventsource.StateDeleting, stored.State)
+}
+
+func TestEventSourceMapping_ConcurrentDuplicateCreatesYieldOneMapping(t *testing.T) {
+	first, store, src := newMappingEnv(t)
+	const n = 8
+	gateways := []*env{first}
+	for len(gateways) < n {
+		e := newEnv(t)
+		_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) { b["Timeout"] = 30 }))
+		require.NoError(t, err)
+		e.deps.Mappings = first.deps.Mappings
+		e.deps.Sources = src
+		gateways = append(gateways, e)
+	}
+	codes := make(chan error, n)
+	var wg sync.WaitGroup
+	for _, e := range gateways {
+		wg.Go(func() {
+			_, err := e.do(http.MethodPost, mappingsPath, mappingBody(nil))
+			codes <- err
+		})
+	}
+	wg.Wait()
+	close(codes)
+	created := 0
+	for err := range codes {
+		if err == nil {
+			created++
+			continue
+		}
+		testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
+	}
+	assert.Equal(t, 1, created)
+	entries, err := store.List(t.Context())
+	require.NoError(t, err)
+	assert.Len(t, entries, 1)
+}
+
+func TestEventSourceMapping_ListIsRefusedWithoutReadingTheStore(t *testing.T) {
+	e, store, _ := newMappingEnv(t)
+	_, err := e.do(http.MethodPost, mappingsPath, mappingBody(nil))
+	require.NoError(t, err)
+	for _, path := range []string{mappingsPath, mappingsPath + "?FunctionName=hello", mappingsPath + "?MaxItems=2"} {
+		_, err = e.do(http.MethodGet, path, nil)
+		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+	}
+	entries, err := store.List(t.Context())
+	require.NoError(t, err)
+	assert.Len(t, entries, 1, "the refusal leaves the mapping in place")
+}
+
+func TestEventSourceMapping_RefusesWhatThisBuildDoesNotImplement(t *testing.T) {
+	e, store, _ := newMappingEnv(t)
+	cases := map[string]func(map[string]any){
+		"FIFO queue":      func(b map[string]any) { b["EventSourceArn"] = queueARN + ".fifo" },
+		"another account": func(b map[string]any) { b["EventSourceArn"] = "arn:aws:sqs:" + gwtest.Region + ":210987654321:orders" },
+		"another Region":  func(b map[string]any) { b["EventSourceArn"] = "arn:aws:sqs:us-east-1:" + gwtest.AccountID + ":orders" },
+		"not SQS": func(b map[string]any) {
+			b["EventSourceArn"] = "arn:aws:kinesis:" + gwtest.Region + ":" + gwtest.AccountID + ":stream/s"
+		},
+		"filter criteria":   func(b map[string]any) { b["FilterCriteria"] = map[string]any{"Filters": []any{}} },
+		"disabled":          func(b map[string]any) { b["Enabled"] = false },
+		"batching window":   func(b map[string]any) { b["MaximumBatchingWindowInSeconds"] = 5 },
+		"batch over 10":     func(b map[string]any) { b["BatchSize"] = 11 },
+		"batch zero":        func(b map[string]any) { b["BatchSize"] = 0 },
+		"qualified name":    func(b map[string]any) { b["FunctionName"] = "hello:1" },
+		"missing source":    func(b map[string]any) { delete(b, "EventSourceArn") },
+		"missing function":  func(b map[string]any) { delete(b, "FunctionName") },
+		"starting position": func(b map[string]any) { b["StartingPosition"] = "LATEST" },
+	}
+	for name, mutate := range cases {
+		t.Run(name, func(t *testing.T) {
+			_, err := e.do(http.MethodPost, mappingsPath, mappingBody(mutate))
+			testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+		})
+	}
+	entries, err := store.List(t.Context())
+	require.NoError(t, err)
+	assert.Empty(t, entries)
+}
+
+func TestEventSourceMapping_VisibilityTimeoutMustCoverTheFunctionTimeout(t *testing.T) {
+	e, store, src := newMappingEnv(t)
+	src.visibility = 29
+	_, err := e.do(http.MethodPost, mappingsPath, mappingBody(nil))
+	testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+
+	src.visibility, src.err = 0, awserrors.Errorf(awserrors.ErrorAccessDeniedException, "denied")
+	_, err = e.do(http.MethodPost, mappingsPath, mappingBody(nil))
+	testutil.RequireAWSCode(t, err, awserrors.ErrorAccessDeniedException)
+
+	entries, err := store.List(t.Context())
+	require.NoError(t, err)
+	assert.Empty(t, entries)
+
+	_, err = e.do(http.MethodPost, mappingsPath, mappingBody(func(b map[string]any) { b["FunctionName"] = "absent" }))
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+}
+
+func TestEventSourceMapping_AnotherAccountsMappingIsNotFound(t *testing.T) {
+	e, store, _ := newMappingEnv(t)
+	m := eventsource.Mapping{
+		FormatVersion: eventsource.FormatVersion, UUID: "0a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d", AccountID: "210987654321",
+		Region: gwtest.Region, FunctionName: "hello", EventSourceARN: "arn:aws:sqs:" + gwtest.Region + ":210987654321:orders",
+		BatchSize: 1, State: eventsource.StateEnabled,
+	}
+	require.NoError(t, store.Create(t.Context(), m))
+	_, err := e.do(http.MethodGet, mappingsPath+m.UUID, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+	_, err = e.do(http.MethodDelete, mappingsPath+m.UUID, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+	_, err = e.do(http.MethodGet, mappingsPath+"not-a-uuid", nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+}
+
+func TestEventSourceMapping_GetAndDeleteAuthorizeAgainstTheMappingARN(t *testing.T) {
+	const uuid = "0a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d"
+	mapping := "arn:aws:lambda:" + gwtest.Region + ":" + gwtest.AccountID + ":event-source-mapping:" + uuid
+	e, store, _ := newMappingEnv(t, gwtest.Statement("Allow", "*", "*"), gwtest.Statement("Deny", "lambda:*", mapping))
+	require.NoError(t, store.Create(t.Context(), eventsource.Mapping{
+		FormatVersion: eventsource.FormatVersion, UUID: uuid, AccountID: gwtest.AccountID, Region: gwtest.Region,
+		FunctionName: "hello", EventSourceARN: queueARN, BatchSize: 1, State: eventsource.StateEnabled,
+	}))
+	_, err := e.do(http.MethodGet, mappingsPath+uuid, nil)
+	gwtest.AssertDenied(t, err)
+	_, err = e.do(http.MethodDelete, mappingsPath+uuid, nil)
+	gwtest.AssertDenied(t, err)
+	m, _, err := store.Get(t.Context(), uuid)
+	require.NoError(t, err)
+	assert.Equal(t, eventsource.StateEnabled, m.State)
+}

spinifex/domains/lambda/awsapi/handler_test.go

@@ -765,8 +765,9 @@ func TestAuthorization(t *testing.T) {
 
 func TestRegistration_ResolvesActionsAndInventory(t *testing.T) {
 	reg := awsapi.NewRegistration(awsapi.Deps{})
-	assert.Equal(t, []string{"CreateFunction", "GetFunction", "GetFunctionCodeSigningConfig", "GetFunctionConfiguration", "Invoke", "ListVersionsByFunction"}, reg.Inventory.Registered)
+	assert.Equal(t, []string{"CreateEventSourceMapping", "CreateFunction", "DeleteEventSourceMapping", "GetEventSourceMapping", "GetFunction", "GetFunctionCodeSigningConfig", "GetFunctionConfiguration", "Invoke", "ListEventSourceMappings", "ListVersionsByFunction"}, reg.Inventory.Registered)
 	assert.Equal(t, "ListVersionsByFunction", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x/versions", nil)))
+	assert.Equal(t, "DeleteEventSourceMapping", reg.ResolveAction(httptest.NewRequest(http.MethodDelete, "/2015-03-31/event-source-mappings/x", nil)))
 	assert.Equal(t, "GetFunctionCodeSigningConfig", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2020-06-30/functions/x/code-signing-config", nil)))
 	assert.Equal(t, "CreateFunction", reg.ResolveAction(httptest.NewRequest(http.MethodPost, "/2015-03-31/functions", nil)))
 	assert.Equal(t, "GetFunction", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x", nil)))

spinifex/domains/lambda/awsapi/invoke.go

@@ -114,16 +114,32 @@ func (d Deps) invoke(ctx context.Context, c *call) (reply, error) {
 	if err != nil {
 		return reply{}, err
 	}
+	out, err := d.runSync(ctx, rec, revision, c.Invoke.Qualifier, c.Body)
+	if err != nil {
+		return reply{}, err
+	}
+	hdr := http.Header{}
+	hdr.Set("X-Amz-Executed-Version", out.ExecutedVersion)
+	if out.FunctionError != "" {
+		hdr.Set("X-Amz-Function-Error", out.FunctionError)
+	}
+	return reply{Status: http.StatusOK, Body: rawJSON(out.Payload), Header: hdr}, nil
+}
+
+// runSync runs one synchronous attempt on a resolved record and classifies
+// its result; every error is the AWS error Invoke returns. A nil error means
+// the result is valid, with or without a function error.
+func (d Deps) runSync(ctx context.Context, rec function.Record, revision uint64, qualifier string, payload []byte) (SyncResult, error) {
 	if rec.State != function.StateActive {
-		return reply{}, awserrors.Errorf(awserrors.ErrorResourceConflictException,
+		return SyncResult{}, awserrors.Errorf(awserrors.ErrorResourceConflictException,
 			"The operation cannot be performed at this time. The function is currently in the following state: %s", rec.State)
 	}
 	if d.Exec == nil || rec.Code.State != function.CodeStored {
-		return reply{}, serviceError("function execution is unavailable")
+		return SyncResult{}, serviceError("function execution is unavailable")
 	}
 
 	if err := ctx.Err(); err != nil {
-		return reply{}, err
+		return SyncResult{}, err
 	}
 	// Once accepted, a disconnecting caller does not cancel the attempt; only
 	// the wait is bounded, and a late result is discarded.
@@ -133,31 +149,37 @@ func (d Deps) invoke(ctx context.Context, c *call) (reply, error) {
 	out, err := d.Exec.InvokeSync(actx, SyncInvocation{
 		Function:  rec,
 		Revision:  revision,
-		Qualifier: c.Invoke.Qualifier,
+		Qualifier: qualifier,
-		Payload:   c.Body,
+		Payload:   payload,
 		WaitUntil: waitUntil,
 	})
 	if refusal, ok := errors.AsType[*RefusalError](err); ok {
 		if _, coded := awserrors.ResolveErrorCode(refusal.Err); coded {
-			return reply{}, refusal.Err
+			return SyncResult{}, refusal.Err
 		}
 		err = refusal
 	}
 	if err != nil {
 		slog.ErrorContext(ctx, "Lambda: synchronous attempt outcome unknown", "function_id", rec.FunctionID, "err", err)
-		return reply{}, serviceError("the invocation outcome is unknown")
+		return SyncResult{}, serviceError("the invocation outcome is unknown")
 	}
 	if out.ExecutedVersion == "" || len(out.Payload) > maxSyncPayload {
 		slog.ErrorContext(ctx, "Lambda: executor returned an invalid result", "function_id", rec.FunctionID,
 			"payload_bytes", len(out.Payload))
-		return reply{}, serviceError("the invocation outcome is unknown")
+		return SyncResult{}, serviceError("the invocation outcome is unknown")
 	}
-	hdr := http.Header{}
+	return out, nil
-	hdr.Set("X-Amz-Executed-Version", out.ExecutedVersion)
+}
-	if out.FunctionError != "" {
+
-		hdr.Set("X-Amz-Function-Error", out.FunctionError)
+// InvokeEventSource runs one event-source batch on $LATEST through the same
+// state, code and result checks as Invoke, without Invoke's IAM check: the
+// mapping was authorized when it was created.
+func (d Deps) InvokeEventSource(ctx context.Context, accountID, region, name string, payload []byte) (SyncResult, error) {
+	rec, revision, err := d.Meta.Get(ctx, accountID, region, name)
+	if err != nil {
+		return SyncResult{}, err
 	}
-	return reply{Status: http.StatusOK, Body: rawJSON(out.Payload), Header: hdr}, nil
+	return d.runSync(ctx, rec, revision, "", payload)
 }
 
 // checkInvokeOptions refuses every request option this build does not

spinifex/domains/lambda/awsapi/registration.go

@@ -54,6 +54,12 @@ type Deps struct {
 	AEAD cipher.AEAD
 	// Exec runs synchronous attempts; nil refuses every Invoke.
 	Exec Executor
+	// Mappings opens the event-source mapping store; nil refuses every
+	// mapping action.
+	Mappings func(ctx context.Context) (MappingStore, error)
+	// Sources reads a queue as a function's execution role; nil refuses
+	// CreateEventSourceMapping.
+	Sources SourceChecker
 }
 
 // NewRegistration returns the Lambda REST-JSON registration.
@@ -136,6 +142,8 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 		if name, _, err := resolveFunctionName(inv.Region, inv.AccountID, params[0]); err == nil {
 			resource = functionARN(inv.Region, inv.AccountID, name)
 		}
+	case "GetEventSourceMapping", "DeleteEventSourceMapping":
+		resource = mappingARN(inv.Region, inv.AccountID, params[0])
 	case "Invoke":
 		// Invoke validates deterministically before authorizing, and its
 		// resource comes from that same validated target. URL.Query drops
@@ -160,8 +168,7 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 		}
 	}
 	// Only CreateFunction and GetFunction touch the code store.
-	needsCodes := action != "GetFunctionConfiguration" && action != "GetFunctionCodeSigningConfig" &&
+	needsCodes := action == "CreateFunction" || action == "GetFunction"
-		action != "ListVersionsByFunction" && action != "Invoke"
 	if d.Meta == nil || (d.Codes == nil && needsCodes) {
 		return errors.New(awserrors.ErrorServiceException)
 	}

spinifex/domains/lambda/awsapi/routes.go

@@ -66,6 +66,30 @@ var routes = []route{
 		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
 			return d.invoke(ctx, c)
 		}},
+	{Method: "POST", Pattern: "/2015-03-31/event-source-mappings/", Action: "CreateEventSourceMapping",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.createEventSourceMapping(ctx, c)
+		}},
+	{Method: "POST", Pattern: "/2015-03-31/event-source-mappings", Action: "CreateEventSourceMapping",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.createEventSourceMapping(ctx, c)
+		}},
+	{Method: "GET", Pattern: "/2015-03-31/event-source-mappings/", Action: "ListEventSourceMappings",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.listEventSourceMappings(ctx, c)
+		}},
+	{Method: "GET", Pattern: "/2015-03-31/event-source-mappings", Action: "ListEventSourceMappings",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.listEventSourceMappings(ctx, c)
+		}},
+	{Method: "GET", Pattern: "/2015-03-31/event-source-mappings/{UUID}", Action: "GetEventSourceMapping",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.getEventSourceMapping(ctx, c)
+		}},
+	{Method: "DELETE", Pattern: "/2015-03-31/event-source-mappings/{UUID}", Action: "DeleteEventSourceMapping",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.deleteEventSourceMapping(ctx, c)
+		}},
 }
 
 var router = rest.NewRouter("lambda", routes)

spinifex/domains/lambda/eventsource/mapping.go

@@ -0,0 +1,170 @@
+// Package eventsource holds the Lambda event-source mapping record and its
+// JetStream KV store.
+package eventsource
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// FormatVersion is the only record layout this build reads or writes.
+const FormatVersion = 1
+
+// KVBucket holds every mapping, keyed by UUID; the UUID is global in AWS.
+const KVBucket = "lambda-event-source-mappings"
+
+// Mapping states this build records. A mapping is Enabled from creation and
+// Deleting from the delete request until its poller has stopped.
+const (
+	StateEnabled  = "Enabled"
+	StateDeleting = "Deleting"
+)
+
+// ReasonUserInitiated is the transition reason of a state the caller asked for.
+const ReasonUserInitiated = "USER_INITIATED"
+
+// ErrNotFound is returned for a UUID with no mapping.
+var ErrNotFound = errors.New("eventsource: mapping not found")
+
+// ErrConflict is returned when the record changed since it was read.
+var ErrConflict = errors.New("eventsource: mapping changed concurrently")
+
+// Mapping is one SQS event-source mapping.
+type Mapping struct {
+	FormatVersion  int    `json:"formatVersion"`
+	UUID           string `json:"uuid"`
+	AccountID      string `json:"accountId"`
+	Region         string `json:"region"`
+	FunctionName   string `json:"functionName"`
+	FunctionARN    string `json:"functionArn"`
+	EventSourceARN string `json:"eventSourceArn"`
+	BatchSize      int64  `json:"batchSize"`
+	State          string `json:"state"`
+	// StateTransitionReason is USER_INITIATED, or the last receive failure
+	// prefixed "PROBLEM: " while polling cannot proceed.
+	StateTransitionReason string    `json:"stateTransitionReason"`
+	LastModified          time.Time `json:"lastModified"`
+}
+
+// Store reads and writes mappings in KVBucket.
+type Store struct{ kv jetstream.KeyValue }
+
+// Open returns the store, creating its bucket on first use.
+func Open(ctx context.Context, js jetstream.KeyValueManager) (*Store, error) {
+	kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucket, 1)
+	if err != nil {
+		return nil, fmt.Errorf("eventsource: open bucket: %w", err)
+	}
+	return &Store{kv: kv}, nil
+}
+
+func key(uuid string) string { return "esm." + uuid }
+
+// Create writes a new mapping; a UUID already present is a conflict.
+func (s *Store) Create(ctx context.Context, m Mapping) error {
+	b, err := json.Marshal(m)
+	if err != nil {
+		return err
+	}
+	if _, err := s.kv.Create(ctx, key(m.UUID), b); err != nil {
+		if errors.Is(err, jetstream.ErrKeyExists) {
+			return ErrConflict
+		}
+		return fmt.Errorf("eventsource: create: %w", err)
+	}
+	return nil
+}
+
+// Get returns the mapping and its KV revision.
+func (s *Store) Get(ctx context.Context, uuid string) (Mapping, uint64, error) {
+	e, err := s.kv.Get(ctx, key(uuid))
+	if errors.Is(err, jetstream.ErrKeyNotFound) {
+		return Mapping{}, 0, ErrNotFound
+	}
+	if err != nil {
+		return Mapping{}, 0, fmt.Errorf("eventsource: get: %w", err)
+	}
+	m, err := decode(e.Value())
+	return m, e.Revision(), err
+}
+
+// Entry is one listed mapping with the revision it was read at.
+type Entry struct {
+	Mapping  Mapping
+	Revision uint64
+}
+
+// List returns every mapping in the bucket once.
+func (s *Store) List(ctx context.Context) ([]Entry, error) {
+	lister, err := s.kv.ListKeys(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("eventsource: list: %w", err)
+	}
+	defer func() { _ = lister.Stop() }()
+	var out []Entry
+	// A KV listing may repeat a key that changes while it is being listed.
+	seen := map[string]bool{}
+	for k := range lister.Keys() {
+		if seen[k] {
+			continue
+		}
+		seen[k] = true
+		e, err := s.kv.Get(ctx, k)
+		if errors.Is(err, jetstream.ErrKeyNotFound) {
+			continue
+		}
+		if err != nil {
+			return nil, fmt.Errorf("eventsource: list get: %w", err)
+		}
+		m, err := decode(e.Value())
+		if err != nil {
+			return nil, err
+		}
+		out = append(out, Entry{Mapping: m, Revision: e.Revision()})
+	}
+	return out, nil
+}
+
+// Update replaces the mapping only if it is still at revision.
+func (s *Store) Update(ctx context.Context, m Mapping, revision uint64) (uint64, error) {
+	b, err := json.Marshal(m)
+	if err != nil {
+		return 0, err
+	}
+	rev, err := s.kv.Update(ctx, key(m.UUID), b, revision)
+	if err != nil {
+		if errors.Is(err, jetstream.ErrKeyExists) {
+			return 0, ErrConflict
+		}
+		return 0, fmt.Errorf("eventsource: update: %w", err)
+	}
+	return rev, nil
+}
+
+// Remove deletes the mapping only if it is still at revision.
+func (s *Store) Remove(ctx context.Context, uuid string, revision uint64) error {
+	if err := s.kv.Delete(ctx, key(uuid), jetstream.LastRevision(revision)); err != nil {
+		if errors.Is(err, jetstream.ErrKeyExists) {
+			return ErrConflict
+		}
+		return fmt.Errorf("eventsource: remove: %w", err)
+	}
+	return nil
+}
+
+func decode(b []byte) (Mapping, error) {
+	var m Mapping
+	if err := json.Unmarshal(b, &m); err != nil {
+		return Mapping{}, fmt.Errorf("eventsource: decode: %w", err)
+	}
+	if m.FormatVersion != FormatVersion {
+		return Mapping{}, fmt.Errorf("eventsource: record format %d is not %d", m.FormatVersion, FormatVersion)
+	}
+	return m, nil
+}

spinifex/domains/lambda/eventsource/mapping_test.go

@@ -0,0 +1,42 @@
+package eventsource_test
+
+import (
+	"testing"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/eventsource"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestStore_RevisionsGuardEveryChange(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	s, err := eventsource.Open(t.Context(), js)
+	require.NoError(t, err)
+
+	entries, err := s.List(t.Context())
+	require.NoError(t, err)
+	assert.Empty(t, entries)
+
+	m := eventsource.Mapping{FormatVersion: eventsource.FormatVersion, UUID: "u1", State: eventsource.StateEnabled}
+	require.NoError(t, s.Create(t.Context(), m))
+	require.ErrorIs(t, s.Create(t.Context(), m), eventsource.ErrConflict)
+
+	got, rev, err := s.Get(t.Context(), "u1")
+	require.NoError(t, err)
+	assert.Equal(t, m, got)
+
+	m.State = eventsource.StateDeleting
+	newRev, err := s.Update(t.Context(), m, rev)
+	require.NoError(t, err)
+	_, err = s.Update(t.Context(), m, rev)
+	require.ErrorIs(t, err, eventsource.ErrConflict, "a stale revision never overwrites")
+	require.ErrorIs(t, s.Remove(t.Context(), "u1", rev), eventsource.ErrConflict, "a stale revision never removes")
+
+	require.NoError(t, s.Remove(t.Context(), "u1", newRev))
+	_, _, err = s.Get(t.Context(), "u1")
+	require.ErrorIs(t, err, eventsource.ErrNotFound)
+	entries, err = s.List(t.Context())
+	require.NoError(t, err)
+	assert.Empty(t, entries)
+}

spinifex/domains/lambda/function/activate.go

@@ -0,0 +1,155 @@
+package function
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"strings"
+	"time"
+
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// Readiness reports why a function whose code is stored cannot run yet, or nil
+// when it can. Its error text becomes the function's StateReason.
+type Readiness func(ctx context.Context, rec Record) error
+
+// SetReadiness enables activation. Without it no function ever leaves Pending.
+func (s *MetaService) SetReadiness(r Readiness) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.ready = r
+	if s.nudge == nil {
+		s.nudge = make(chan struct{}, 1)
+	}
+}
+
+func (s *MetaService) readiness() (Readiness, chan struct{}) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.ready, s.nudge
+}
+
+// wake asks the activation loop for an early pass; it never blocks.
+func (s *MetaService) wake() {
+	if _, nudge := s.readiness(); nudge != nil {
+		select {
+		case nudge <- struct{}{}:
+		default:
+		}
+	}
+}
+
+// RunActivation activates pending functions until ctx ends: once at start, on
+// every code-stored completion, and every interval, so a crash between storing
+// the code and activating it is converged by the next pass.
+func (s *MetaService) RunActivation(ctx context.Context, interval time.Duration) {
+	ready, nudge := s.readiness()
+	if ready == nil {
+		return
+	}
+	t := time.NewTicker(interval)
+	defer t.Stop()
+	for {
+		if err := s.ActivatePending(ctx); err != nil && ctx.Err() == nil {
+			slog.Warn("Lambda: activation pass incomplete; retrying", "err", err)
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case <-nudge:
+		case <-t.C:
+		}
+	}
+}
+
+// ActivatePending makes one pass over every account's functions, moving each
+// Pending function with stored code to Active when readiness allows it.
+func (s *MetaService) ActivatePending(ctx context.Context) error {
+	ready, _ := s.readiness()
+	if ready == nil {
+		return nil
+	}
+	var accounts []string
+	names := s.js.KeyValueStoreNames(ctx)
+	for name := range names.Name() {
+		if id, ok := strings.CutPrefix(name, KVBucketAccountPrefix); ok {
+			accounts = append(accounts, id)
+		}
+	}
+	if err := names.Error(); err != nil {
+		return fmt.Errorf("lambda: list account buckets: %w", err)
+	}
+	var errs []error
+	for _, accountID := range accounts {
+		if err := s.activateAccount(ctx, accountID, ready); err != nil {
+			errs = append(errs, err)
+		}
+	}
+	return errors.Join(errs...)
+}
+
+func (s *MetaService) activateAccount(ctx context.Context, accountID string, ready Readiness) error {
+	kv, err := s.bucket(ctx, accountID)
+	if err != nil {
+		return err
+	}
+	keys, err := kv.ListKeysFiltered(ctx, kvFunctionKeyPrefix+">")
+	if errors.Is(err, jetstream.ErrNoKeysFound) {
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("lambda: list functions of account %s: %w", accountID, err)
+	}
+	var errs []error
+	for key := range keys.Keys() {
+		if err := s.activateOne(ctx, kv, key, ready); err != nil {
+			errs = append(errs, err)
+		}
+	}
+	return errors.Join(errs...)
+}
+
+// activateOne writes the readiness outcome by compare-and-swap at the revision
+// it read, so a concurrent update or delete wins and is never overwritten.
+func (s *MetaService) activateOne(ctx context.Context, kv jetstream.KeyValue, key string, ready Readiness) error {
+	entry, err := kv.Get(ctx, key)
+	if errors.Is(err, jetstream.ErrKeyNotFound) || errors.Is(err, jetstream.ErrKeyDeleted) {
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("lambda: read %s: %w", key, err)
+	}
+	rec, err := decode(entry.Value())
+	if err != nil {
+		return err
+	}
+	if rec.State != StatePending || rec.Code.State != CodeStored {
+		return nil
+	}
+	if notReady := ready(ctx, rec); notReady != nil {
+		if rec.StateReason == notReady.Error() {
+			return nil
+		}
+		rec.StateReasonCode, rec.StateReason = ReasonCodeCreating, notReady.Error()
+	} else {
+		rec.State, rec.StateReasonCode, rec.StateReason = StateActive, "", ""
+	}
+	data, err := json.Marshal(rec)
+	if err != nil {
+		return err
+	}
+	if _, err := kv.Update(ctx, key, data, entry.Revision()); err != nil {
+		if errors.Is(err, jetstream.ErrKeyExists) {
+			return nil
+		}
+		return fmt.Errorf("lambda: write readiness of %s: %w", key, err)
+	}
+	if rec.State == StateActive {
+		slog.Info("Lambda: function active", "account_id", rec.AccountID, "region", rec.Region,
+			"function_name", rec.FunctionName, "function_id", rec.FunctionID)
+	}
+	return nil
+}

spinifex/domains/lambda/function/activate_test.go

@@ -0,0 +1,120 @@
+package function_test
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	actAccount = "000000000001"
+	actRegion  = "ap-southeast-2"
+	actDigest  = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+)
+
+func newOwner(t *testing.T) *function.MetaService {
+	t.Helper()
+	_, nc, _ := testutil.StartTestJetStream(t)
+	return function.NewMetaService(testutil.NewJetStream(t, nc))
+}
+
+func create(t *testing.T, svc *function.MetaService, name string) uint64 {
+	t.Helper()
+	resp, err := svc.Create(t.Context(), &function.CreateRequest{Region: actRegion, Record: function.Record{
+		FunctionID: "id-" + name, FunctionName: name, Role: "arn:aws:iam::000000000001:role/r",
+		Runtime: "provided.al2023", Handler: "bootstrap", Timeout: 3, MemorySize: 128,
+		Code: function.Code{RevisionID: "rev-" + name},
+	}}, actAccount)
+	require.NoError(t, err)
+	return resp.Revision
+}
+
+func get(t *testing.T, svc *function.MetaService, name string) *function.GetResponse {
+	t.Helper()
+	resp, err := svc.Get(t.Context(), &function.GetRequest{Region: actRegion, FunctionName: name}, actAccount)
+	require.NoError(t, err)
+	return resp
+}
+
+func storeCode(t *testing.T, svc *function.MetaService, name string, rev uint64) {
+	t.Helper()
+	_, err := svc.CodeStored(t.Context(), &function.CodeStoredRequest{Region: actRegion, FunctionName: name, Revision: rev, SHA256: actDigest, Size: 9}, actAccount)
+	require.NoError(t, err)
+}
+
+func TestActivation_WithoutReadinessNothingLeavesPending(t *testing.T) {
+	svc := newOwner(t)
+	storeCode(t, svc, "hello", create(t, svc, "hello"))
+
+	require.NoError(t, svc.ActivatePending(t.Context()))
+	ctx, cancel := context.WithCancel(t.Context())
+	cancel()
+	svc.RunActivation(ctx, time.Millisecond)
+
+	got := get(t, svc, "hello").Record
+	assert.Equal(t, function.StatePending, got.State)
+	assert.Equal(t, function.ReasonCreating, got.StateReason)
+}
+
+func TestActivation_UnmetConditionStaysPendingWithItsReason(t *testing.T) {
+	svc := newOwner(t)
+	storeCode(t, svc, "hello", create(t, svc, "hello"))
+	svc.SetReadiness(func(context.Context, function.Record) error { return errors.New("The node is not serving.") })
+
+	require.NoError(t, svc.ActivatePending(t.Context()))
+	first := get(t, svc, "hello")
+	assert.Equal(t, function.StatePending, first.Record.State)
+	assert.Equal(t, function.ReasonCodeCreating, first.Record.StateReasonCode)
+	assert.Equal(t, "The node is not serving.", first.Record.StateReason)
+
+	require.NoError(t, svc.ActivatePending(t.Context()))
+	assert.Equal(t, first.Revision, get(t, svc, "hello").Revision, "an unchanged reason is not rewritten")
+}
+
+func TestActivation_ReadyStoredFunctionBecomesActive(t *testing.T) {
+	svc := newOwner(t)
+	storeCode(t, svc, "hello", create(t, svc, "hello"))
+	storingRev := create(t, svc, "storing")
+	var seen []string
+	svc.SetReadiness(func(_ context.Context, rec function.Record) error {
+		seen = append(seen, rec.FunctionName)
+		return nil
+	})
+
+	require.NoError(t, svc.ActivatePending(t.Context()))
+
+	got := get(t, svc, "hello").Record
+	assert.Equal(t, function.StateActive, got.State)
+	assert.Empty(t, got.StateReason)
+	assert.Empty(t, got.StateReasonCode)
+	assert.Equal(t, function.CodeStored, got.Code.State)
+	storing := get(t, svc, "storing")
+	assert.Equal(t, function.StatePending, storing.Record.State, "a function still storing its code is never activated")
+	assert.Equal(t, storingRev, storing.Revision)
+	assert.Equal(t, []string{"hello"}, seen)
+}
+
+func TestActivation_CodeStoredWakesTheLoop(t *testing.T) {
+	svc := newOwner(t)
+	storeCode(t, svc, "first", create(t, svc, "first"))
+	// Readiness is enabled after first is stored, so no wake is buffered and
+	// only the start pass can activate first.
+	svc.SetReadiness(func(context.Context, function.Record) error { return nil })
+	ctx, cancel := context.WithCancel(t.Context())
+	done := make(chan struct{})
+	go func() { defer close(done); svc.RunActivation(ctx, time.Hour) }()
+	t.Cleanup(func() { cancel(); <-done })
+	require.Eventually(t, func() bool { return get(t, svc, "first").Record.State == function.StateActive },
+		10*time.Second, 20*time.Millisecond, "the start pass activates what was already stored")
+
+	storeCode(t, svc, "hello", create(t, svc, "hello"))
+
+	require.Eventually(t, func() bool { return get(t, svc, "hello").Record.State == function.StateActive },
+		10*time.Second, 20*time.Millisecond, "after the start pass, only the code-stored wake can activate within the hour")
+}

spinifex/domains/lambda/function/meta.go

@@ -44,6 +44,8 @@ type MetaService struct {
 
 	mu      sync.Mutex
 	buckets map[string]jetstream.KeyValue
+	ready   Readiness
+	nudge   chan struct{}
 }
 
 // NewMetaService builds the service over a JetStream handle.
@@ -306,6 +308,7 @@ func (s *MetaService) CodeStored(ctx context.Context, req *CodeStoredRequest, ac
 		return nil, fmt.Errorf("lambda: update function record: %w", err)
 	}
 	attach(ctx, kv, up, upRev, rev)
+	s.wake()
 	return &CodeStoredResponse{Record: rec, Revision: rev}, nil
 }
 

spinifex/domains/ochre/fakes_test.go

@@ -360,6 +360,8 @@ func (f *fakeInstanceLauncher) TerminateSystemInstance(instanceID string) error
 	return nil
 }
 
+func (f *fakeInstanceLauncher) RebootSystemInstance(string) error { return nil }
+
 // fakeAMIResolver always resolves the same serving AMI.
 type fakeAMIResolver struct{}
 

spinifex/domains/rds/instance/delete.go

@@ -66,6 +66,12 @@ func (s *Service) DeleteDBInstance(ctx context.Context, input *rds.DeleteDBInsta
 			"DB instance %s is already being deleted with %s; a retry must repeat that choice", id, snapshotChoice(rec.FinalSnapshotIdentifier))
 	}
 
+	// The handler that accepted the delete tears it down, and the reconciler
+	// resumes one whose handler is gone, so a repeat only reports the state.
+	if rec.Status == StatusDeleting {
+		return &rds.DeleteDBInstanceOutput{DBInstance: s.projectDBInstance(rec)}, nil
+	}
+
 	// Reserve the final snapshot before the instance enters deleting. A collision
 	// therefore fails while the VM and data volume are still intact, and a crash
 	// afterwards leaves a creating record the reconciler can resolve.
@@ -74,29 +80,29 @@ func (s *Service) DeleteDBInstance(ctx context.Context, input *rds.DeleteDBInsta
 		return nil, err
 	}
 
-	// A repeat of a delete already under way re-runs the teardown rather than
+	now := time.Now().UTC()
-	// being rejected: the first call may have died partway through it.
+	rec.Status = StatusDeleting
-	if rec.Status != StatusDeleting {
+	rec.FailureReason = ""
-		now := time.Now().UTC()
+	rec.FinalSnapshotIdentifier = finalSnapshot
-		rec.Status = StatusDeleting
+	rec.TransitionStartedAt = &now
-		rec.FailureReason = ""
+	rec.UpdatedAt = now
-		rec.FinalSnapshotIdentifier = finalSnapshot
+	if err := updateJSON(ctx, kv, DBInstanceKey(id), rev, rec); err != nil {
-		rec.TransitionStartedAt = &now
+		if reservation.creator {
-		rec.UpdatedAt = now
+			s.rollbackFinalSnapshotReservation(ctx, kv, finalSnapshot, reservation.revision)
-		if err := updateJSON(ctx, kv, DBInstanceKey(id), rev, rec); err != nil {
-			if reservation.creator {
-				s.rollbackFinalSnapshotReservation(ctx, kv, finalSnapshot, reservation.revision)
-			}
-			if errors.Is(err, kvstore.ErrConflict) {
-				return nil, awserrors.Errorf(awserrors.ErrorDBInstanceInvalidState,
-					"DB instance %s changed state concurrently; retry the delete", id)
-			}
-			return nil, err
 		}
-		s.RecordEvent(ctx, accountID, EventSourceTypeDBInstance, id, "DB instance deleted.", EventCategoryDeletion)
+		if errors.Is(err, kvstore.ErrConflict) {
+			return nil, awserrors.Errorf(awserrors.ErrorDBInstanceInvalidState,
+				"DB instance %s changed state concurrently; retry the delete", id)
+		}
+		return nil, err
 	}
+	s.RecordEvent(ctx, accountID, EventSourceTypeDBInstance, id, "DB instance deleted.", EventCategoryDeletion)
 
-	if err := s.teardownDBInstance(ctx, kv, accountID, rec, reservation.creator); err != nil {
+	// Ends exactly when the reconciler's guard does, so the teardown is this
+	// handler's alone until the reconciler may resume it.
+	teardownCtx, cancel := context.WithDeadline(ctx, deleteClaimEnds(rec))
+	defer cancel()
+	if err := s.teardownDBInstance(teardownCtx, kv, accountID, rec, reservation.creator); err != nil {
 		// The reconciler finishes the delete once the modify lets go.
 		if errors.Is(err, errModifyInProgress) {
 			return &rds.DeleteDBInstanceOutput{DBInstance: s.projectDBInstance(rec)}, nil
@@ -202,6 +208,11 @@ func (s *Service) teardownDBInstance(ctx context.Context, kv *kvstore.Bucket, ac
 		s.stopEngineOrRecordFallback(ctx, accountID, rec, "deleting")
 	}
 
+	// The engine stop swallows a deadline, and a handler past its claim must not
+	// start a terminate the reconciler may already be running.
+	if err := ctx.Err(); err != nil {
+		return err
+	}
 	if err := s.terminateInstanceVM(ctx, rec.InstanceID, rec.DataVolumeID); err != nil {
 		return fmt.Errorf("rds: terminate the VM behind %s: %w", rec.DBInstanceIdentifier, err)
 	}

spinifex/domains/rds/instance/delete_test.go

@@ -1,9 +1,11 @@
 package instance
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"testing"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/rds"
@@ -176,13 +178,12 @@ func TestDeleteDBInstance_ReservesTheFinalSnapshotBeforeCuttingIt(t *testing.T)
 }
 
 // A dead worker can leave the EC2 snapshot cut while its RDS record still says
-// creating. A retry adopts that exact account-scoped snapshot instead of
+// creating. The resumed teardown adopts that exact account-scoped snapshot
-// cutting another one and permanently pinning the source volume twice.
+// instead of cutting another one and permanently pinning the source volume twice.
-func TestDeleteDBInstance_AdoptsAnInterruptedFinalSnapshot(t *testing.T) {
+func TestReconciler_AdoptsAnInterruptedFinalSnapshot(t *testing.T) {
 	t.Parallel()
 	h := newLifecycleHarness(t, false)
-	rec := availableRecord()
+	rec := abandonedDelete(availableRecord())
-	rec.Status = StatusDeleting
 	rec.FinalSnapshotIdentifier = "orders-db-final"
 	seedInstance(t, h.svc, rec)
 
@@ -201,13 +202,9 @@ func TestDeleteDBInstance_AdoptsAnInterruptedFinalSnapshot(t *testing.T) {
 		},
 	}
 
-	_, err = h.svc.DeleteDBInstance(t.Context(), &rds.DeleteDBInstanceInput{
+	require.NoError(t, onePass(t, NewReconciler(h.svc, "node-a")))
-		DBInstanceIdentifier:      aws.String(testDBID),
-		FinalDBSnapshotIdentifier: aws.String(rec.FinalSnapshotIdentifier),
-	}, testAccountID)
-	require.NoError(t, err)
 
-	assert.Empty(t, h.snaps.created, "the retry must adopt the cut snapshot, not create another")
+	assert.Empty(t, h.snaps.created, "the resumed teardown must adopt the cut snapshot, not create another")
 	stored, found := h.snapshotRecord(t, rec.FinalSnapshotIdentifier)
 	require.True(t, found)
 	assert.Equal(t, SnapshotStatusAvailable, stored.Status)
@@ -325,26 +322,18 @@ func TestDeleteDBInstance_RetainsTheVolumeWhenTheVolumeStoreReportsItInUse(t *te
 
 // A teardown that died partway through is replayed, so every step has to treat
 // a resource that is already gone as work it has done.
-func TestDeleteDBInstance_IsIdempotentAcrossARetriedTeardown(t *testing.T) {
+func TestReconciler_IsIdempotentAcrossARetriedTeardown(t *testing.T) {
 	t.Parallel()
 	h := newLifecycleHarness(t, false)
-	rec := availableRecord()
+	rec := abandonedDelete(availableRecord())
-	// The shape a delete leaves behind when its caller died mid-teardown.
-	rec.Status = StatusDeleting
 	rec.FinalSnapshotIdentifier = "orders-db-final"
 	seedInstance(t, h.svc, rec)
-
+	reconciler := NewReconciler(h.svc, "node-a")
-	input := &rds.DeleteDBInstanceInput{
+	require.NoError(t, onePass(t, reconciler))
-		DBInstanceIdentifier:      aws.String(testDBID),
-		FinalDBSnapshotIdentifier: aws.String("orders-db-final"),
-	}
-	_, err := h.svc.DeleteDBInstance(t.Context(), input, testAccountID)
-	require.NoError(t, err)
 
 	// The record left behind by the interrupted attempt, replayed.
 	seedInstance(t, h.svc, rec)
-	_, err = h.svc.DeleteDBInstance(t.Context(), input, testAccountID)
+	require.NoError(t, onePass(t, reconciler))
-	require.NoError(t, err)
 
 	assert.Len(t, h.snaps.created, 1, "a replayed teardown must not take a second final snapshot")
 	assert.False(t, h.recordExists(t, testDBID))
@@ -355,9 +344,7 @@ func TestDeleteDBInstance_IsIdempotentAcrossARetriedTeardown(t *testing.T) {
 func TestReconciler_ResumesAnInterruptedDelete(t *testing.T) {
 	t.Parallel()
 	h := newLifecycleHarness(t, false)
-	rec := availableRecord()
+	seedInstance(t, h.svc, abandonedDelete(availableRecord()))
-	rec.Status = StatusDeleting
-	seedInstance(t, h.svc, rec)
 
 	reconciler := NewReconciler(h.svc, "node-a")
 	require.NoError(t, onePass(t, reconciler))
@@ -366,6 +353,87 @@ func TestReconciler_ResumesAnInterruptedDelete(t *testing.T) {
 	assert.False(t, h.recordExists(t, testDBID))
 }
 
+// A second teardown racing the handler's sees the VM mid-shutdown as gone and
+// fails on the still-attached volume, so the handler's delete is left to it.
+func TestReconciler_LeavesADeleteItsHandlerMayStillBeRunning(t *testing.T) {
+	t.Parallel()
+	h := newLifecycleHarness(t, false)
+	rec := availableRecord()
+	started := time.Now().UTC()
+	rec.Status = StatusDeleting
+	rec.TransitionStartedAt = &started
+	seedInstance(t, h.svc, rec)
+
+	require.NoError(t, onePass(t, NewReconciler(h.svc, "node-a")))
+
+	assert.Empty(t, h.launcher.terminated)
+	assert.True(t, h.recordExists(t, testDBID))
+}
+
+// A repeat must not start a teardown of its own alongside the one under way;
+// the reconciler is what resumes an abandoned one.
+func TestDeleteDBInstance_ARepeatStartsNoTeardown(t *testing.T) {
+	t.Parallel()
+	h := newLifecycleHarness(t, false)
+	seedInstance(t, h.svc, abandonedDelete(availableRecord()))
+
+	out, err := h.svc.DeleteDBInstance(t.Context(), skipFinalSnapshot(), testAccountID)
+	require.NoError(t, err)
+
+	assert.Equal(t, string(StatusDeleting), aws.StringValue(out.DBInstance.DBInstanceStatus))
+	assert.Empty(t, h.launcher.terminated)
+	assert.True(t, h.recordExists(t, testDBID))
+}
+
+// The reconciler resumes a delete from the persisted stamp, so the handler's
+// teardown must stop at that same instant, however long the acceptance took.
+func TestDeleteDBInstance_TeardownEndsWhenTheClaimDoes(t *testing.T) {
+	t.Parallel()
+	h := newLifecycleHarness(t, false)
+	seedInstance(t, h.svc, availableRecord())
+	var claimEnds time.Time
+	h.snaps.beforeCreate = func() { stored := h.record(t); claimEnds = deleteClaimEnds(&stored) }
+
+	_, err := h.svc.DeleteDBInstance(t.Context(), &rds.DeleteDBInstanceInput{
+		DBInstanceIdentifier:      aws.String(testDBID),
+		FinalDBSnapshotIdentifier: aws.String("orders-db-final"),
+	}, testAccountID)
+	require.NoError(t, err)
+
+	require.False(t, claimEnds.IsZero())
+	assert.True(t, h.snaps.deadline.Equal(claimEnds), "deadline %s, claim ends %s", h.snaps.deadline, claimEnds)
+}
+
+// A graceful stop that runs into the claim's end is swallowed, so the teardown
+// itself has to notice and leave the terminate to the reconciler.
+func TestTeardownDBInstance_StartsNoTerminatePastItsDeadline(t *testing.T) {
+	t.Parallel()
+	h := newLifecycleHarness(t, false)
+	h.agent.silenceType(CommandStopEngine)
+	rec := availableRecord()
+	rec.Status = StatusDeleting
+	rec.FinalSnapshotIdentifier = "orders-db-final"
+	seedInstance(t, h.svc, rec)
+	kv, err := h.svc.bucket(t.Context(), testAccountID)
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond)
+	defer cancel()
+	err = h.svc.teardownDBInstance(ctx, kv, testAccountID, &rec, true)
+
+	require.ErrorIs(t, err, context.DeadlineExceeded)
+	assert.Empty(t, h.launcher.terminated)
+	assert.True(t, h.recordExists(t, testDBID))
+}
+
+// The shape a delete leaves once its handler's request budget has run out.
+func abandonedDelete(rec DBInstanceRecord) DBInstanceRecord {
+	started := time.Now().UTC().Add(-deleteTimeout - time.Minute)
+	rec.Status = StatusDeleting
+	rec.TransitionStartedAt = &started
+	return rec
+}
+
 // A stop whose caller died leaves the VM possibly still running, so the stop is
 // re-issued rather than assumed.
 func TestReconciler_ResumesAnInterruptedStop(t *testing.T) {

spinifex/domains/rds/instance/lifecycle_test.go

@@ -117,7 +117,9 @@ func (f *fakeInstanceCommander) DetachVolumeFromStopped(_ context.Context, insta
 // DescribeSnapshots reports against the data volume, which is what decides
 // between deleting the volume and retaining it.
 type fakeSnapshots struct {
-	created       []*ec2.CreateSnapshotInput
+	created []*ec2.CreateSnapshotInput
+	// The context deadline the last CreateSnapshot ran under.
+	deadline      time.Time
 	holding       []string
 	deleted       []string
 	createErr     error
@@ -132,7 +134,8 @@ type fakeSnapshots struct {
 
 var _ snapshotProvider = (*fakeSnapshots)(nil)
 
-func (f *fakeSnapshots) CreateSnapshot(_ context.Context, input *ec2.CreateSnapshotInput, _ string) (*ec2.Snapshot, error) {
+func (f *fakeSnapshots) CreateSnapshot(ctx context.Context, input *ec2.CreateSnapshotInput, _ string) (*ec2.Snapshot, error) {
+	f.deadline, _ = ctx.Deadline()
 	if f.beforeCreate != nil {
 		f.beforeCreate()
 	}

spinifex/domains/rds/instance/reconciler.go

@@ -610,6 +610,11 @@ func (r *Reconciler) reconcileStopping(ctx context.Context, kv *kvstore.Bucket,
 // did; only a teardown still stuck past the bound is called failed, which is
 // what lets the customer retry the delete.
 func (r *Reconciler) reconcileDeleting(ctx context.Context, kv *kvstore.Bucket, rev uint64, accountID string, rec *DBInstanceRecord) error {
+	// The handler that accepted the delete tears it down until its claim ends; a
+	// second teardown alongside it sees the VM mid-shutdown as gone.
+	if time.Now().Before(deleteClaimEnds(rec)) {
+		return nil
+	}
 	err := r.svc.teardownDBInstance(ctx, kv, accountID, rec, false)
 	if err == nil || errors.Is(err, errFinalSnapshotInProgress) || errors.Is(err, errModifyInProgress) {
 		return nil
@@ -795,6 +800,12 @@ func (s *Service) findEC2SnapshotFor(ctx context.Context, accountID, dbSnapshotI
 // When the transition began. A record written by an older control plane carries
 // no stamp, so its last write stands in — it is never earlier than the
 // transition, so the bound cannot be under-counted.
+// When the handler that accepted a delete gives up its teardown, measured from
+// the persisted stamp so the handler and the reconciler agree on it.
+func deleteClaimEnds(rec *DBInstanceRecord) time.Time {
+	return transitionStarted(rec).Add(deleteTimeout)
+}
+
 func transitionStarted(rec *DBInstanceRecord) time.Time {
 	if rec.TransitionStartedAt != nil {
 		return *rec.TransitionStartedAt

spinifex/operator/accountlifecycle/absence_proof_silent_node_test.go

@@ -1,89 +0,0 @@
-package accountlifecycle
-
-//test:in-package — the volume reaper and its fake cluster are unexported, and
-// these drive its force path against a node that never answers.
-
-// Every test here waits out the fixed describe fan-out deadline a silent node
-// costs, two or three times over. This file sorts first so they take parallel
-// slots before the package's short tests queue ahead of them.
-
-import (
-	"testing"
-
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// A lone node's NotFound says nothing about the node that stayed silent, so
-// the holder is not proven gone and the volume keeps its attachment.
-func TestVolumeReaperForceRetainsAVolumeWhenALoneNodeAnswersNotFoundWhileAnotherIsSilent(t *testing.T) {
-	t.Parallel()
-	assertForceRetainsTheHeldVolume(t, func(cluster *fakeCluster) {
-		cluster.fail("ec2.DescribeInstances", awserrors.ErrorInvalidInstanceIDNotFound)
-	}, []string{fakeClusterNode, "node-2"})
-}
-
-// A terminated copy is not proof either while the node that may still run the
-// holder has not answered.
-func TestVolumeReaperForceRetainsAVolumeWhenATerminatedCopyIsFoundWhileTheHolderNodeIsSilent(t *testing.T) {
-	t.Parallel()
-	assertForceRetainsTheHeldVolume(t, func(cluster *fakeCluster) {
-		cluster.reply("ec2.DescribeTerminatedInstances", instancesIn("terminated", "i-held"))
-	}, []string{fakeClusterNode, "node-2"})
-}
-
-// The holder lives on node-2, which never answers, while node-1 answers every
-// fan-out twice. A count of replies reads that as complete and the holder as
-// gone; only judging by responder identity sees that node-2 is missing.
-func TestVolumeReaperForceRetainsAVolumeWhenOneNodeAnswersTwiceAndTheHolderNodeIsSilent(t *testing.T) {
-	t.Parallel()
-	cluster := newFakeCluster(t)
-	volume := newHeldVolume(cluster, "vol-held", "i-on-silent-node")
-	cluster.echoAsNode(t, "ec2.DescribeInstances", fakeClusterNode)
-
-	reaper := newVolumeReaper(cluster.nc, fakeClusterNode, "node-2")
-	err := reaper.Delete(silentClusterCtx(t), "000000000042", Resource{Kind: "volume", ID: "vol-held"}, true)
-
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "vol-held")
-	assert.Contains(t, err.Error(), "i-on-silent-node")
-	deleted, holder, detaches := volume.state()
-	assert.False(t, deleted, "a holder on a silent node is not proven gone")
-	assert.Equal(t, "i-on-silent-node", holder)
-	assert.Zero(t, detaches)
-	assert.NotContains(t, cluster.called(), forceDetachSubject)
-}
-
-func TestForcedTeardownKeepsTheAccountWhenOneNodeAnswersTwiceAndTheHolderNodeIsSilent(t *testing.T) {
-	t.Parallel()
-	cluster := newFakeCluster(t)
-	volume := newHeldVolume(cluster, "vol-held", "i-on-silent-node")
-	cluster.echoAsNode(t, "ec2.DescribeInstances", fakeClusterNode)
-
-	accounts := newFakeAccounts()
-	engine := quietEngine(accounts, newVolumeReaper(cluster.nc, fakeClusterNode, "node-2"))
-	req := request()
-	req.Force = true
-
-	result, err := engine.Teardown(silentClusterCtx(t), req)
-
-	require.ErrorIs(t, err, ErrResourcesStuck)
-	require.Equal(t, 1, result.StuckCount())
-	var stuck Stuck
-	for _, stage := range result.Stages {
-		if len(stage.Stuck) > 0 {
-			stuck = stage.Stuck[0]
-		}
-	}
-	assert.Equal(t, "vol-held", stuck.Resource.ID)
-	assert.Contains(t, stuck.Reason, "vol-held")
-	assert.Contains(t, stuck.Reason, "i-on-silent-node")
-	assert.False(t, result.AccountDeleted)
-	assert.False(t, accounts.deleted)
-	assert.Equal(t, AccountStatusTerminating, accounts.account.Status)
-	deleted, holder, detaches := volume.state()
-	assert.False(t, deleted)
-	assert.Equal(t, "i-on-silent-node", holder)
-	assert.Zero(t, detaches)
-}

spinifex/operator/accountlifecycle/cluster.go

@@ -35,6 +35,7 @@ func NewClusterEngine(
 	svc IAMBackend,
 	buckets BucketStore,
 	cleaner PlatformCleaner,
+	options ...ClusterOption,
 ) (*Engine, error) {
 	// Refused rather than skipped. A teardown missing its bucket reaper still
 	// reports every stage drained and still deletes the account record, leaving
@@ -95,5 +96,20 @@ func NewClusterEngine(
 	// May be nil: a dry run needs no cleanup, and Teardown and Exercise refuse
 	// to start without it.
 	engine.Cleanup = cleaner
+	for _, option := range options {
+		option(engine)
+	}
 	return engine, nil
 }
+
+// ClusterOption binds an optional capability to the engine NewClusterEngine builds.
+type ClusterOption func(*Engine)
+
+// WithComputeCleanup binds the EC2 compute and storage cleanup. Teardown
+// refuses to start without both, so an engine built without them can still
+// answer a dry run but never deletes an account.
+func WithComputeCleanup(compute ComputeStage, storage StorageStage) ClusterOption {
+	return func(e *Engine) {
+		e.Compute, e.Storage = compute, storage
+	}
+}

spinifex/operator/accountlifecycle/converge.go

@@ -0,0 +1,317 @@
+package accountlifecycle
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"slices"
+	"strings"
+	"time"
+
+	operatorv1 "github.com/mulgadc/spinifex/contracts/operator/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountcleanup"
+	otelsetup "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+)
+
+// ComputeStage is the EC2 compute cleanup and its read-only remainder.
+type ComputeStage interface {
+	accountcleanup.ComputeCleaner
+	accountcleanup.ComputeRemainder
+}
+
+// StorageStage is the EC2 storage cleanup and its read-only remainder.
+type StorageStage interface {
+	accountcleanup.StorageCleaner
+	accountcleanup.StorageRemainder
+}
+
+var errComputeCleanupMissing = errors.New("teardown needs the EC2 compute and storage cleanup")
+
+// Converge bounds the loop that repeats the compute and storage cleanup inside
+// one teardown. Each cleanup call returns without waiting for teardown, so the
+// loop is what turns a terminate into a deleted volume.
+type Converge struct {
+	// Budget is the longest the loop runs, well under the job's own budget.
+	Budget time.Duration
+	// MinBackoff and MaxBackoff bound the wait between passes, which doubles.
+	MinBackoff time.Duration
+	MaxBackoff time.Duration
+	// IncompletePasses is how many passes in a row may end with a read that did
+	// not complete before the loop gives up.
+	IncompletePasses int
+}
+
+// DefaultConverge is sized under the 2 h job budget and over the 10 min
+// stuck-terminate backstop, so a wedged terminate still ends the loop clean.
+func DefaultConverge() Converge {
+	return Converge{Budget: 30 * time.Minute, MinBackoff: 5 * time.Second, MaxBackoff: time.Minute, IncompletePasses: 3}
+}
+
+func (e *Engine) convergeConfig() Converge {
+	cfg := DefaultConverge()
+	if e.Converge.Budget > 0 {
+		cfg.Budget = e.Converge.Budget
+	}
+	if e.Converge.MinBackoff > 0 {
+		cfg.MinBackoff = e.Converge.MinBackoff
+	}
+	if e.Converge.MaxBackoff > 0 {
+		cfg.MaxBackoff = e.Converge.MaxBackoff
+	}
+	if e.Converge.IncompletePasses > 0 {
+		cfg.IncompletePasses = e.Converge.IncompletePasses
+	}
+	return cfg
+}
+
+// holdsFence stops a holder that has been superseded or cancelled before it
+// issues another terminate or delete. It reads the account back and requires
+// it to still be TERMINATING under this operation and fence.
+func (e *Engine) holdsFence(ctx context.Context, req Request) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	account, err := e.Accounts.GetAccount(req.AccountID)
+	if err != nil {
+		return fmt.Errorf("read account to confirm the fence: %w", err)
+	}
+	op := account.TerminationOp
+	if account.Status != AccountStatusTerminating || op == nil || op.OperationID != req.OperationID || op.Fence != req.Fence {
+		return errFenceLost
+	}
+	return nil
+}
+
+var errFenceLost = errors.New("the account is no longer held under this operation and fence")
+
+func waitContext(ctx context.Context, d time.Duration) error {
+	timer := time.NewTimer(d)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-timer.C:
+		return nil
+	}
+}
+
+// convergent reports whether time alone can resolve an item.
+func convergent(it accountcleanup.Item) bool {
+	if it.Disposition == accountcleanup.DispositionPending {
+		return true
+	}
+	switch it.Reason {
+	case accountcleanup.ReasonTerminating, accountcleanup.ReasonVolumeDeleting, accountcleanup.ReasonTeardownUnconfirmed:
+		return true
+	}
+	return false
+}
+
+// passSignature identifies what a pass left, so two identical passes in a row
+// with nothing convergent show the loop is not going anywhere.
+func passSignature(legacy []Stuck, outcomes ...*accountcleanup.Outcome) string {
+	var lines []string
+	for _, s := range legacy {
+		lines = append(lines, "legacy/"+s.Resource.Kind+"/"+s.Resource.ID+"/"+s.Reason)
+	}
+	for _, out := range outcomes {
+		for _, it := range out.Items {
+			lines = append(lines, it.Kind+"/"+it.ResourceID+"/"+it.Disposition+"/"+it.Reason+"/"+it.State)
+		}
+		lines = append(lines, "incomplete:"+strings.Join(out.IncompleteReasons, ","))
+	}
+	slices.Sort(lines)
+	return strings.Join(lines, "\n")
+}
+
+// convergeCompute repeats the compute cleanup, the storage-position legacy
+// reapers, the storage cleanup and the remainder read until the account holds
+// no instance, spot request or volume, or the loop proves it cannot make
+// progress. It ends the teardown with the stuck items named, never with a
+// shorter list: a lost fence or a cancelled context is an error.
+func (e *Engine) convergeCompute(ctx context.Context, req Request, result *Result) error {
+	cfg := e.convergeConfig()
+	started := time.Now()
+	deadline := started.Add(cfg.Budget)
+	backoff := cfg.MinBackoff
+	wait := e.wait
+	if wait == nil {
+		wait = waitContext
+	}
+
+	attachments := StageResult{Stage: StageAttachments}
+	storage := StageResult{Stage: StageStorage}
+	var lastSignature string
+	var idlePasses, incompletePasses int
+
+	for pass := 1; ; pass++ {
+		if err := e.holdsFence(ctx, req); err != nil {
+			return err
+		}
+		computeOut, err := e.Compute.CleanupCompute(ctx, &accountcleanup.ComputeRequest{
+			AccountID: req.AccountID, CorrelationRef: req.OperationID, Force: req.Force,
+		})
+		if err != nil {
+			return fmt.Errorf("compute cleanup: %w", err)
+		}
+		legacy, err := e.runStage(ctx, StageStorage, req)
+		if err != nil {
+			return err
+		}
+		storageOut, err := e.Storage.CleanupStorage(ctx, &accountcleanup.StorageRequest{
+			AccountID: req.AccountID, CorrelationRef: req.OperationID, Force: req.Force,
+		})
+		if err != nil {
+			return fmt.Errorf("storage cleanup: %w", err)
+		}
+		computeLeft, err := e.Compute.ComputeRemaining(ctx, req.AccountID)
+		if err != nil {
+			return fmt.Errorf("compute remainder: %w", err)
+		}
+		storageLeft, err := e.Storage.StorageRemaining(ctx, req.AccountID)
+		if err != nil {
+			return fmt.Errorf("storage remainder: %w", err)
+		}
+
+		attachments.Deleted = appendDeleted(attachments.Deleted, computeOut)
+		storage.Deleted = appendDeleted(storage.Deleted, storageOut)
+		for _, r := range legacy.Deleted {
+			if !containsResource(storage.Deleted, r) {
+				storage.Deleted = append(storage.Deleted, r)
+			}
+		}
+		storage.Stuck = legacy.Stuck
+		if e.OnPass != nil {
+			e.OnPass(ctx, pass)
+		}
+		e.logger().InfoContext(ctx, "Teardown convergence pass",
+			"account_id", req.AccountID, "pass", pass,
+			"compute_remaining", len(computeLeft.Items), "storage_remaining", len(storageLeft.Items),
+			"legacy_stuck", len(legacy.Stuck), "elapsed_ms", otelsetup.Millis(time.Since(started)))
+
+		if len(computeLeft.Items) == 0 && len(storageLeft.Items) == 0 &&
+			computeLeft.Status == accountcleanup.StatusClean && storageLeft.Status == accountcleanup.StatusClean &&
+			len(legacy.Stuck) == 0 {
+			return e.finishConverge(ctx, result, attachments, storage, started)
+		}
+
+		var moving bool
+		for _, out := range []*accountcleanup.Outcome{computeOut, storageOut} {
+			moving = moving || slices.ContainsFunc(out.Items, convergent)
+		}
+		if len(computeOut.IncompleteReasons)+len(storageOut.IncompleteReasons)+len(computeLeft.IncompleteReasons)+len(storageLeft.IncompleteReasons) > 0 {
+			incompletePasses++
+		} else {
+			incompletePasses = 0
+		}
+		signature := passSignature(legacy.Stuck, computeOut, storageOut, computeLeft, storageLeft)
+		if signature == lastSignature && !moving {
+			idlePasses++
+		} else {
+			idlePasses = 0
+		}
+		lastSignature = signature
+
+		reason := ""
+		switch {
+		case idlePasses >= 1:
+			reason = "only items time cannot resolve remain"
+		case incompletePasses >= cfg.IncompletePasses:
+			reason = "a read stayed incomplete"
+		case time.Now().Add(backoff).After(deadline):
+			reason = "the convergence budget ended"
+		}
+		if reason != "" {
+			attachments.Stuck = stuckFrom(attachments.Stuck[:0], computeOut, computeLeft)
+			storage.Stuck = append(legacy.Stuck, stuckFrom(nil, storageOut, storageLeft)...)
+			if err := e.finishConverge(ctx, result, attachments, storage, started); err != nil {
+				return err
+			}
+			e.logger().WarnContext(ctx, "Account teardown stopped converging",
+				"accountID", req.AccountID, "reason", reason, "passes", pass)
+			return fmt.Errorf("%d resources left (%s): %w", result.StuckCount(), reason, ErrResourcesStuck)
+		}
+
+		if err := wait(ctx, backoff); err != nil {
+			return err
+		}
+		backoff = min(backoff*2, cfg.MaxBackoff)
+	}
+}
+
+func (e *Engine) finishConverge(ctx context.Context, result *Result, attachments, storage StageResult, started time.Time) error {
+	elapsed := time.Since(started).String()
+	attachments.Elapsed, storage.Elapsed = elapsed, elapsed
+	for _, stage := range []StageResult{attachments, storage} {
+		result.Stages = append(result.Stages, stage)
+		if e.OnStage != nil {
+			e.OnStage(ctx, stage)
+		}
+	}
+	return nil
+}
+
+func appendDeleted(into []Resource, out *accountcleanup.Outcome) []Resource {
+	for _, it := range out.Items {
+		if it.Disposition == accountcleanup.DispositionDeleted {
+			r := Resource{Kind: it.Kind, ID: it.ResourceID}
+			if !containsResource(into, r) {
+				into = append(into, r)
+			}
+		}
+	}
+	return into
+}
+
+// stuckFrom names every item the account still holds with the reason the
+// cleanup gave for keeping it, or that the record remains. An incomplete read
+// is named too, because it proves nothing about what is left.
+func stuckFrom(into []Stuck, cleanup, remainder *accountcleanup.Outcome) []Stuck {
+	reasons := map[string]string{}
+	for _, it := range cleanup.Items {
+		if it.Reason != "" && it.Disposition != accountcleanup.DispositionDeleted {
+			reasons[it.Kind+"/"+it.ResourceID] = it.Reason
+		}
+	}
+	for _, it := range remainder.Items {
+		reason := reasons[it.Kind+"/"+it.ResourceID]
+		if reason == "" {
+			reason = it.Reason
+		}
+		into = append(into, Stuck{Resource: Resource{Kind: it.Kind, ID: it.ResourceID, Detail: it.State}, Reason: reason})
+	}
+	for _, reason := range slices.Concat(cleanup.IncompleteReasons, remainder.IncompleteReasons) {
+		into = append(into, Stuck{Resource: Resource{Kind: "inventory", ID: reason}, Reason: reason})
+	}
+	return into
+}
+
+// rereadComputeStorage reads the instances, spot requests and volumes the
+// account still holds immediately before the record is removed, so anything
+// that appeared after the convergence loop ended keeps the account. A read
+// that fails or is incomplete is named, never taken as empty.
+func (e *Engine) rereadComputeStorage(ctx context.Context, accountID string, result *Result) {
+	compute, cerr := e.Compute.ComputeRemaining(ctx, accountID)
+	storage, serr := e.Storage.StorageRemaining(ctx, accountID)
+	e.addFinalStuck(result, StageAttachments, accountID, compute, cerr)
+	e.addFinalStuck(result, StageStorage, accountID, storage, serr)
+}
+
+func (e *Engine) addFinalStuck(result *Result, stage Stage, accountID string, out *accountcleanup.Outcome, err error) {
+	var stuck []Stuck
+	if err != nil || out == nil {
+		stuck = []Stuck{{Resource: Resource{Kind: "inventory", ID: stage.String()}, Reason: operatorv1.AccountingReasonRereadIncomplete}}
+	} else {
+		stuck = stuckFrom(nil, &accountcleanup.Outcome{}, out)
+	}
+	if len(stuck) == 0 {
+		return
+	}
+	for i := range result.Stages {
+		if result.Stages[i].Stage == stage {
+			result.Stages[i].Stuck = append(result.Stages[i].Stuck, stuck...)
+			return
+		}
+	}
+	result.Stages = append(result.Stages, StageResult{Stage: stage, Elapsed: "0s", Stuck: stuck})
+}

spinifex/operator/accountlifecycle/converge_test.go

@@ -0,0 +1,256 @@
+package accountlifecycle
+
+//test:in-package — the convergence loop drives unexported engine fields and
+// the unexported backoff hook.
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountcleanup"
+)
+
+// scriptedStage answers each pass from a script, repeating the last answer.
+type scriptedStage struct {
+	cleanups   []*accountcleanup.Outcome
+	remainders []*accountcleanup.Outcome
+	cleanupN   int
+	remainN    int
+	forces     []bool
+}
+
+func pick(list []*accountcleanup.Outcome, n int) *accountcleanup.Outcome {
+	if n >= len(list) {
+		n = len(list) - 1
+	}
+	return list[n]
+}
+
+func (s *scriptedStage) run(forceIn bool) *accountcleanup.Outcome {
+	s.forces = append(s.forces, forceIn)
+	out := pick(s.cleanups, s.cleanupN)
+	s.cleanupN++
+	return out
+}
+
+func (s *scriptedStage) remaining() *accountcleanup.Outcome {
+	out := pick(s.remainders, s.remainN)
+	s.remainN++
+	return out
+}
+
+type scriptedCompute struct{ scriptedStage }
+
+func (s *scriptedCompute) CleanupCompute(_ context.Context, req *accountcleanup.ComputeRequest) (*accountcleanup.ComputeResult, error) {
+	return s.run(req.Force), nil
+}
+
+func (s *scriptedCompute) ComputeRemaining(context.Context, string) (*accountcleanup.Outcome, error) {
+	return s.remaining(), nil
+}
+
+type scriptedStorage struct{ scriptedStage }
+
+func (s *scriptedStorage) CleanupStorage(_ context.Context, req *accountcleanup.StorageRequest) (*accountcleanup.StorageResult, error) {
+	return s.run(req.Force), nil
+}
+
+func (s *scriptedStorage) StorageRemaining(context.Context, string) (*accountcleanup.Outcome, error) {
+	return s.remaining(), nil
+}
+
+func outcome(items ...accountcleanup.Item) *accountcleanup.Outcome {
+	out := &accountcleanup.Outcome{AccountID: testAccountID, Status: accountcleanup.StatusClean, Items: items}
+	return out
+}
+
+func incomplete(reason string) *accountcleanup.Outcome {
+	return &accountcleanup.Outcome{AccountID: testAccountID, Status: accountcleanup.StatusIncomplete, IncompleteReasons: []string{reason}}
+}
+
+func item(kind, id, state, disposition, reason string) accountcleanup.Item {
+	return accountcleanup.Item{Kind: kind, ResourceID: id, State: state, Disposition: disposition, Reason: reason}
+}
+
+// convergeEngine runs a teardown with no real delay between passes.
+func convergeEngine(accounts AccountStore, compute *scriptedCompute, storage *scriptedStorage) (*Engine, *int) {
+	engine := quietEngine(accounts)
+	engine.Compute, engine.Storage = compute, storage
+	waits := 0
+	engine.wait = func(ctx context.Context, _ time.Duration) error {
+		waits++
+		return ctx.Err()
+	}
+	return engine, &waits
+}
+
+var (
+	runningInstance     = item(accountcleanup.KindInstance, "i-1", "running", accountcleanup.DispositionPending, accountcleanup.ReasonTerminating)
+	termInstance        = item(accountcleanup.KindInstance, "i-1", "terminated", accountcleanup.DispositionDeleted, "")
+	blockedHolderVolume = item(accountcleanup.KindVolume, "vol-1", "in-use", accountcleanup.DispositionBlocked, accountcleanup.ReasonHolderPresent)
+)
+
+func TestConvergeRunningInstanceFinishesInOneJob(t *testing.T) {
+	compute := &scriptedCompute{scriptedStage{
+		cleanups:   []*accountcleanup.Outcome{outcome(runningInstance), outcome(termInstance)},
+		remainders: []*accountcleanup.Outcome{outcome(item(accountcleanup.KindInstance, "i-1", "shutting-down", accountcleanup.DispositionResidue, accountcleanup.ReasonRecordRemains)), outcome()},
+	}}
+	storage := &scriptedStorage{scriptedStage{
+		cleanups:   []*accountcleanup.Outcome{outcome(blockedHolderVolume), outcome(item(accountcleanup.KindVolume, "vol-1", "available", accountcleanup.DispositionDeleted, ""))},
+		remainders: []*accountcleanup.Outcome{outcome(item(accountcleanup.KindVolume, "vol-1", "in-use", accountcleanup.DispositionResidue, accountcleanup.ReasonRecordRemains)), outcome()},
+	}}
+	accounts := newFakeAccounts()
+	engine, waits := convergeEngine(accounts, compute, storage)
+
+	result, err := engine.Teardown(context.Background(), request())
+	if err != nil {
+		t.Fatalf("a running instance must converge in one job, got %v", err)
+	}
+	if !result.AccountDeleted || !accounts.deleted {
+		t.Fatalf("account must be deleted once the loop ends clean: %+v", result)
+	}
+	if *waits != 1 || compute.cleanupN != 2 || storage.cleanupN != 2 {
+		t.Fatalf("expected two passes and one backoff, got passes=%d/%d waits=%d", compute.cleanupN, storage.cleanupN, *waits)
+	}
+}
+
+func TestConvergeStopsOnBlockerTimeCannotResolve(t *testing.T) {
+	managed := item(accountcleanup.KindInstance, "i-managed", "running", accountcleanup.DispositionBlocked, accountcleanup.ReasonManagedServiceOwner)
+	remain := item(accountcleanup.KindInstance, "i-managed", "running", accountcleanup.DispositionResidue, accountcleanup.ReasonRecordRemains)
+	compute := &scriptedCompute{scriptedStage{cleanups: []*accountcleanup.Outcome{outcome(managed)}, remainders: []*accountcleanup.Outcome{outcome(remain)}}}
+	storage := &scriptedStorage{scriptedStage{cleanups: []*accountcleanup.Outcome{outcome()}, remainders: []*accountcleanup.Outcome{outcome()}}}
+	accounts := newFakeAccounts()
+	engine, _ := convergeEngine(accounts, compute, storage)
+	engine.Converge = Converge{Budget: time.Hour}
+
+	result, err := engine.Teardown(context.Background(), request())
+	if !errors.Is(err, ErrResourcesStuck) {
+		t.Fatalf("a non-converging blocker must end the job stuck, got %v", err)
+	}
+	if accounts.deleted {
+		t.Fatal("account must not be deleted while an instance remains")
+	}
+	if compute.cleanupN > 2 {
+		t.Fatalf("loop must end before the budget on a blocker time cannot resolve, ran %d passes", compute.cleanupN)
+	}
+	var named bool
+	for _, stage := range result.Stages {
+		for _, s := range stage.Stuck {
+			named = named || (s.Resource.ID == "i-managed" && s.Reason == accountcleanup.ReasonManagedServiceOwner)
+		}
+	}
+	if !named {
+		t.Fatalf("stuck item must be named with the cleanup's reason: %+v", result.Stages)
+	}
+}
+
+// supersededAccounts reports a different holder once the first cleanup ran.
+type supersededAccounts struct {
+	*fakeAccounts
+
+	after *scriptedCompute
+}
+
+func (s *supersededAccounts) GetAccount(id string) (*Account, error) {
+	account, err := s.fakeAccounts.GetAccount(id)
+	if err != nil || s.after.cleanupN == 0 {
+		return account, err
+	}
+	taken := *account
+	taken.TerminationOp = &TerminationOp{OperationID: "other-op", Fence: account.TerminationOp.Fence + 1}
+	return &taken, nil
+}
+
+func TestConvergeLostFenceStopsBeforeNextPass(t *testing.T) {
+	compute := &scriptedCompute{scriptedStage{
+		cleanups:   []*accountcleanup.Outcome{outcome(runningInstance)},
+		remainders: []*accountcleanup.Outcome{outcome(item(accountcleanup.KindInstance, "i-1", "shutting-down", accountcleanup.DispositionResidue, accountcleanup.ReasonRecordRemains))},
+	}}
+	storage := &scriptedStorage{scriptedStage{cleanups: []*accountcleanup.Outcome{outcome()}, remainders: []*accountcleanup.Outcome{outcome()}}}
+	accounts := &supersededAccounts{fakeAccounts: newFakeAccounts(), after: compute}
+	engine, _ := convergeEngine(accounts, compute, storage)
+
+	// The deadline lets a loop that ignores the lost fence end, so it fails
+	// the pass count below instead of hanging.
+	ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
+	defer cancel()
+	_, err := engine.Teardown(ctx, request())
+	if err == nil || errors.Is(err, ErrResourcesStuck) {
+		t.Fatalf("a lost fence must stop the loop with an error, got %v", err)
+	}
+	if compute.cleanupN != 1 {
+		t.Fatalf("no terminate may be issued after the fence is lost, ran %d passes", compute.cleanupN)
+	}
+	if accounts.deleted {
+		t.Fatal("a superseded holder must not delete the account")
+	}
+}
+
+func TestConvergeCancelledContextStopsLoop(t *testing.T) {
+	compute := &scriptedCompute{scriptedStage{
+		cleanups:   []*accountcleanup.Outcome{outcome(runningInstance)},
+		remainders: []*accountcleanup.Outcome{outcome(item(accountcleanup.KindInstance, "i-1", "shutting-down", accountcleanup.DispositionResidue, accountcleanup.ReasonRecordRemains))},
+	}}
+	storage := &scriptedStorage{scriptedStage{cleanups: []*accountcleanup.Outcome{outcome()}, remainders: []*accountcleanup.Outcome{outcome()}}}
+	engine, _ := convergeEngine(newFakeAccounts(), compute, storage)
+	ctx, cancel := context.WithCancel(context.Background())
+	engine.OnPass = func(context.Context, int) { cancel() }
+
+	if _, err := engine.Teardown(ctx, request()); !errors.Is(err, context.Canceled) {
+		t.Fatalf("a cancelled job must stop the loop, got %v", err)
+	}
+	if compute.cleanupN != 1 {
+		t.Fatalf("no pass may start after cancel, ran %d", compute.cleanupN)
+	}
+}
+
+func TestConvergeIncompleteReadsEndStuckNotClean(t *testing.T) {
+	compute := &scriptedCompute{scriptedStage{cleanups: []*accountcleanup.Outcome{incomplete(accountcleanup.ReasonInstanceInventoryIncomplete)}, remainders: []*accountcleanup.Outcome{incomplete(accountcleanup.ReasonInstanceInventoryIncomplete)}}}
+	storage := &scriptedStorage{scriptedStage{cleanups: []*accountcleanup.Outcome{outcome()}, remainders: []*accountcleanup.Outcome{outcome()}}}
+	accounts := newFakeAccounts()
+	engine, _ := convergeEngine(accounts, compute, storage)
+
+	_, err := engine.Teardown(context.Background(), request())
+	if !errors.Is(err, ErrResourcesStuck) || accounts.deleted {
+		t.Fatalf("an unreadable inventory must never read as an empty account: err=%v deleted=%v", err, accounts.deleted)
+	}
+}
+
+// The loop keeps the account while the storage owner reports a volume still
+// recorded attached. That the owner refuses such a volume is the owner's own
+// invariant test in accountcleanup, against its real cleanup.
+func TestConvergeKeepsTheAccountWhileAnAttachedVolumeRemains(t *testing.T) {
+	compute := &scriptedCompute{scriptedStage{cleanups: []*accountcleanup.Outcome{outcome()}, remainders: []*accountcleanup.Outcome{outcome()}}}
+	storage := &scriptedStorage{scriptedStage{
+		cleanups:   []*accountcleanup.Outcome{outcome(blockedHolderVolume)},
+		remainders: []*accountcleanup.Outcome{outcome(item(accountcleanup.KindVolume, "vol-1", "in-use", accountcleanup.DispositionResidue, accountcleanup.ReasonRecordRemains))},
+	}}
+	for _, force := range []bool{false, true} {
+		accounts := newFakeAccounts()
+		engine, _ := convergeEngine(accounts, compute, storage)
+		req := request()
+		req.Force = force
+		storage.cleanupN, storage.remainN = 0, 0
+
+		_, err := engine.Teardown(context.Background(), req)
+		if !errors.Is(err, ErrResourcesStuck) || accounts.deleted {
+			t.Fatalf("force=%v: a volume still recorded attached must keep the account, err=%v deleted=%v", force, err, accounts.deleted)
+		}
+	}
+}
+
+func TestConvergeMissingCleanerFailsClosed(t *testing.T) {
+	engine := quietEngine(newFakeAccounts())
+	engine.Storage = nil
+	if _, err := engine.Teardown(context.Background(), request()); !errors.Is(err, errComputeCleanupMissing) {
+		t.Fatalf("compute without storage must refuse to start, got %v", err)
+	}
+}
+
+// cleanStages are compute and storage cleanups of an account that holds nothing.
+func cleanStages() (*scriptedCompute, *scriptedStorage) {
+	return &scriptedCompute{scriptedStage{cleanups: []*accountcleanup.Outcome{outcome()}, remainders: []*accountcleanup.Outcome{outcome()}}},
+		&scriptedStorage{scriptedStage{cleanups: []*accountcleanup.Outcome{outcome()}, remainders: []*accountcleanup.Outcome{outcome()}}}
+}

spinifex/operator/accountlifecycle/cutover_test.go

@@ -0,0 +1,169 @@
+package accountlifecycle
+
+//test:in-package — these drive the engine's unexported stage wiring with
+// scripted compute and storage owners, so each can count what it was asked.
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountcleanup"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// failingRemainder answers the compute remainder read until its nth call.
+type failingRemainder struct {
+	ComputeStage
+
+	failOn int
+	calls  int
+}
+
+func (f *failingRemainder) ComputeRemaining(ctx context.Context, accountID string) (*accountcleanup.Outcome, error) {
+	f.calls++
+	if f.calls == f.failOn {
+		return nil, errors.New("owner unreachable")
+	}
+	return f.ComputeStage.ComputeRemaining(ctx, accountID)
+}
+
+func stuckIDs(result *Result) []string {
+	var ids []string
+	for _, stage := range result.Stages {
+		for _, s := range stage.Stuck {
+			ids = append(ids, s.Resource.ID)
+		}
+	}
+	return ids
+}
+
+// An account deletion that was never accepted must reach no terminate and no
+// volume delete: the quiesce refuses first, whatever Force says.
+func TestS7_NoTerminateOrVolumeDeleteWithoutAnAcceptedDeletion(t *testing.T) {
+	for _, force := range []bool{false, true} {
+		accounts := newFakeAccounts()
+		accounts.statusErr = errors.New("account is owned by a newer fence")
+		// The record already reads as ours, so only the quiesce itself stands
+		// between this holder and the terminate.
+		accounts.account.Status = AccountStatusTerminating
+		accounts.account.TerminationOp = &TerminationOp{OperationID: "op-1", Fence: 1}
+		engine := quietEngine(accounts)
+		compute, storage := cleanStages()
+		engine.Compute, engine.Storage = compute, storage
+
+		req := request()
+		req.OperationID, req.Fence, req.Force = "op-1", 1, force
+		_, err := engine.Teardown(context.Background(), req)
+
+		require.Error(t, err)
+		assert.Zero(t, compute.cleanupN, "force=%v: no terminate", force)
+		assert.Zero(t, storage.cleanupN, "force=%v: no volume delete", force)
+		assert.False(t, accounts.deleted)
+	}
+}
+
+// With the compute and storage owners missing the teardown refuses before it
+// quiesces anything, so a mis-wired gateway cannot half-delete an account.
+func TestS7_MissingComputeOwnerRefusesBeforeAnyStageRuns(t *testing.T) {
+	accounts := newFakeAccounts()
+	engine := quietEngine(accounts)
+	engine.Compute = nil
+
+	_, err := engine.Teardown(context.Background(), request())
+
+	require.ErrorIs(t, err, errComputeCleanupMissing)
+	assert.Empty(t, accounts.quiesces)
+	assert.False(t, accounts.deleted)
+}
+
+// A resource that appears after the loop ended, while the later stages ran,
+// still keeps the account, and it is named.
+func TestFinalRereadKeepsTheAccountWhenAnInstanceArrivesLate(t *testing.T) {
+	accounts := newFakeAccounts()
+	engine := quietEngine(accounts)
+	engine.Compute = &scriptedCompute{scriptedStage{
+		cleanups:   []*accountcleanup.Outcome{outcome()},
+		remainders: []*accountcleanup.Outcome{outcome(), outcome(item(accountcleanup.KindInstance, "i-late", "running", accountcleanup.DispositionResidue, accountcleanup.ReasonRecordRemains))},
+	}}
+
+	result, err := engine.Teardown(context.Background(), request())
+
+	require.ErrorIs(t, err, ErrResourcesStuck)
+	assert.False(t, accounts.deleted)
+	assert.Contains(t, stuckIDs(result), "i-late")
+}
+
+// A final read that cannot be answered proves nothing, so the account stays.
+func TestFinalRereadKeepsTheAccountWhenTheReadFails(t *testing.T) {
+	accounts := newFakeAccounts()
+	engine := quietEngine(accounts)
+	compute, _ := cleanStages()
+	engine.Compute = &failingRemainder{ComputeStage: compute, failOn: 2}
+
+	result, err := engine.Teardown(context.Background(), request())
+
+	require.ErrorIs(t, err, ErrResourcesStuck)
+	assert.False(t, accounts.deleted)
+	assert.Contains(t, stuckIDs(result), StageAttachments.String())
+}
+
+// An ECS, EKS or RDS owner that cannot finish deleting may be running on these
+// instances, so none is terminated under it.
+func TestStuckComputeOwnerLeavesTheInstancesAlone(t *testing.T) {
+	accounts := newFakeAccounts()
+	cluster := &fakeReaper{
+		kind: "ecs-cluster", stage: StageCompute, blockedUntilForce: true,
+		resources: []Resource{{Kind: "ecs-cluster", ID: "web"}},
+	}
+	engine := quietEngine(accounts, cluster)
+	compute, _ := cleanStages()
+	engine.Compute = compute
+
+	result, err := engine.Teardown(context.Background(), request())
+
+	require.ErrorIs(t, err, ErrResourcesStuck)
+	assert.Zero(t, compute.cleanupN, "no instance is terminated while the owner holds resources")
+	assert.False(t, accounts.deleted)
+	assert.Contains(t, stuckIDs(result), "web")
+}
+
+// The dry run reports what the actual deletion would destroy.
+func TestDryRunListsInstancesSpotRequestsAndVolumes(t *testing.T) {
+	accounts := newFakeAccounts()
+	engine := quietEngine(accounts)
+	engine.Compute = &scriptedCompute{scriptedStage{
+		cleanups: []*accountcleanup.Outcome{outcome()},
+		remainders: []*accountcleanup.Outcome{outcome(
+			item(accountcleanup.KindInstance, "i-1", "running", accountcleanup.DispositionPending, ""),
+			item(accountcleanup.KindSpotRequest, "sir-1", "open", accountcleanup.DispositionPending, ""),
+		)},
+	}}
+	engine.Storage = &scriptedStorage{scriptedStage{
+		cleanups:   []*accountcleanup.Outcome{outcome()},
+		remainders: []*accountcleanup.Outcome{outcome(item(accountcleanup.KindVolume, "vol-1", "available", accountcleanup.DispositionPending, ""))},
+	}}
+
+	result, err := engine.DryRun(context.Background(), testAccountID)
+
+	require.NoError(t, err)
+	var ids []string
+	for _, stage := range result.Stages {
+		for _, r := range stage.Deleted {
+			ids = append(ids, r.ID)
+		}
+	}
+	assert.ElementsMatch(t, []string{"i-1", "sir-1", "vol-1"}, ids)
+}
+
+// A dry run that cannot read the owners fails rather than listing less.
+func TestDryRunFailsWhenTheComputeOwnerCannotBeRead(t *testing.T) {
+	engine := quietEngine(newFakeAccounts())
+	compute, _ := cleanStages()
+	engine.Compute = &failingRemainder{ComputeStage: compute, failOn: 1}
+
+	_, err := engine.DryRun(context.Background(), testAccountID)
+
+	require.Error(t, err)
+}

spinifex/operator/accountlifecycle/ec2_platform_inventory_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3"
 	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountcleanup"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountinventory"
 	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
@@ -209,6 +210,21 @@ func TestNoLegacyReaperOwnsAnL09Kind(t *testing.T) {
 	}
 }
 
+// Instances, spot requests and volumes are the EC2 compute and storage
+// owners', so no legacy reaper may list or delete them either.
+func TestNoLegacyReaperOwnsAnL03L05Kind(t *testing.T) {
+	var reapers []Reaper
+	reapers = append(reapers, ECSReapers(nil)...)
+	reapers = append(reapers, EKSReapers(nil)...)
+	reapers = append(reapers, RDSReapers(nil)...)
+	reapers = append(reapers, ELBv2Reapers(nil)...)
+	reapers = append(reapers, EC2ExtraReapers(nil)...)
+	reapers = append(reapers, EC2Reapers(nil, 1, nil)...)
+	for _, reaper := range reapers {
+		assert.NotContains(t, []string{accountcleanup.KindInstance, accountcleanup.KindSpotRequest, accountcleanup.KindVolume}, reaper.Kind())
+	}
+}
+
 // A caller deadline shorter than the adapter's own timeout ends the request
 // at that deadline, and a context already done sends nothing.
 func TestEC2PlatformInventoryClientHonoursTheCallersDeadline(t *testing.T) {

spinifex/operator/accountlifecycle/engine.go

@@ -7,9 +7,11 @@ import (
 	"log/slog"
 	"slices"
 	"sort"
+	"strings"
 	"time"
 
 	operatorv1 "github.com/mulgadc/spinifex/contracts/operator/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountcleanup"
 )
 
 // AccountStatusTerminating mirrors iamdomain.AccountStatusTerminating.
@@ -41,6 +43,21 @@ type Engine struct {
 	// OnExercised receives what a non-finalizing run did after quiescence.
 	OnExercised func(ctx context.Context, result operatorv1.ExerciseResult)
 
+	// Compute and Storage are EC2's instance, spot request and volume cleanup.
+	// When set, Teardown repeats them within Converge's bounds in place of the
+	// legacy attachments and storage stages.
+	Compute ComputeStage
+	Storage StorageStage
+
+	// Converge bounds the repeat loop. The zero value uses DefaultConverge.
+	Converge Converge
+
+	// OnPass reports each finished pass of the loop so the caller can persist progress.
+	OnPass func(ctx context.Context, pass int)
+
+	// wait replaces the backoff delay in tests.
+	wait func(ctx context.Context, d time.Duration) error
+
 	// ec2Inventory is EC2's read-only platform inventory: Inventory lists the
 	// L09 kinds from it, and the quiescence evaluation and final re-read use it.
 	ec2Inventory ec2PlatformInventory
@@ -104,11 +121,44 @@ func (e *Engine) Inventory(ctx context.Context, accountID string) (*Result, erro
 			}
 			stageResult.Deleted = append(stageResult.Deleted, listed...)
 		}
+		if stage == StageAttachments || stage == StageStorage {
+			listed, err := e.listComputeStorage(ctx, stage, accountID)
+			if err != nil {
+				return nil, err
+			}
+			stageResult.Deleted = append(stageResult.Deleted, listed...)
+		}
 		result.Stages = append(result.Stages, stageResult)
 	}
 	return result, nil
 }
 
+// listComputeStorage reads what the EC2 compute and storage owners still hold.
+// An unanswered or incomplete read is an error, never a shorter list.
+func (e *Engine) listComputeStorage(ctx context.Context, stage Stage, accountID string) ([]Resource, error) {
+	if e.Compute == nil || e.Storage == nil {
+		return nil, errComputeCleanupMissing
+	}
+	var out *accountcleanup.Outcome
+	var err error
+	if stage == StageAttachments {
+		out, err = e.Compute.ComputeRemaining(ctx, accountID)
+	} else {
+		out, err = e.Storage.StorageRemaining(ctx, accountID)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("inventory %s: %w", stage, err)
+	}
+	if len(out.IncompleteReasons) > 0 {
+		return nil, fmt.Errorf("inventory %s incomplete: %s", stage, strings.Join(out.IncompleteReasons, ","))
+	}
+	var found []Resource
+	for _, it := range out.Items {
+		found = append(found, Resource{Kind: it.Kind, ID: it.ResourceID, Detail: it.State})
+	}
+	return found, nil
+}
+
 // DryRun is the explicit operator dry run: exactly what Inventory returns.
 func (e *Engine) DryRun(ctx context.Context, accountID string) (*Result, error) {
 	return e.Inventory(ctx, accountID)
@@ -130,6 +180,9 @@ func (e *Engine) Teardown(ctx context.Context, req Request) (*Result, error) {
 	if e.Cleanup == nil || e.ec2Inventory == nil {
 		return nil, errPlatformCleanupMissing
 	}
+	if e.Compute == nil || e.Storage == nil {
+		return nil, errComputeCleanupMissing
+	}
 
 	generation, err := e.Accounts.QuiesceAccount(req.AccountID, req.OperationID, req.Fence)
 	if err != nil {
@@ -148,6 +201,9 @@ func (e *Engine) Teardown(ctx context.Context, req Request) (*Result, error) {
 	}
 	var platform *platformRun
 	for _, stage := range Stages() {
+		if stage == StageAttachments || stage == StageStorage {
+			continue
+		}
 		if stage == StagePlatform {
 			var err error
 			if platform, err = e.cleanPlatform(ctx, req, generation); err != nil {
@@ -165,8 +221,21 @@ func (e *Engine) Teardown(ctx context.Context, req Request) (*Result, error) {
 		if e.OnStage != nil {
 			e.OnStage(ctx, stageResult)
 		}
+		if stage == StageCompute {
+			if len(stageResult.Stuck) > 0 {
+				// An ECS, EKS or RDS owner that still holds resources may be
+				// running on these instances, so none is terminated under it.
+				e.logger().WarnContext(ctx, "Account teardown: compute owners still hold resources, instances left alone",
+					"accountID", account.AccountID, "stuck", len(stageResult.Stuck))
+				continue
+			}
+			if err := e.convergeCompute(ctx, req, result); err != nil {
+				return result, err
+			}
+		}
 	}
 	e.rereadPlatform(ctx, req, platform, result)
+	e.rereadComputeStorage(ctx, req.AccountID, result)
 
 	if stuck := result.StuckCount(); stuck > 0 {
 		// The account stays TERMINATING on purpose. Removing the record now

spinifex/operator/accountlifecycle/engine_test.go

@@ -130,6 +130,7 @@ func quietEngine(accounts AccountStore, reapers ...Reaper) *Engine {
 	// stage need not stand up the EC2 owners.
 	engine.Cleanup = &stubCleaner{result: cleanResult()}
 	engine.ec2Inventory = &stubInventory{reply: *completeInventory()}
+	engine.Compute, engine.Storage = cleanStages()
 	return engine
 }
 

spinifex/operator/accountlifecycle/exercise.go

@@ -157,6 +157,10 @@ type InstanceReferenceReader struct {
 
 var _ accountcleanup.InstanceReferenceReader = (*InstanceReferenceReader)(nil)
 
+// terminalInstanceStates are the states an instance no longer holds anything
+// in. Anything else is still occupying a volume, an address and a host.
+var terminalInstanceStates = map[string]bool{"terminated": true, "shutting-down": true}
+
 // NewInstanceReferenceReader builds the reader over the configured node set.
 func NewInstanceReferenceReader(nc *nats.Conn, nodeIDs []string) *InstanceReferenceReader {
 	return &InstanceReferenceReader{nc: nc, nodeIDs: nodeIDs, budget: ec2PlatformInventoryBudget}
@@ -165,7 +169,7 @@ func NewInstanceReferenceReader(nc *nats.Conn, nodeIDs []string) *InstanceRefere
 func (r *InstanceReferenceReader) AccountInstanceReferences(ctx context.Context, accountID string) (*accountcleanup.References, error) {
 	ctx, cancel := context.WithTimeout(ctx, r.budget)
 	defer cancel()
-	out, err := ec2instanceapi.DescribeAccountInstancesComplete(ctx, r.nc, r.nodeIDs, accountID)
+	out, err := ec2instanceapi.DescribeAccountInstancesStrict(ctx, r.nc, r.nodeIDs, accountID)
 	if err != nil {
 		return nil, err
 	}

spinifex/operator/accountlifecycle/quiescence_test.go

@@ -241,9 +241,9 @@ func TestTeardownReportsQuiescenceAfterTheQuiesceAndBeforeTheFirstStage(t *testi
 	assert.True(t, accounts.deleted)
 }
 
-// A fence the read-back does not confirm must not spend an inventory call, and a
+// A fence the read-back does not confirm must not spend an inventory call, and
-// failed read-back must not stop the teardown: this step records and gates nothing.
+// the convergence loop must not run a pass behind a fence it cannot read.
-func TestTeardownRecordsAFailedReadbackAndStillRunsEveryStage(t *testing.T) {
+func TestTeardownRecordsAFailedReadbackAndRunsNoPassBehindIt(t *testing.T) {
 	accounts := newFakeAccounts()
 	engine := quietEngine(accounts)
 	inventory := &stubInventory{reply: *completeInventory()}
@@ -257,11 +257,11 @@ func TestTeardownRecordsAFailedReadbackAndStillRunsEveryStage(t *testing.T) {
 	req.OperationID, req.Fence = "op-1", 1
 	_, err := engine.Teardown(context.Background(), req)
 
-	require.NoError(t, err)
+	require.Error(t, err)
 	require.Len(t, got.Unresolved, 1)
 	assert.Equal(t, operatorv1.ReasonIAMReadbackFailed, got.Unresolved[0].Reason)
-	assert.Equal(t, 1, inventory.calls, "only the final re-read: no evaluation reads inventory behind an unconfirmed fence")
+	assert.Equal(t, 0, inventory.calls, "no evaluation reads inventory behind an unconfirmed fence")
-	assert.True(t, accounts.deleted)
+	assert.False(t, accounts.deleted)
 }
 
 // readbackFails serves the pre-quiesce read and fails the one after it.

spinifex/operator/accountlifecycle/realcleanup_test.go

@@ -0,0 +1,104 @@
+package accountlifecycle
+
+import (
+	"context"
+	"errors"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountcleanup"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountinventory"
+	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
+	ec2spotinstance "github.com/mulgadc/spinifex/spinifex/domains/ec2/spotinstance"
+)
+
+// Stores that answer as an account does once its instance record has expired:
+// no instance, no outstanding teardown, no spot request, no tag residue.
+type emptyComputeStores struct{}
+
+func (emptyComputeStores) ScanAccount(context.Context, string) (*ec2spotinstance.AccountScan, error) {
+	return &ec2spotinstance.AccountScan{}, nil
+}
+
+func (emptyComputeStores) CancelSpotInstanceRequests(context.Context, *ec2.CancelSpotInstanceRequestsInput, string) (*ec2.CancelSpotInstanceRequestsOutput, error) {
+	return &ec2.CancelSpotInstanceRequestsOutput{}, nil
+}
+
+func (emptyComputeStores) AccountInstances(context.Context, string) (*ec2.DescribeInstancesOutput, error) {
+	return &ec2.DescribeInstancesOutput{}, nil
+}
+
+func (emptyComputeStores) AccountTeardownOutstanding(string) ([]ec2instance.OutstandingInstance, error) {
+	return nil, nil
+}
+
+func (emptyComputeStores) AccountManagedInstances(context.Context, string) (map[string]bool, error) {
+	return map[string]bool{}, nil
+}
+
+func (emptyComputeStores) ScanAccountResourceIDs(context.Context, string, ...string) ([]string, error) {
+	return nil, nil
+}
+
+func (emptyComputeStores) DeleteAllTags(context.Context, string, string) error { return nil }
+
+type heldVolumes struct{ items []accountinventory.VolumeItem }
+
+func (h heldVolumes) VolumeInventory(context.Context, string) ([]accountinventory.VolumeItem, error) {
+	return h.items, nil
+}
+
+type countingEC2 struct{ terminates, deletes atomic.Int32 }
+
+func (c *countingEC2) TerminateInstance(context.Context, string, string) error {
+	c.terminates.Add(1)
+	return nil
+}
+
+func (c *countingEC2) DeleteVolume(context.Context, string, string) error {
+	c.deletes.Add(1)
+	return nil
+}
+
+// ADR-0003:S7. The instance's terminated record has expired and the instance is
+// absent from the inventory, but its volume is still recorded in-use and
+// attached. With the real compute and storage cleanup bound to these stores,
+// nothing is deleted whatever Force says, the account is not removed, and the
+// volume is named.
+func TestS7_ExpiredTerminatedRecordAndAttachedVolumeDeleteNothingAndKeepTheAccount(t *testing.T) {
+	for _, force := range []bool{false, true} {
+		stores, ec2ops := emptyComputeStores{}, &countingEC2{}
+		accounts := newFakeAccounts()
+		engine := quietEngine(accounts)
+		engine.Compute = accountcleanup.NewCompute(stores, stores, ec2ops, stores, stores)
+		engine.Storage = accountcleanup.NewStorage(
+			heldVolumes{items: []accountinventory.VolumeItem{{VolumeID: "vol-held", State: "in-use", AttachedInstance: "i-gone", DeleteOnTermination: true}}},
+			ec2ops, stores, stores, stores, stores)
+		engine.wait = func(ctx context.Context, _ time.Duration) error { return ctx.Err() }
+
+		req := request()
+		req.Force = force
+		result, err := engine.Teardown(context.Background(), req)
+
+		if !errors.Is(err, ErrResourcesStuck) {
+			t.Fatalf("force=%v: a held volume must end the teardown stuck, got %v", force, err)
+		}
+		if ec2ops.deletes.Load() != 0 || ec2ops.terminates.Load() != 0 {
+			t.Fatalf("force=%v: nothing may be deleted or terminated, deletes=%d terminates=%d", force, ec2ops.deletes.Load(), ec2ops.terminates.Load())
+		}
+		if accounts.deleted {
+			t.Fatalf("force=%v: the account must not be removed", force)
+		}
+		named := false
+		for _, stage := range result.Stages {
+			for _, s := range stage.Stuck {
+				named = named || s.Resource.ID == "vol-held"
+			}
+		}
+		if !named {
+			t.Fatalf("force=%v: the stuck volume must be named in the result", force)
+		}
+	}
+}

spinifex/operator/accountlifecycle/reapers_ec2.go

@@ -2,11 +2,8 @@ package accountlifecycle
 
 import (
 	"context"
-	"fmt"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"slices"
 	"strings"
-	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -14,10 +11,8 @@ import (
 	ec2eipapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/eip"
 	ec2igwapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/igw"
 	ec2imageapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/image"
-	ec2instanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/instance"
 	ec2routetableapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/routetable"
 	ec2snapshotapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/snapshot"
-	ec2volumeapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/volume"
 	ec2vpcapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/vpc"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/nats-io/nats.go"
@@ -30,13 +25,10 @@ import (
 // accounting correct without teardown having to reimplement any of it.
 func EC2Reapers(nc *nats.Conn, expectedNodes int, nodeIDs []string) []Reaper {
 	return []Reaper{
-		&instanceReaper{nc: nc, nodeIDs: nodeIDs},
-
 		// Snapshots before volumes: a volume with a live snapshot refuses to
 		// delete, because snapshot-backed clones read from its chunk prefix.
 		&snapshotReaper{nc: nc},
 		&imageReaper{nc: nc},
-		&volumeReaper{nc: nc, nodeIDs: nodeIDs},
 
 		// Reverse of creation order. The VPC cannot go until everything
 		// inside it has, and the gateway enforces that.
@@ -55,107 +47,6 @@ func EC2Reapers(nc *nats.Conn, expectedNodes int, nodeIDs []string) []Reaper {
 	}
 }
 
-// terminalInstanceStates are the states an instance no longer holds anything
-// in. Anything else is still occupying a volume, an address and a host.
-var terminalInstanceStates = map[string]bool{"terminated": true, "shutting-down": true}
-
-type instanceReaper struct {
-	nc *nats.Conn
-	// nodeIDs is the configured node set; all must answer before an empty
-	// listing counts as drained.
-	nodeIDs []string
-
-	// describeOpts tunes the inventory fan-out; production leaves it empty.
-	describeOpts []ec2instanceapi.DescribeOption
-}
-
-func (r *instanceReaper) Kind() string { return "instance" }
-func (r *instanceReaper) Stage() Stage { return StageCompute }
-
-func (r *instanceReaper) List(ctx context.Context, accountID string) ([]Resource, error) {
-	// Complete or nothing: a partial answer would read as "no instances left"
-	// and let storage deletion start while a guest still holds a volume.
-	out, err := ec2instanceapi.DescribeAccountInstancesComplete(ctx, r.nc, r.nodeIDs, accountID, r.describeOpts...)
-	if err != nil {
-		return nil, err
-	}
-
-	var found []Resource
-	for _, reservation := range out.Reservations {
-		for _, instance := range reservation.Instances {
-			if instance == nil || instance.InstanceId == nil {
-				continue
-			}
-			state := ""
-			if instance.State != nil && instance.State.Name != nil {
-				state = *instance.State.Name
-			}
-			if terminalInstanceStates[state] {
-				continue
-			}
-			found = append(found, Resource{Kind: r.Kind(), ID: *instance.InstanceId, Detail: state})
-		}
-	}
-	return found, nil
-}
-
-func (r *instanceReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error {
-	_, err := ec2instanceapi.TerminateInstances(ctx, &ec2.TerminateInstancesInput{
-		InstanceIds: []*string{aws.String(resource.ID)},
-	}, r.nc, accountID)
-	return ignoreAlreadyGone(err)
-}
-
-type volumeReaper struct {
-	nc *nats.Conn
-	// nodeIDs is the configured node set; every one must answer, by identity,
-	// before a volume's holder can be proven gone or the delete guard passes.
-	nodeIDs []string
-}
-
-func (r *volumeReaper) Kind() string { return "volume" }
-func (r *volumeReaper) Stage() Stage { return StageStorage }
-
-func (r *volumeReaper) List(ctx context.Context, accountID string) ([]Resource, error) {
-	out, err := ec2volumeapi.DescribeVolumes(ctx, &ec2.DescribeVolumesInput{}, r.nc, accountID)
-	if err != nil {
-		return nil, err
-	}
-
-	var found []Resource
-	for _, volume := range out.Volumes {
-		if volume == nil || volume.VolumeId == nil {
-			continue
-		}
-		resource := Resource{Kind: r.Kind(), ID: *volume.VolumeId}
-		// The attachment is the reason a volume will not go, so carry it: it
-		// is the difference between "stuck" and "stuck on i-abc".
-		for _, attachment := range volume.Attachments {
-			if attachment != nil && attachment.InstanceId != nil {
-				resource.Detail = "attached to " + *attachment.InstanceId
-				break
-			}
-		}
-		found = append(found, resource)
-	}
-	return found, nil
-}
-
-func (r *volumeReaper) Delete(ctx context.Context, accountID string, resource Resource, force bool) error {
-	_, err := ec2volumeapi.DeleteVolume(ctx, &ec2.DeleteVolumeInput{
-		VolumeId: aws.String(resource.ID),
-	}, r.nc, r.nodeIDs, accountID)
-	if isAlreadyGone(err) {
-		return nil
-	}
-	if !force {
-		return err
-	}
-	// Force path: the ordinary delete refuses an attached volume, and a holder
-	// that is gone but still on the record would otherwise strand it forever.
-	return forceDeleteVolume(ctx, r.nc, r.nodeIDs, accountID, resource.ID, err)
-}
-
 type snapshotReaper struct{ nc *nats.Conn }
 
 func (r *snapshotReaper) Kind() string { return "snapshot" }
@@ -599,131 +490,3 @@ func ignoreAlreadyGone(err error) error {
 	}
 	return err
 }
-
-// forceDetachSubject clears a volume's attachment in the control plane only.
-// It is answered by any node, unlike the ordinary detach, which routes to the
-// host running the instance and so cannot help when that host is the problem.
-const forceDetachSubject = "ec2.ForceDetachVolume"
-
-// forceDetachTimeout bounds the metadata-only detach. It touches one document
-// and needs nothing from the guest, so a slow answer means the cluster is
-// unwell rather than that the work is large.
-const forceDetachTimeout = 30 * time.Second
-
-// forceDeleteVolume clears an attachment the ordinary delete refuses to
-// override, then deletes, but only once every instance the volume names is
-// proven absent. Anything short of proof leaves the volume and its record alone.
-func forceDeleteVolume(ctx context.Context, nc *nats.Conn, nodeIDs []string, accountID, volumeID string, refusal error) error {
-	holders, err := volumeHolders(ctx, nc, accountID, volumeID)
-	if err != nil {
-		if isAlreadyGone(err) {
-			return nil
-		}
-		return fmt.Errorf("volume %s retained: cannot read its attachment: %w", volumeID, err)
-	}
-	if len(holders) == 0 {
-		// No holder on record, so the refusal is not an attachment and
-		// clearing the record would change nothing but the evidence.
-		return refusal
-	}
-	for _, instanceID := range holders {
-		if err := proveInstanceAbsent(ctx, nc, nodeIDs, accountID, instanceID); err != nil {
-			return fmt.Errorf("volume %s retained: %w", volumeID, err)
-		}
-	}
-
-	_, err = natsmsg.NATSRequest[ec2.VolumeAttachment](ctx, nc, forceDetachSubject,
-		&ec2.DetachVolumeInput{VolumeId: aws.String(volumeID), Force: aws.Bool(true)},
-		forceDetachTimeout, accountID)
-	if err != nil && !isAlreadyGone(err) {
-		return fmt.Errorf("force detach %s: %w", volumeID, err)
-	}
-
-	// The ordinary delete re-checks that the record is unattached and that no
-	// live instance maps the volume, so a re-attach in between still refuses.
-	_, err = ec2volumeapi.DeleteVolume(ctx, &ec2.DeleteVolumeInput{
-		VolumeId: aws.String(volumeID),
-	}, nc, nodeIDs, accountID)
-	return ignoreAlreadyGone(err)
-}
-
-// volumeHolders returns the instances the volume's own record says hold it,
-// read fresh rather than from the listing that queued the delete.
-func volumeHolders(ctx context.Context, nc *nats.Conn, accountID, volumeID string) ([]string, error) {
-	out, err := ec2volumeapi.DescribeVolumes(ctx, &ec2.DescribeVolumesInput{
-		VolumeIds: []*string{aws.String(volumeID)},
-	}, nc, accountID)
-	if err != nil {
-		return nil, err
-	}
-	var holders []string
-	for _, volume := range out.Volumes {
-		if volume == nil || aws.StringValue(volume.VolumeId) != volumeID {
-			continue
-		}
-		for _, attachment := range volume.Attachments {
-			if attachment != nil && aws.StringValue(attachment.InstanceId) != "" {
-				holders = append(holders, aws.StringValue(attachment.InstanceId))
-			}
-		}
-	}
-	return holders, nil
-}
-
-// proveInstanceAbsent accepts only a definitive InvalidInstanceID.NotFound or
-// a terminated copy from a describe naming the instance, then confirms both on
-// a listing every configured node answered by identity. Anything else is not proof.
-func proveInstanceAbsent(ctx context.Context, nc *nats.Conn, nodeIDs []string, accountID, instanceID string) error {
-	if len(nodeIDs) == 0 {
-		return fmt.Errorf("cannot prove holding instance %s is gone: the configured node set is unknown", instanceID)
-	}
-	named, err := ec2instanceapi.DescribeInstancesChecked(ctx, &ec2.DescribeInstancesInput{
-		InstanceIds: []*string{aws.String(instanceID)},
-	}, nc, 0, nodeIDs, accountID)
-	if err != nil {
-		if code, ok := awserrors.ResolveErrorCode(err); !ok || code != awserrors.ErrorInvalidInstanceIDNotFound {
-			return fmt.Errorf("cannot prove holding instance %s is gone: %w", instanceID, err)
-		}
-	} else {
-		found, err := requireTerminated(named.Reservations, instanceID)
-		if err != nil {
-			return err
-		}
-		if !found {
-			return fmt.Errorf("cannot prove holding instance %s is gone: the describe naming it returned neither the instance nor NotFound", instanceID)
-		}
-	}
-
-	// The named describe stops collecting once the id turns up, and a lone
-	// node's 4xx can surface as NotFound, so neither answer is complete alone.
-	listing, err := ec2instanceapi.DescribeAccountInstancesComplete(ctx, nc, nodeIDs, accountID)
-	if err != nil {
-		return fmt.Errorf("cannot prove holding instance %s is gone: %w", instanceID, err)
-	}
-	_, err = requireTerminated(listing.Reservations, instanceID)
-	return err
-}
-
-// requireTerminated reports whether instanceID appears in reservations, and
-// fails if any copy of it is in a state other than terminated.
-func requireTerminated(reservations []*ec2.Reservation, instanceID string) (found bool, err error) {
-	for _, reservation := range reservations {
-		if reservation == nil {
-			continue
-		}
-		for _, instance := range reservation.Instances {
-			if instance == nil || aws.StringValue(instance.InstanceId) != instanceID {
-				continue
-			}
-			found = true
-			state := "in an unknown state"
-			if instance.State != nil && aws.StringValue(instance.State.Name) != "" {
-				state = aws.StringValue(instance.State.Name)
-			}
-			if state != ec2.InstanceStateNameTerminated {
-				return true, fmt.Errorf("holding instance %s is still %s", instanceID, state)
-			}
-		}
-	}
-	return found, nil
-}

spinifex/operator/accountlifecycle/reapers_ec2_extra.go

@@ -7,66 +7,21 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	ec2eigw "github.com/mulgadc/spinifex/spinifex/domains/ec2/eigw"
 	ec2natgw "github.com/mulgadc/spinifex/spinifex/domains/ec2/natgw"
-	ec2spotinstance "github.com/mulgadc/spinifex/spinifex/domains/ec2/spotinstance"
 	"github.com/nats-io/nats.go"
 )
 
 // EC2ExtraReapers returns the EC2 reapers that hang off the VPC rather than off
 // an instance.
 //
-// Spot requests come first and in the compute stage: a live request can launch
+// Spot requests are not here: the EC2 compute cleanup cancels them before it
-// a replacement after the instance reaper has drained, so leaving one running
+// terminates anything, so none can launch a replacement.
-// makes teardown race its own fulfilment rather than merely leak.
 func EC2ExtraReapers(nc *nats.Conn) []Reaper {
 	return []Reaper{
-		&spotRequestReaper{svc: ec2spotinstance.NewNATSSpotInstanceService(nc)},
 		&natGatewayReaper{svc: ec2natgw.NewNATSNatGatewayService(nc)},
 		&egressOnlyIGWReaper{svc: ec2eigw.NewNATSEgressOnlyIGWService(nc)},
 	}
 }
 
-// terminalSpotStates are the states a request can no longer launch from.
-// Anything else may still produce an instance.
-var terminalSpotStates = map[string]bool{"cancelled": true, "closed": true, "failed": true}
-
-type spotRequestReaper struct {
-	svc ec2spotinstance.SpotInstanceService
-}
-
-func (r *spotRequestReaper) Kind() string { return "spot-request" }
-func (r *spotRequestReaper) Stage() Stage { return StageCompute }
-
-func (r *spotRequestReaper) List(ctx context.Context, accountID string) ([]Resource, error) {
-	out, err := r.svc.DescribeSpotInstanceRequests(ctx, &ec2.DescribeSpotInstanceRequestsInput{}, accountID)
-	if err != nil {
-		return nil, err
-	}
-
-	var found []Resource
-	for _, request := range out.SpotInstanceRequests {
-		if request == nil || request.SpotInstanceRequestId == nil {
-			continue
-		}
-		state := aws.StringValue(request.State)
-		if terminalSpotStates[state] {
-			continue
-		}
-		found = append(found, Resource{
-			Kind:   r.Kind(),
-			ID:     aws.StringValue(request.SpotInstanceRequestId),
-			Detail: state,
-		})
-	}
-	return found, nil
-}
-
-func (r *spotRequestReaper) Delete(ctx context.Context, accountID string, resource Resource, _ bool) error {
-	_, err := r.svc.CancelSpotInstanceRequests(ctx, &ec2.CancelSpotInstanceRequestsInput{
-		SpotInstanceRequestIds: []*string{aws.String(resource.ID)},
-	}, accountID)
-	return ignoreAlreadyGone(err)
-}
-
 // deletedNatGatewayStates are the states a NAT gateway has already released its
 // address and its subnet placement in.
 var deletedNatGatewayStates = map[string]bool{"deleted": true}

spinifex/operator/accountlifecycle/reapers_ec2_extra_test.go

@@ -12,36 +12,11 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	ec2eigw "github.com/mulgadc/spinifex/spinifex/domains/ec2/eigw"
 	ec2natgw "github.com/mulgadc/spinifex/spinifex/domains/ec2/natgw"
-	ec2spotinstance "github.com/mulgadc/spinifex/spinifex/domains/ec2/spotinstance"
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-// The embedded interfaces are deliberately nil: a reaper that calls anything
-// beyond the two methods it is supposed to use panics rather than passing.
-type fakeSpot struct {
-	ec2spotinstance.SpotInstanceService
-
-	requests  []*ec2.SpotInstanceRequest
-	cancelled []string
-	err       error
-}
-
-func (f *fakeSpot) DescribeSpotInstanceRequests(_ context.Context, _ *ec2.DescribeSpotInstanceRequestsInput, _ string) (*ec2.DescribeSpotInstanceRequestsOutput, error) {
-	return &ec2.DescribeSpotInstanceRequestsOutput{SpotInstanceRequests: f.requests}, nil
-}
-
-func (f *fakeSpot) CancelSpotInstanceRequests(_ context.Context, in *ec2.CancelSpotInstanceRequestsInput, _ string) (*ec2.CancelSpotInstanceRequestsOutput, error) {
-	if f.err != nil {
-		return nil, f.err
-	}
-	for _, id := range in.SpotInstanceRequestIds {
-		f.cancelled = append(f.cancelled, aws.StringValue(id))
-	}
-	return &ec2.CancelSpotInstanceRequestsOutput{}, nil
-}
-
 type fakeNatGateways struct {
 	ec2natgw.NatGatewayService
 
@@ -82,42 +57,6 @@ func (f *fakeEgressGateways) DeleteEgressOnlyInternetGateway(_ context.Context,
 	return &ec2.DeleteEgressOnlyInternetGatewayOutput{}, nil
 }
 
-// A cancelled request cannot launch anything, so listing it would leave the
-// stage waiting on a row that never clears — spot requests are never removed
-// from the account, only moved to a terminal state.
-func TestSpotRequestReaperSkipsTerminalRequests(t *testing.T) {
-	svc := &fakeSpot{requests: []*ec2.SpotInstanceRequest{
-		{SpotInstanceRequestId: aws.String("sir-open"), State: aws.String("open")},
-		{SpotInstanceRequestId: aws.String("sir-active"), State: aws.String("active")},
-		{SpotInstanceRequestId: aws.String("sir-cancelled"), State: aws.String("cancelled")},
-		{SpotInstanceRequestId: aws.String("sir-closed"), State: aws.String("closed")},
-		{SpotInstanceRequestId: aws.String("sir-failed"), State: aws.String("failed")},
-	}}
-	reaper := &spotRequestReaper{svc: svc}
-
-	assert.Equal(t, StageCompute, reaper.Stage())
-
-	found, err := reaper.List(t.Context(), "000000000002")
-	require.NoError(t, err)
-	require.Len(t, found, 2)
-	assert.Equal(t, "sir-open", found[0].ID)
-	assert.Equal(t, "open", found[0].Detail)
-	assert.Equal(t, "sir-active", found[1].ID)
-
-	require.NoError(t, reaper.Delete(t.Context(), "000000000002", found[0], false))
-	assert.Equal(t, []string{"sir-open"}, svc.cancelled)
-}
-
-// A request cancelled by someone else between the listing and the delete is
-// the ordinary second-pass case, not a failure.
-func TestSpotRequestReaperTreatsAMissingRequestAsCancelled(t *testing.T) {
-	svc := &fakeSpot{err: errors.New("InvalidSpotInstanceRequestID.NotFound: no such request")}
-	reaper := &spotRequestReaper{svc: svc}
-
-	err := reaper.Delete(t.Context(), "000000000002", Resource{ID: "sir-gone"}, false)
-	assert.NoError(t, err)
-}
-
 // A deleting gateway still holds its address and its subnet, so it has to keep
 // listing until it is actually gone or the address reaper runs too early.
 func TestNatGatewayReaperKeepsDeletingGateways(t *testing.T) {

spinifex/operator/accountlifecycle/reapers_ec2_inventory_test.go

@@ -1,198 +0,0 @@
-package accountlifecycle
-
-//test:in-package — the instance reaper is unexported, and whether its listing
-// can be mistaken for an empty account is the safety property under test.
-
-import (
-	"context"
-	"encoding/json"
-	"testing"
-	"time"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/mulgadc/spinifex/internal/testkit"
-	ec2instanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/instance"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
-	"github.com/nats-io/nats.go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-var inventoryNodes = []string{"node-1", "node-2"}
-
-// testInventoryReaper builds the instance reaper for a two-node cluster, with
-// a short fan-out deadline so a silent node costs the test little.
-func testInventoryReaper(nc *nats.Conn) *instanceReaper {
-	return &instanceReaper{nc: nc, nodeIDs: inventoryNodes, describeOpts: []ec2instanceapi.DescribeOption{
-		ec2instanceapi.WithFanoutTimeout(300 * time.Millisecond),
-	}}
-}
-
-// answerAsNode replies on subject the way a daemon does, carrying the node ID
-// header the fan-out attributes frames by.
-func answerAsNode(t *testing.T, nc *nats.Conn, subject, nodeID string, payload []byte) {
-	t.Helper()
-	sub, err := nc.Subscribe(subject, func(msg *nats.Msg) {
-		reply := nats.NewMsg(msg.Reply)
-		reply.Data = payload
-		if nodeID != "" {
-			reply.Header.Set(natsmsg.NodeIDHeader, nodeID)
-		}
-		_ = msg.RespondMsg(reply)
-	})
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = sub.Unsubscribe() })
-}
-
-func describeOutput(t *testing.T, instanceIDs ...string) []byte {
-	t.Helper()
-	out := ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{}}
-	if len(instanceIDs) > 0 {
-		reservation := &ec2.Reservation{ReservationId: aws.String("r-" + instanceIDs[0])}
-		for _, id := range instanceIDs {
-			reservation.Instances = append(reservation.Instances, &ec2.Instance{
-				InstanceId: aws.String(id),
-				State:      &ec2.InstanceState{Name: aws.String("running")},
-			})
-		}
-		out.Reservations = append(out.Reservations, reservation)
-	}
-	payload, err := json.Marshal(out)
-	require.NoError(t, err)
-	return payload
-}
-
-// emptyInstanceBuckets answers the stopped and terminated bucket queries with
-// nothing, so only the running-instance fan-out decides completeness.
-func emptyInstanceBuckets(t *testing.T, nc *nats.Conn) {
-	t.Helper()
-	answerAsNode(t, nc, "ec2.DescribeStoppedInstances", "", describeOutput(t))
-	answerAsNode(t, nc, "ec2.DescribeTerminatedInstances", "", describeOutput(t))
-}
-
-// A node that never answers is the timeout case: its running guests are absent
-// from the aggregate, so an empty answer from the rest is not evidence the
-// account has no instances.
-func TestInstanceReaperFailsClosedWhenANodeDoesNotAnswer(t *testing.T) {
-	t.Parallel()
-	_, nc := testutil.StartTestNATS(t)
-	emptyInstanceBuckets(t, nc)
-	answerAsNode(t, nc, "ec2.DescribeInstances", "node-1", describeOutput(t))
-	require.NoError(t, nc.Flush())
-
-	found, err := testInventoryReaper(nc).List(testCtx(t), testAccountID)
-
-	require.Error(t, err, "a listing missing a node must not read as an empty account, got %v", found)
-	code, _, ok := awserrors.ResolveErrorDetail(err)
-	require.True(t, ok)
-	assert.Equal(t, awserrors.ErrorServiceUnavailable, code)
-}
-
-// The partial list is the dangerous shape: the reaper would terminate what it
-// can see and then report the stage drained on the next, equally partial pass.
-func TestInstanceReaperFailsClosedOnAPartialList(t *testing.T) {
-	t.Parallel()
-	_, nc := testutil.StartTestNATS(t)
-	emptyInstanceBuckets(t, nc)
-	answerAsNode(t, nc, "ec2.DescribeInstances", "node-1", describeOutput(t, "i-on-node-1"))
-	require.NoError(t, nc.Flush())
-
-	found, err := testInventoryReaper(nc).List(testCtx(t), testAccountID)
-
-	require.Error(t, err, "a partial listing must not be returned as the inventory, got %v", found)
-}
-
-// An error frame from a node is no better than silence: it says nothing about
-// what that node is running.
-func TestInstanceReaperFailsClosedWhenANodeAnswersWithAnError(t *testing.T) {
-	_, nc := testutil.StartTestNATS(t)
-	emptyInstanceBuckets(t, nc)
-	answerAsNode(t, nc, "ec2.DescribeInstances", "node-1", describeOutput(t))
-	answerAsNode(t, nc, "ec2.DescribeInstances", "node-2", awserrors.GenerateErrorPayload(awserrors.ErrorServerInternal))
-	require.NoError(t, nc.Flush())
-
-	_, err := testInventoryReaper(nc).List(testCtx(t), testAccountID)
-
-	require.Error(t, err)
-}
-
-// Completeness is judged against the configured node set, not a count of
-// replies: two frames from one node while another is silent is still missing
-// a node.
-func TestInstanceReaperFailsClosedWhenOneNodeAnswersTwice(t *testing.T) {
-	t.Parallel()
-	_, nc := testutil.StartTestNATS(t)
-	emptyInstanceBuckets(t, nc)
-	answerAsNode(t, nc, "ec2.DescribeInstances", "node-1", describeOutput(t))
-	answerAsNode(t, nc, "ec2.DescribeInstances", "node-1", describeOutput(t))
-	require.NoError(t, nc.Flush())
-
-	_, err := testInventoryReaper(nc).List(testCtx(t), testAccountID)
-
-	require.Error(t, err)
-}
-
-// Stopped instances live in a bucket, not on a node. A failed bucket query
-// hides them as surely as a silent node hides running ones.
-func TestInstanceReaperFailsClosedWhenTheStoppedBucketFails(t *testing.T) {
-	_, nc := testutil.StartTestNATS(t)
-	answerAsNode(t, nc, "ec2.DescribeStoppedInstances", "", awserrors.GenerateErrorPayload(awserrors.ErrorServerInternal))
-	answerAsNode(t, nc, "ec2.DescribeTerminatedInstances", "", describeOutput(t))
-	answerAsNode(t, nc, "ec2.DescribeInstances", "node-1", describeOutput(t))
-	answerAsNode(t, nc, "ec2.DescribeInstances", "node-2", describeOutput(t))
-	require.NoError(t, nc.Flush())
-
-	_, err := testInventoryReaper(nc).List(testCtx(t), testAccountID)
-
-	require.Error(t, err)
-}
-
-// The healthy cluster still lists: every configured node answered, so the
-// union is the account's inventory.
-func TestInstanceReaperListsEveryNodeWhenAllAnswer(t *testing.T) {
-	_, nc := testutil.StartTestNATS(t)
-	emptyInstanceBuckets(t, nc)
-	answerAsNode(t, nc, "ec2.DescribeInstances", "node-1", describeOutput(t, "i-on-node-1"))
-	answerAsNode(t, nc, "ec2.DescribeInstances", "node-2", describeOutput(t, "i-on-node-2"))
-	require.NoError(t, nc.Flush())
-
-	started := time.Now()
-	found, err := testInventoryReaper(nc).List(testCtx(t), testAccountID)
-	require.NoError(t, err)
-
-	var ids []string
-	for _, resource := range found {
-		ids = append(ids, resource.ID)
-	}
-	assert.ElementsMatch(t, []string{"i-on-node-1", "i-on-node-2"}, ids)
-	assert.Less(t, time.Since(started), 2*time.Second, "a fully answered fan-out must not wait out its deadline")
-}
-
-// End to end through the engine: with a node silent, the compute stage must
-// not read as drained and the account record must survive.
-func TestTeardownDoesNotReportComputeDrainedWhileANodeIsSilent(t *testing.T) {
-	t.Parallel()
-	_, nc := testutil.StartTestNATS(t)
-	emptyInstanceBuckets(t, nc)
-	answerAsNode(t, nc, "ec2.DescribeInstances", "node-1", describeOutput(t))
-	require.NoError(t, nc.Flush())
-
-	accounts := newFakeAccounts()
-	volumes := &fakeReaper{kind: "volume", stage: StageStorage,
-		resources: []Resource{{Kind: "volume", ID: "vol-held-by-node-2"}}}
-	engine := quietEngine(accounts, testInventoryReaper(nc), volumes)
-
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	defer cancel()
-	result, err := engine.Teardown(ctx, request())
-
-	require.Error(t, err, "teardown must not succeed on an incomplete instance inventory")
-	assert.ErrorContains(t, err, "list instance")
-	assert.False(t, accounts.deleted, "the account record must survive an incomplete inventory")
-	for _, stage := range result.Stages {
-		assert.NotEqual(t, StageCompute, stage.Stage, "the compute stage must not be reported finished")
-	}
-	assert.Equal(t, 0, volumes.deletesAttempted, "storage must not be touched before compute is proven drained")
-}

spinifex/operator/accountlifecycle/reapers_ec2_test.go

@@ -106,29 +106,6 @@ func (c *fakeCluster) fail(subject, code string) {
 	c.errorCode[subject] = code
 }
 
-// handle answers subject from state the test owns, for doubles whose answer
-// must change as the reaper acts — an attachment a force detach clears.
-func (c *fakeCluster) handle(subject string, handler func(data []byte) (any, string)) {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	c.handlers[subject] = handler
-}
-
-// echoAsNode adds a second reply to every request on subject, attributed to
-// node, so one node can answer twice while another stays silent.
-func (c *fakeCluster) echoAsNode(t *testing.T, subject, node string) {
-	t.Helper()
-	sub, err := c.nc.Subscribe(subject, func(msg *nats.Msg) {
-		out := nats.NewMsg(msg.Reply)
-		out.Data = []byte(`{}`)
-		out.Header.Set(natsmsg.NodeIDHeader, node)
-		_ = msg.RespondMsg(out)
-	})
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = sub.Unsubscribe() })
-	require.NoError(t, c.nc.Flush())
-}
-
 func (c *fakeCluster) called() []string {
 	c.mu.Lock()
 	defer c.mu.Unlock()
@@ -150,13 +127,6 @@ func testCtx(t *testing.T) context.Context {
 	return ctx
 }
 
-// silentClusterCtx outlasts the full fan-out deadline a silent node costs.
-func silentClusterCtx(t *testing.T) context.Context {
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-	t.Cleanup(cancel)
-	return ctx
-}
-
 // The stage graph is what keeps teardown converging, so it is asserted rather
 // than left to the order the constructor happens to list.
 func TestEC2ReapersAreOrderedByDependency(t *testing.T) {
@@ -168,16 +138,10 @@ func TestEC2ReapersAreOrderedByDependency(t *testing.T) {
 	}
 
 	assert.Equal(t, []string{
-		"instance",
+		"snapshot", "image",
-		"snapshot", "image", "volume",
 		"address", "network-interface", "internet-gateway", "route-table", "subnet", "security-group", "vpc",
 	}, kinds)
 
-	// A volume with a live snapshot refuses to delete, so the reverse order
-	// would strand every snapshot-backed volume in the account.
-	assert.Less(t, indexOfKind(reapers, "snapshot"), indexOfKind(reapers, "volume"))
-	assert.Equal(t, StageCompute, reapers[indexOfKind(reapers, "instance")].Stage())
-	assert.Equal(t, StageStorage, reapers[indexOfKind(reapers, "volume")].Stage())
 	assert.Equal(t, StageNetwork, reapers[indexOfKind(reapers, "vpc")].Stage())
 }
 
@@ -190,333 +154,6 @@ func indexOfKind(reapers []Reaper, kind string) int {
 	return -1
 }
 
-// A terminated instance holds nothing. Listing it would keep the compute stage
-// polling until its timeout and then report it stuck, for no reason.
-func TestInstanceReaperSkipsTerminalStates(t *testing.T) {
-	cluster := newFakeCluster(t)
-	cluster.reply("ec2.DescribeInstances", ec2.DescribeInstancesOutput{
-		Reservations: []*ec2.Reservation{{Instances: []*ec2.Instance{
-			{InstanceId: aws.String("i-running"), State: &ec2.InstanceState{Name: aws.String("running")}},
-			{InstanceId: aws.String("i-gone"), State: &ec2.InstanceState{Name: aws.String("terminated")}},
-			{InstanceId: aws.String("i-going"), State: &ec2.InstanceState{Name: aws.String("shutting-down")}},
-			{State: &ec2.InstanceState{Name: aws.String("running")}},
-		}}},
-	})
-
-	reaper := &instanceReaper{nc: cluster.nc, nodeIDs: []string{fakeClusterNode}}
-	found, err := reaper.List(testCtx(t), "000000000042")
-	require.NoError(t, err)
-
-	require.Len(t, found, 1)
-	assert.Equal(t, "i-running", found[0].ID)
-	assert.Equal(t, "running", found[0].Detail)
-}
-
-// Termination is addressed to the node hosting the instance, which is what
-// makes a stuck host the case --force exists for.
-func TestInstanceReaperTerminates(t *testing.T) {
-	cluster := newFakeCluster(t)
-	reaper := &instanceReaper{nc: cluster.nc, nodeIDs: []string{fakeClusterNode}}
-
-	require.NoError(t, reaper.Delete(testCtx(t), "000000000042",
-		Resource{Kind: "instance", ID: "i-0abc0123456789def"}, false))
-
-	assert.Contains(t, cluster.called(), "ec2.cmd.i-0abc0123456789def")
-}
-
-// The attachment is the reason a volume will not go, so it has to reach the
-// stuck report — "stuck" and "stuck on i-abc" are different bug reports.
-func TestVolumeReaperCarriesItsAttachment(t *testing.T) {
-	cluster := newFakeCluster(t)
-	cluster.reply("ec2.DescribeVolumes", ec2.DescribeVolumesOutput{
-		Volumes: []*ec2.Volume{
-			{VolumeId: aws.String("vol-free")},
-			{VolumeId: aws.String("vol-held"), Attachments: []*ec2.VolumeAttachment{
-				{InstanceId: aws.String("i-abc")},
-			}},
-		},
-	})
-
-	reaper := newVolumeReaper(cluster.nc, fakeClusterNode)
-	found, err := reaper.List(testCtx(t), "000000000042")
-	require.NoError(t, err)
-
-	require.Len(t, found, 2)
-	assert.Empty(t, found[0].Detail)
-	assert.Equal(t, "attached to i-abc", found[1].Detail)
-}
-
-// Without --force an attached volume must surface the refusal, not swallow it:
-// a stuck resource is a bug we want reported rather than papered over.
-func TestVolumeReaperReportsRefusalWithoutForce(t *testing.T) {
-	cluster := newFakeCluster(t)
-	cluster.fail("ec2.DeleteVolume", "VolumeInUse")
-
-	reaper := newVolumeReaper(cluster.nc, fakeClusterNode)
-	err := reaper.Delete(testCtx(t), "000000000042", Resource{Kind: "volume", ID: "vol-held"}, false)
-
-	require.Error(t, err)
-	assert.NotContains(t, cluster.called(), forceDetachSubject)
-}
-
-// heldVolume is one volume record as its owner keeps it: the ordinary delete
-// refuses while the record names an instance, and a force detach clears it.
-type heldVolume struct {
-	mu       sync.Mutex
-	id       string
-	holder   string
-	deleted  bool
-	detaches int
-}
-
-func newHeldVolume(cluster *fakeCluster, id, holder string) *heldVolume {
-	volume := &heldVolume{id: id, holder: holder}
-	cluster.handle("ec2.DescribeVolumes", func(data []byte) (any, string) {
-		var input ec2.DescribeVolumesInput
-		_ = json.Unmarshal(data, &input)
-		volume.mu.Lock()
-		defer volume.mu.Unlock()
-		if volume.deleted {
-			if len(input.VolumeIds) > 0 {
-				return nil, awserrors.ErrorInvalidVolumeNotFound
-			}
-			return ec2.DescribeVolumesOutput{}, ""
-		}
-		described := &ec2.Volume{VolumeId: aws.String(volume.id), State: aws.String("available")}
-		if volume.holder != "" {
-			described.State = aws.String("in-use")
-			described.Attachments = []*ec2.VolumeAttachment{
-				{VolumeId: aws.String(volume.id), InstanceId: aws.String(volume.holder), State: aws.String("attached")},
-			}
-		}
-		return ec2.DescribeVolumesOutput{Volumes: []*ec2.Volume{described}}, ""
-	})
-	cluster.handle(forceDetachSubject, func([]byte) (any, string) {
-		volume.mu.Lock()
-		defer volume.mu.Unlock()
-		volume.detaches++
-		previous := volume.holder
-		volume.holder = ""
-		return ec2.VolumeAttachment{VolumeId: aws.String(volume.id), InstanceId: aws.String(previous)}, ""
-	})
-	cluster.handle("ec2.DeleteVolume", func([]byte) (any, string) {
-		volume.mu.Lock()
-		defer volume.mu.Unlock()
-		switch {
-		case volume.deleted:
-			return nil, awserrors.ErrorInvalidVolumeNotFound
-		case volume.holder != "":
-			return nil, awserrors.ErrorVolumeInUse
-		}
-		volume.deleted = true
-		return ec2.DeleteVolumeOutput{}, ""
-	})
-	return volume
-}
-
-func (v *heldVolume) state() (deleted bool, holder string, detaches int) {
-	v.mu.Lock()
-	defer v.mu.Unlock()
-	return v.deleted, v.holder, v.detaches
-}
-
-func instancesIn(state string, instanceIDs ...string) ec2.DescribeInstancesOutput {
-	reservation := &ec2.Reservation{ReservationId: aws.String("r-" + state)}
-	for _, id := range instanceIDs {
-		reservation.Instances = append(reservation.Instances, &ec2.Instance{
-			InstanceId: aws.String(id), State: &ec2.InstanceState{Name: aws.String(state)},
-		})
-	}
-	return ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{reservation}}
-}
-
-// Force must not clear an attachment a live guest still holds. The instance
-// carries no block device mapping for the volume, so the attachment record is
-// the only thing standing between the guest and a deleted disk.
-func TestVolumeReaperForceRetainsAVolumeALiveInstanceHolds(t *testing.T) {
-	t.Parallel()
-	for _, state := range []string{"running", "stopped", "shutting-down", "pending"} {
-		t.Run(state, func(t *testing.T) {
-			t.Parallel()
-			cluster := newFakeCluster(t)
-			volume := newHeldVolume(cluster, "vol-held", "i-live")
-			subject := "ec2.DescribeInstances"
-			if state == "stopped" {
-				subject = "ec2.DescribeStoppedInstances"
-			}
-			cluster.reply(subject, instancesIn(state, "i-live"))
-
-			reaper := newVolumeReaper(cluster.nc, fakeClusterNode)
-			err := reaper.Delete(testCtx(t), "000000000042", Resource{Kind: "volume", ID: "vol-held"}, true)
-
-			require.Error(t, err)
-			assert.Contains(t, err.Error(), "vol-held")
-			assert.Contains(t, err.Error(), "i-live")
-			assert.Contains(t, err.Error(), state)
-			deleted, holder, detaches := volume.state()
-			assert.False(t, deleted, "a volume a live guest holds must survive teardown")
-			assert.Equal(t, "i-live", holder, "the attachment must not be cleared under a live guest")
-			assert.Zero(t, detaches)
-		})
-	}
-}
-
-// "Cannot prove gone" is not "gone": every way the proof can fall short keeps
-// the volume and its attachment exactly as they were.
-func TestVolumeReaperForceRetainsAVolumeWhenTheHolderCannotBeProvenGone(t *testing.T) {
-	t.Parallel()
-	cases := []struct {
-		name  string
-		setup func(cluster *fakeCluster)
-		nodes []string
-	}{
-		{"configured node set unknown", func(*fakeCluster) {}, []string{}},
-		{"node describe fails", func(cluster *fakeCluster) {
-			cluster.fail("ec2.DescribeInstances", awserrors.ErrorServerInternal)
-		}, nil},
-		{"terminated bucket unreachable", func(cluster *fakeCluster) {
-			cluster.fail("ec2.DescribeTerminatedInstances", awserrors.ErrorServerInternal)
-		}, nil},
-		{"stopped bucket unreachable", func(cluster *fakeCluster) {
-			cluster.fail("ec2.DescribeStoppedInstances", awserrors.ErrorServerInternal)
-		}, nil},
-		{"sources disagree", func(cluster *fakeCluster) {
-			cluster.reply("ec2.DescribeTerminatedInstances", instancesIn("terminated", "i-held"))
-			cluster.reply("ec2.DescribeInstances", instancesIn("running", "i-held"))
-		}, nil},
-		{"volume describe fails", func(cluster *fakeCluster) {
-			cluster.fail("ec2.DescribeVolumes", awserrors.ErrorServerInternal)
-		}, nil},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-			nodes := tc.nodes
-			if nodes == nil {
-				nodes = []string{fakeClusterNode}
-			}
-			assertForceRetainsTheHeldVolume(t, tc.setup, nodes)
-		})
-	}
-}
-
-// assertForceRetainsTheHeldVolume force-deletes a volume i-held holds and
-// asserts the volume and its attachment survive untouched.
-func assertForceRetainsTheHeldVolume(t *testing.T, setup func(cluster *fakeCluster), nodes []string) {
-	t.Helper()
-	cluster := newFakeCluster(t)
-	volume := newHeldVolume(cluster, "vol-held", "i-held")
-	setup(cluster)
-
-	reaper := newVolumeReaper(cluster.nc, nodes...)
-	err := reaper.Delete(silentClusterCtx(t), "000000000042", Resource{Kind: "volume", ID: "vol-held"}, true)
-
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "vol-held")
-	deleted, holder, detaches := volume.state()
-	assert.False(t, deleted)
-	assert.Equal(t, "i-held", holder)
-	assert.Zero(t, detaches)
-}
-
-// The deadlock force exists for: the instance is gone but its volume record
-// still names it, so the ordinary delete refuses forever. Once the holder is
-// proven absent the attachment is cleared and the volume deleted.
-func TestVolumeReaperForceDeletesOnceTheHolderIsProvenGone(t *testing.T) {
-	t.Parallel()
-	cases := map[string]func(cluster *fakeCluster){
-		"terminated": func(cluster *fakeCluster) {
-			cluster.reply("ec2.DescribeTerminatedInstances", instancesIn("terminated", "i-gone"))
-		},
-		"not found": func(*fakeCluster) {},
-	}
-	for name, setup := range cases {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-			cluster := newFakeCluster(t)
-			volume := newHeldVolume(cluster, "vol-held", "i-gone")
-			setup(cluster)
-
-			reaper := newVolumeReaper(cluster.nc, fakeClusterNode)
-			require.NoError(t, reaper.Delete(testCtx(t), "000000000042",
-				Resource{Kind: "volume", ID: "vol-held"}, true))
-
-			deleted, holder, detaches := volume.state()
-			assert.True(t, deleted)
-			assert.Empty(t, holder)
-			assert.Equal(t, 1, detaches)
-
-			var input ec2.DetachVolumeInput
-			cluster.request(t, forceDetachSubject, &input)
-			assert.Equal(t, "vol-held", aws.StringValue(input.VolumeId))
-		})
-	}
-}
-
-// With no holder on record, the refusal is about something else — a snapshot,
-// a mapping the delete guard found — and clearing the record cannot help.
-func TestVolumeReaperForceLeavesAnUnattachedRefusalAlone(t *testing.T) {
-	cluster := newFakeCluster(t)
-	cluster.reply("ec2.DescribeVolumes", ec2.DescribeVolumesOutput{
-		Volumes: []*ec2.Volume{{VolumeId: aws.String("vol-snapped"), State: aws.String("available")}},
-	})
-	cluster.fail("ec2.DeleteVolume", awserrors.ErrorVolumeInUse)
-
-	reaper := newVolumeReaper(cluster.nc, fakeClusterNode)
-	err := reaper.Delete(testCtx(t), "000000000042", Resource{Kind: "volume", ID: "vol-snapped"}, true)
-
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), awserrors.ErrorVolumeInUse)
-	assert.NotContains(t, cluster.called(), forceDetachSubject)
-}
-
-// End to end through the engine: a forced teardown whose guest is still up
-// leaves the volume, names why, and keeps the account TERMINATING.
-func TestForcedTeardownKeepsTheAccountWhileALiveGuestHoldsAVolume(t *testing.T) {
-	t.Parallel()
-	cluster := newFakeCluster(t)
-	volume := newHeldVolume(cluster, "vol-held", "i-live")
-	cluster.reply("ec2.DescribeInstances", instancesIn("running", "i-live"))
-
-	accounts := newFakeAccounts()
-	engine := quietEngine(accounts, newVolumeReaper(cluster.nc, fakeClusterNode))
-	req := request()
-	req.Force = true
-
-	result, err := engine.Teardown(testCtx(t), req)
-
-	require.ErrorIs(t, err, ErrResourcesStuck)
-	require.Equal(t, 1, result.StuckCount())
-	var stuck Stuck
-	for _, stage := range result.Stages {
-		if len(stage.Stuck) > 0 {
-			stuck = stage.Stuck[0]
-		}
-	}
-	assert.Equal(t, "vol-held", stuck.Resource.ID)
-	assert.Contains(t, stuck.Reason, "vol-held")
-	assert.Contains(t, stuck.Reason, "i-live")
-	assert.Contains(t, stuck.Reason, "running")
-	assert.False(t, result.AccountDeleted)
-	assert.False(t, accounts.deleted)
-	assert.Equal(t, AccountStatusTerminating, accounts.account.Status)
-	deleted, holder, _ := volume.state()
-	assert.False(t, deleted)
-	assert.Equal(t, "i-live", holder)
-}
-
-// A volume that is already gone is a success. Teardown re-runs after a crash
-// and would otherwise never finish what the first pass started.
-func TestVolumeReaperTreatsAMissingVolumeAsDeleted(t *testing.T) {
-	cluster := newFakeCluster(t)
-	cluster.fail("ec2.DeleteVolume", "InvalidVolume.NotFound")
-
-	reaper := newVolumeReaper(cluster.nc, fakeClusterNode)
-
-	assert.NoError(t, reaper.Delete(testCtx(t), "000000000042",
-		Resource{Kind: "volume", ID: "vol-gone"}, false))
-}
-
 // Owner-scoped: an image the account can merely see is not its property and
 // must survive its deletion.
 func TestImageReaperListsOnlyOwnedImages(t *testing.T) {
@@ -781,9 +418,9 @@ func TestSubnetAndVPCReapersReportTheirVPC(t *testing.T) {
 // empty account is how a stage reports itself drained while resources remain.
 func TestAListingFailureIsReported(t *testing.T) {
 	cluster := newFakeCluster(t)
-	cluster.fail("ec2.DescribeVolumes", "InternalError")
+	cluster.fail("ec2.DescribeSnapshots", "InternalError")
 
-	_, err := (newVolumeReaper(cluster.nc, fakeClusterNode)).List(testCtx(t), "000000000042")
+	_, err := (&snapshotReaper{nc: cluster.nc}).List(testCtx(t), "000000000042")
 
 	assert.Error(t, err)
 }
@@ -931,7 +568,3 @@ func TestEC2ReapersRemoveInterfacesBeforeSubnets(t *testing.T) {
 	require.NotEqual(t, -1, eni, "there is no network-interface reaper")
 	require.Less(t, eni, subnet, "interfaces must be reaped before the subnets holding them")
 }
-
-func newVolumeReaper(nc *nats.Conn, nodeIDs ...string) *volumeReaper {
-	return &volumeReaper{nc: nc, nodeIDs: nodeIDs}
-}

spinifex/operator/accountlifecycle/reapers_ecs_test.go

@@ -5,6 +5,7 @@ package accountlifecycle
 // registered in.
 
 import (
+	"context"
 	"encoding/json"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"sync"
@@ -13,6 +14,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ecs"
 	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountcleanup"
 	ecsdomain "github.com/mulgadc/spinifex/spinifex/domains/ecs"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
@@ -173,32 +175,37 @@ func TestECSClusterNameFromARN(t *testing.T) {
 	assert.Empty(t, ecsClusterName(""))
 }
 
+// orderedCompute records when the engine reaches the instance cleanup.
+type orderedCompute struct {
+	ComputeStage
+
+	order *[]string
+}
+
+func (o *orderedCompute) CleanupCompute(ctx context.Context, req *accountcleanup.ComputeRequest) (*accountcleanup.ComputeResult, error) {
+	*o.order = append(*o.order, "instances")
+	return o.ComputeStage.CleanupCompute(ctx, req)
+}
+
 // Container instances are ordinary EC2 instances in the tenant's account, so
-// the instance reaper takes them — but the tasks riding on them have to be
+// the compute cleanup terminates them — but the tasks riding on them have to be
-// stopped first, which is what puts ECS ahead of it in the compute stage.
+// stopped first, so the compute-stage reapers must run before that cleanup.
-func TestECSReapersRunBeforeTheInstanceReaper(t *testing.T) {
+func TestECSClustersAreDeletedBeforeTheInstancesAreTerminated(t *testing.T) {
-	reapers := ECSReapers(nil)
+	var order []string
-	reapers = append(reapers, EC2Reapers(nil, 1, []string{"node-1"})...)
+	cluster := &fakeReaper{
+		kind: "ecs-cluster", stage: StageCompute, order: &order,
+		resources: []Resource{{Kind: "ecs-cluster", ID: "web"}},
+	}
+	accounts := newFakeAccounts()
+	engine := quietEngine(accounts, cluster)
+	compute, _ := cleanStages()
+	engine.Compute = &orderedCompute{ComputeStage: compute, order: &order}
 
-	ecsIndex := indexOfKind(reapers, "ecs-cluster")
+	_, err := engine.Teardown(context.Background(), request())
-	instanceIndex := indexOfKind(reapers, "instance")
 
-	require.NotEqual(t, -1, ecsIndex)
+	require.NoError(t, err)
-	require.NotEqual(t, -1, instanceIndex)
+	require.Equal(t, []string{"ecs-cluster/web", "instances"}, order)
-	assert.Less(t, ecsIndex, instanceIndex)
+	for _, reaper := range ECSReapers(nil) {
-	assert.Equal(t, StageCompute, reapers[ecsIndex].Stage())
+		assert.Equal(t, StageCompute, reaper.Stage())
-
+	}
-	// Both are in the same stage, so the engine's within-stage ordering is
-	// what enforces this rather than the stage graph.
-	assert.Equal(t, reapers[instanceIndex].Stage(), reapers[ecsIndex].Stage())
-}
-
-// SortReapers is what the engine applies, and a stable sort is the only reason
-// the registration order above survives it.
-func TestSortReapersKeepsECSAheadOfInstances(t *testing.T) {
-	reapers := ECSReapers(nil)
-	reapers = append(reapers, EC2Reapers(nil, 1, []string{"node-1"})...)
-	SortReapers(reapers)
-
-	assert.Less(t, indexOfKind(reapers, "ecs-cluster"), indexOfKind(reapers, "instance"))
 }

spinifex/operator/admin/account_delete.go

@@ -124,12 +124,12 @@ func (d Deps) deleteAccount(ctx context.Context, body []byte) (any, error) {
 		return nil, errors.New(awserrors.ErrorInternalError)
 	}
 
-	if !req.DryRun && d.PlatformCleaner == nil {
+	if !req.DryRun && (d.PlatformCleaner == nil || d.ComputeCleaner == nil || d.StorageCleaner == nil) {
 		slog.Error("DeleteAccount: platform cleanup not available")
 		return nil, errors.New(awserrors.ErrorServiceUnavailable)
 	}
 
-	engine, err := accountteardown.NewClusterEngine(ctx, d.NATS, accountteardown.ClusterNodes{Expected: d.ExpectedNodes, DaemonIDs: d.NodeIDs, GatewayIDs: d.GatewayNodeIDs}, d.Termination, d.BucketStore, d.PlatformCleaner)
+	engine, err := accountteardown.NewClusterEngine(ctx, d.NATS, accountteardown.ClusterNodes{Expected: d.ExpectedNodes, DaemonIDs: d.NodeIDs, GatewayIDs: d.GatewayNodeIDs}, d.Termination, d.BucketStore, d.PlatformCleaner, accountteardown.WithComputeCleanup(d.ComputeCleaner, d.StorageCleaner))
 	if err != nil {
 		slog.Error("DeleteAccount: failed to build teardown engine", "err", err)
 		return nil, errors.New(awserrors.ErrorServiceUnavailable)

spinifex/operator/admin/account_delete_dryrun_test.go

@@ -26,7 +26,7 @@ func TestAdminDeleteAccountDryRunAnswersTheInventoryAndCleansNothing(t *testing.
 	gotJSON, err := json.Marshal(got)
 	require.NoError(t, err)
 
-	engine, err := accountteardown.NewClusterEngine(t.Context(), gw.NATS, accountteardown.ClusterNodes{Expected: gw.ExpectedNodes, DaemonIDs: gw.NodeIDs, GatewayIDs: gw.GatewayNodeIDs}, gw.Termination, gw.BucketStore, gw.PlatformCleaner)
+	engine, err := accountteardown.NewClusterEngine(t.Context(), gw.NATS, accountteardown.ClusterNodes{Expected: gw.ExpectedNodes, DaemonIDs: gw.NodeIDs, GatewayIDs: gw.GatewayNodeIDs}, gw.Termination, gw.BucketStore, gw.PlatformCleaner, accountteardown.WithComputeCleanup(gw.ComputeCleaner, gw.StorageCleaner))
 	require.NoError(t, err)
 	inventory, err := engine.Inventory(t.Context(), account.AccountID)
 	require.NoError(t, err)

spinifex/operator/admin/account_delete_job_test.go

@@ -8,7 +8,9 @@ import (
 	"context"
 	"encoding/json"
 	operatorv1 "github.com/mulgadc/spinifex/contracts/operator/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountcleanup"
 	"strings"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -148,6 +150,8 @@ func newDeleteAccountGateway(t *testing.T) Deps {
 		GatewayNodeIDs:  []string{"node-1"},
 		BucketStore:     emptyBucketStore{},
 		PlatformCleaner: &panicCleaner{},
+		ComputeCleaner:  emptyCompute{},
+		StorageCleaner:  emptyStorage{},
 	}
 }
 
@@ -607,3 +611,85 @@ func TestDeletionTrackerHeartbeatStops(t *testing.T) {
 	tracker.persist(t.Context())
 	assert.NotEmpty(t, describeDeletion(t, gw, "000000000042").UpdatedAt)
 }
+
+// emptyCompute and emptyStorage stand in for the EC2 cleanup of an account that
+// holds no instance, spot request or volume.
+type (
+	emptyCompute struct{}
+	emptyStorage struct{}
+)
+
+func cleanOutcome(accountID string) *accountcleanup.Outcome {
+	return &accountcleanup.Outcome{AccountID: accountID, Status: accountcleanup.StatusClean}
+}
+
+func (emptyCompute) CleanupCompute(_ context.Context, req *accountcleanup.ComputeRequest) (*accountcleanup.ComputeResult, error) {
+	return cleanOutcome(req.AccountID), nil
+}
+
+func (emptyCompute) ComputeRemaining(_ context.Context, accountID string) (*accountcleanup.Outcome, error) {
+	return cleanOutcome(accountID), nil
+}
+
+func (emptyStorage) CleanupStorage(_ context.Context, req *accountcleanup.StorageRequest) (*accountcleanup.StorageResult, error) {
+	return cleanOutcome(req.AccountID), nil
+}
+
+func (emptyStorage) StorageRemaining(_ context.Context, accountID string) (*accountcleanup.Outcome, error) {
+	return cleanOutcome(accountID), nil
+}
+
+// blockedCompute holds an instance the cleanup cannot release until unblocked,
+// the way a managed service still owning a worker does.
+type blockedCompute struct {
+	blocked atomic.Bool
+}
+
+func (b *blockedCompute) outcome(accountID string) *accountcleanup.Outcome {
+	out := cleanOutcome(accountID)
+	if b.blocked.Load() {
+		out.Items = []accountcleanup.Item{{
+			Kind: accountcleanup.KindInstance, ResourceID: "i-held", State: "running",
+			Disposition: accountcleanup.DispositionBlocked, Reason: accountcleanup.ReasonManagedServiceOwner,
+		}}
+	}
+	return out
+}
+
+func (b *blockedCompute) CleanupCompute(_ context.Context, req *accountcleanup.ComputeRequest) (*accountcleanup.ComputeResult, error) {
+	return b.outcome(req.AccountID), nil
+}
+
+func (b *blockedCompute) ComputeRemaining(_ context.Context, accountID string) (*accountcleanup.Outcome, error) {
+	return b.outcome(accountID), nil
+}
+
+// A job that ends FAILED naming a blocker the loop cannot clear keeps the
+// account, and an operator rerun after the blocker clears converges it.
+func TestAdminDeleteAccountFailedJobConvergesOnOperatorRerun(t *testing.T) {
+	t.Parallel()
+	gw := newDeleteAccountGateway(t)
+	compute := &blockedCompute{}
+	compute.blocked.Store(true)
+	gw.ComputeCleaner = compute
+	account := newTenantAccount(t, gw, "tenant@example.com")
+
+	_, err := gw.deleteAccount(t.Context(),
+		deleteAccountBody(t, account.AccountID, "tenant@example.com", strings.Repeat("a", 32)))
+	require.NoError(t, err)
+	require.Eventually(t, func() bool {
+		return describeDeletion(t, gw, account.AccountID).State == operatorv1.DeletionStateFailed
+	}, 60*time.Second, 50*time.Millisecond)
+	failed := describeDeletion(t, gw, account.AccountID)
+	assert.NotEmpty(t, failed.Error)
+	_, stillThere := gw.IAM.GetAccount(account.AccountID)
+	require.NoError(t, stillThere, "a FAILED job must keep the account")
+
+	compute.blocked.Store(false)
+	_, err = gw.deleteAccount(t.Context(),
+		deleteAccountBody(t, account.AccountID, "tenant@example.com", strings.Repeat("b", 32)))
+	require.NoError(t, err)
+	require.Eventually(t, func() bool {
+		return describeDeletion(t, gw, account.AccountID).State == operatorv1.DeletionStateCompleted
+	}, 60*time.Second, 50*time.Millisecond)
+}

spinifex/operator/admin/account_delete_resumer.go

@@ -99,7 +99,7 @@ func (d Deps) resumeDeletion(
 	revision uint64,
 	now time.Time,
 ) bool {
-	engine, err := accountteardown.NewClusterEngine(ctx, d.NATS, accountteardown.ClusterNodes{Expected: d.ExpectedNodes, DaemonIDs: d.NodeIDs, GatewayIDs: d.GatewayNodeIDs}, d.Termination, d.BucketStore, d.PlatformCleaner)
+	engine, err := accountteardown.NewClusterEngine(ctx, d.NATS, accountteardown.ClusterNodes{Expected: d.ExpectedNodes, DaemonIDs: d.NodeIDs, GatewayIDs: d.GatewayNodeIDs}, d.Termination, d.BucketStore, d.PlatformCleaner, accountteardown.WithComputeCleanup(d.ComputeCleaner, d.StorageCleaner))
 	if err != nil {
 		// Nothing was written, so a later scan retries.
 		slog.Error("DeletionResumer: failed to build teardown engine", "account_id", existing.AccountID, "operation_id", existing.OperationID, "err", err)

spinifex/operator/admin/registration.go

@@ -57,7 +57,11 @@ type Deps struct {
 	Termination accountteardown.IAMBackend
 	// PlatformCleaner is the in-process EC2 platform cleanup every DeleteAccount
 	// runs. Nil refuses a request that is not a dry run.
-	PlatformCleaner   accountteardown.PlatformCleaner
+	PlatformCleaner accountteardown.PlatformCleaner
+	// ComputeCleaner and StorageCleaner are the EC2 instance, spot request and
+	// volume cleanup. Nil refuses a request that is not a dry run.
+	ComputeCleaner    accountteardown.ComputeStage
+	StorageCleaner    accountteardown.StorageStage
 	Quota             *quota.Service
 	ExpectedNodes     int
 	NodeIDs           []string

spinifex/operator/cli/admin_eks.go

@@ -3,11 +3,16 @@ package cli
 import (
 	"context"
 	"fmt"
+	"maps"
 	"os"
+	"slices"
 	"time"
 
+	"github.com/aws/aws-sdk-go/service/ec2"
+	ec2inventory "github.com/mulgadc/spinifex/spinifex/domains/ec2/inventory"
 	ekscluster "github.com/mulgadc/spinifex/spinifex/domains/eks/cluster"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/spf13/cobra"
 )
 
@@ -75,3 +80,57 @@ func runAdminEksRestoreSnapshot(cmd *cobra.Command, _ []string) {
 	fmt.Printf("New control plane %s launched (snapshot %s)\n", out.NewInstanceID, out.Snapshot)
 	fmt.Printf("Status: %s\n", out.Status)
 }
+
+var adminEksOrphansCmd = &cobra.Command{
+	Use:   "orphans",
+	Short: "List running EKS control-plane VMs that have no cluster record",
+	Long: `orphans reports each running EKS control-plane VM whose cluster record, or whole
+account bucket, is missing (ORPHAN), and each it could not judge because a read
+failed (UNREADABLE). It only reads and never creates or removes anything.
+
+It lists the system account's instances through the strict inventory, so if any
+configured node does not answer it prints "listing incomplete" and exits non-zero
+without a partial list. It cannot say which node was silent. An instance with no
+ENI, or whose ENI is gone or not tagged as an EKS control plane, is skipped.
+
+Exit status is non-zero when the listing is incomplete or any row is ORPHAN or
+UNREADABLE.`,
+	Run: runAdminEksOrphans,
+}
+
+func init() {
+	adminEksCmd.AddCommand(adminEksOrphansCmd)
+}
+
+func runAdminEksOrphans(_ *cobra.Command, _ []string) {
+	cfg, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	defer nc.Close()
+
+	js, err := jetstream.New(nc)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	nodeIDs := slices.Sorted(maps.Keys(cfg.Nodes))
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	list := func(ctx context.Context) (*ec2.DescribeInstancesOutput, error) {
+		return ec2inventory.NewNATSInstances(nc, nodeIDs).ListAccountInstances(ctx, awsidentifiers.GlobalAccountID)
+	}
+	eni := ekscluster.ENITagReader(ec2inventory.NewENITagReader(js, awsidentifiers.GlobalAccountID))
+
+	orphans, unreadable, err := ekscluster.ReportControlPlaneOrphans(ctx, os.Stdout, js, list, eni, time.Now())
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	fmt.Printf("%d orphan(s), %d unreadable\n", orphans, unreadable)
+	if orphans+unreadable > 0 {
+		os.Exit(1)
+	}
+}

spinifex/runtime/compute/lambda/testlink/contract.go

@@ -49,6 +49,8 @@ type Request struct {
 	CodeURL     string          `json:"code_url"`
 	CodeSize    int64           `json:"code_size"`
 	CodeSHA256  string          `json:"code_sha256"`
+	// ImageDigest is the runtime image the gateway expects to run the code.
+	ImageDigest string `json:"image_digest"`
 	Payload     []byte          `json:"payload"`
 
 	// cred arrives beside the request on the control socket, never inside it.

spinifex/runtime/compute/lambda/testlink/control.go

@@ -23,7 +23,7 @@ import (
 // ControlName is the abstract Unix socket name the node's daemon serves. An
 // abstract name has no directory entry; the kernel frees it when the last
 // reference to the listening socket closes.
-const ControlName = "spinifex-lambda-testlink-control"
+const ControlName = "spinifex-lambda-testlink-control/3"
 
 // Service accounts at each end of the control socket.
 const (
@@ -31,10 +31,36 @@ const (
 	DaemonUser  = "spinifex-daemon"
 )
 
+// joinAfterFunc arms f on ctx's cancellation and returns a stop that, unlike
+// context.AfterFunc's, waits for f when it has already started.
+func joinAfterFunc(ctx context.Context, f func()) func() {
+	done := make(chan struct{})
+	stop := context.AfterFunc(ctx, func() { defer close(done); f() })
+	return func() {
+		if !stop() {
+			<-done
+		}
+	}
+}
+
 // MaxCredentialBytes bounds the encoded credential; a session's three values
 // fit well inside it.
 const MaxCredentialBytes = 8 << 10
 
+// Declared control-socket bounds, independent of any message bus. Request and
+// Reply payloads are capped raw; the frame cap leaves MetadataHeadroomBytes
+// beside a full base64 payload, and metadata has no bound of its own.
+const (
+	MaxPayloadBytes       = 6 << 20
+	MetadataHeadroomBytes = 64 << 10
+	// payloadFieldSlack covers the payload key, quotes and comma.
+	payloadFieldSlack = 64
+	MaxFrameBytes     = int64((MaxPayloadBytes+2)/3*4 + MetadataHeadroomBytes + payloadFieldSlack)
+	// MaxAdmitted bounds connections the daemon holds at once, from accept
+	// to close; the daemon runs one attempt, so one more is answered busy.
+	MaxAdmitted = 2
+)
+
 // Credential is one execution-role session for one attempt; it goes to the
 // guest in the first frame and never on NATS, to disk or into a log.
 type Credential = hostlink.Credential
@@ -118,25 +144,42 @@ func readFrame(r io.Reader, limit int64) ([]byte, error) {
 // ControlServer answers one request per connection from the gateway's UID.
 type ControlServer struct {
 	GatewayUID int
-	// RequestLimit bounds the encoded Request alone, as the NATS path did, and
+	// RequestLimit bounds the encoded Request frame and ReplyLimit the
-	// ReplyLimit the encoded Reply; a reply over it becomes unknown.
+	// encoded Reply; a reply over it, or over MaxPayloadBytes, becomes unknown.
 	RequestLimit int64
 	ReplyLimit   int64
 	// IOTimeout bounds reading the request and writing the reply.
 	IOTimeout time.Duration
 	Handle    func(ctx context.Context, req []byte, cred Credential) Reply
+	// AcceptStopped, if set, runs once Serve stops accepting, before it waits
+	// for the connections it started.
+	AcceptStopped func()
 }
 
 // Serve accepts until ctx ends or the listener fails, then waits for every
 // connection it started.
+// At most MaxAdmitted connections are held at once, each from accept to
+// close; others wait unaccepted in the kernel's listen backlog.
 func (s *ControlServer) Serve(ctx context.Context, ln *net.UnixListener) error {
-	stop := context.AfterFunc(ctx, func() { _ = ln.Close() })
+	// Serve may return before the callback runs, so it closes the listener
-	defer stop()
+	// too, and joins a callback already started before returning.
+	defer joinAfterFunc(ctx, func() { _ = ln.Close() })()
+	defer func() { _ = ln.Close() }()
 	var wg sync.WaitGroup
 	defer wg.Wait()
+	if s.AcceptStopped != nil {
+		defer s.AcceptStopped()
+	}
+	slots := make(chan struct{}, MaxAdmitted)
 	for {
+		select {
+		case slots <- struct{}{}:
+		case <-ctx.Done():
+			return nil
+		}
 		c, err := ln.AcceptUnix()
 		if err != nil {
+			<-slots
 			if ctx.Err() != nil {
 				return nil
 			}
@@ -145,6 +188,10 @@ func (s *ControlServer) Serve(ctx context.Context, ln *net.UnixListener) error {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
+			defer func() { <-slots }()
+			// Closing on cancellation unblocks a read or write in progress; the
+			// close is joined before the slot is released.
+			defer joinAfterFunc(ctx, func() { _ = c.Close() })()
 			s.serveConn(ctx, c)
 		}()
 	}
@@ -176,6 +223,10 @@ func (s *ControlServer) serveConn(ctx context.Context, c *net.UnixConn) {
 	_ = c.SetReadDeadline(time.Time{})
 
 	rep := s.Handle(ctx, req, cred)
+	// The raw payload is checked before the reply is encoded.
+	if len(rep.Payload) > MaxPayloadBytes {
+		rep = Reply{Outcome: OutcomeUnknown, Error: "result too large for the test path"}
+	}
 	body, err := json.Marshal(rep)
 	if err == nil && int64(len(body)) > s.ReplyLimit {
 		body, err = json.Marshal(Reply{Outcome: OutcomeUnknown, Error: "result too large for the test path"})
@@ -218,8 +269,11 @@ func (c *ControlClient) Exchange(ctx context.Context, req []byte, cred Credentia
 	if replyBy.IsZero() || !replyBy.After(time.Now()) {
 		return Reply{}, fmt.Errorf("%w: reply bound is zero or already past", ErrNotSent)
 	}
+	// The dial stops at replyBy even when ctx has no deadline.
+	dialCtx, cancelDial := context.WithDeadline(ctx, replyBy)
 	var d net.Dialer
-	conn, err := d.DialContext(ctx, "unix", "@"+c.Name)
+	conn, err := d.DialContext(dialCtx, "unix", "@"+c.Name)
+	cancelDial()
 	if err != nil {
 		return Reply{}, fmt.Errorf("%w: dial test-link control: %w", ErrNotSent, err)
 	}
@@ -269,6 +323,9 @@ func (c *ControlClient) Exchange(ctx context.Context, req []byte, cred Credentia
 	if err := json.Unmarshal(body, &rep); err != nil {
 		return Reply{}, errors.New("unreadable test-link reply")
 	}
+	if len(rep.Payload) > MaxPayloadBytes {
+		return Reply{}, fmt.Errorf("test-link reply payload of %d bytes exceeds %d", len(rep.Payload), MaxPayloadBytes)
+	}
 	return rep, nil
 }
 

spinifex/runtime/compute/lambda/testlink/control_bounds_test.go

@@ -0,0 +1,569 @@
+//go:build lambdatestlink
+
+//test:in-package — reads frames with the unexported readFrame and drives ControlServer limits directly
+package testlink
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"encoding/json"
+	"net"
+	"os"
+	"runtime"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
+
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
+)
+
+// fullServe runs a ControlServer at the declared bounds on ln and returns a
+// cancel that waits for Serve.
+func fullServe(t *testing.T, ln *net.UnixListener, handle func(context.Context, []byte, Credential) Reply) func() {
+	t.Helper()
+	srv := &ControlServer{GatewayUID: os.Getuid(), RequestLimit: MaxFrameBytes, ReplyLimit: MaxFrameBytes, IOTimeout: 60 * time.Second, Handle: handle}
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- srv.Serve(ctx, ln) }()
+	var once sync.Once
+	stop := func() {
+		once.Do(func() {
+			cancel()
+			select {
+			case err := <-done:
+				require.NoError(t, err, "Serve returned an error on shutdown")
+			case <-time.After(20 * time.Second):
+				t.Fatal("Serve did not return")
+			}
+		})
+	}
+	t.Cleanup(stop)
+	return stop
+}
+
+func fullListen(t *testing.T, name string) *net.UnixListener {
+	t.Helper()
+	ln, err := ListenControl(name)
+	require.NoError(t, err)
+	return ln
+}
+
+func fullClient(name string) *ControlClient {
+	return &ControlClient{Name: name, DaemonUID: os.Getuid(), RequestLimit: MaxFrameBytes, ReplyLimit: MaxFrameBytes, IOTimeout: 60 * time.Second}
+}
+
+// requestOfSize returns an encoded Request with a payload of payloadLen and
+// a FunctionARN padded so the frame is exactly size bytes.
+func requestOfSize(t *testing.T, payloadLen, size int) []byte {
+	t.Helper()
+	req := Request{AttemptID: "0123456789ab", Payload: bytes.Repeat([]byte{7}, payloadLen)}
+	b, err := json.Marshal(req)
+	require.NoError(t, err)
+	req.FunctionARN = "a"
+	b, err = json.Marshal(req)
+	require.NoError(t, err)
+	require.LessOrEqual(t, len(b), size)
+	req.FunctionARN = strings.Repeat("a", 1+size-len(b))
+	b, err = json.Marshal(req)
+	require.NoError(t, err)
+	require.Len(t, b, size)
+	return b
+}
+
+// replyOfSize returns a Reply with a payload of payloadLen whose encoding is
+// exactly size bytes.
+func replyOfSize(t *testing.T, payloadLen, size int) Reply {
+	t.Helper()
+	rep := Reply{Outcome: OutcomeDone}
+	if payloadLen > 0 {
+		rep.Payload = bytes.Repeat([]byte{9}, payloadLen)
+	}
+	b, err := json.Marshal(rep)
+	require.NoError(t, err)
+	rep.Error = "e"
+	b, err = json.Marshal(rep)
+	require.NoError(t, err)
+	require.LessOrEqual(t, len(b), size)
+	rep.Error = strings.Repeat("e", 1+size-len(b))
+	b, err = json.Marshal(rep)
+	require.NoError(t, err)
+	require.Len(t, b, size)
+	return rep
+}
+
+func recvBounded[T any](t *testing.T, ch chan T) T {
+	t.Helper()
+	select {
+	case v := <-ch:
+		return v
+	case <-time.After(30 * time.Second):
+		t.Fatal("no value within 30 s")
+		var zero T
+		return zero
+	}
+}
+
+func allocatedDuring(f func()) uint64 {
+	var before, after runtime.MemStats
+	runtime.GC()
+	runtime.ReadMemStats(&before)
+	f()
+	runtime.ReadMemStats(&after)
+	return after.TotalAlloc - before.TotalAlloc
+}
+
+// Test 5: a frame over MaxFrameBytes is rejected on its header, before the
+// receiver allocates its body.
+func TestReadFrame_OverBoundRejectedBeforeAllocation(t *testing.T) {
+	var hdr [4]byte
+	binary.BigEndian.PutUint32(hdr[:], uint32(MaxFrameBytes+1))
+	var err error
+	alloc := allocatedDuring(func() { _, err = readFrame(bytes.NewReader(hdr[:]), MaxFrameBytes) })
+	require.ErrorContains(t, err, "outside 1..")
+	require.Less(t, alloc, uint64(1<<20), "no body buffer was allocated")
+}
+
+// Test 5: a direct peer's frame with a small payload and metadata far over
+// 64 KiB, within MaxFrameBytes, reaches Handle: metadata has no separate bound.
+func TestControl_LargeMetadataWithinTheFrameIsAccepted(t *testing.T) {
+	name := controlName(t)
+	got := make(chan []byte, 1)
+	fullServe(t, fullListen(t, name), func(_ context.Context, req []byte, _ Credential) Reply {
+		got <- req
+		return Reply{Outcome: OutcomeDone}
+	})
+	req := requestOfSize(t, 16, 1<<20)
+	_, err := fullClient(name).Exchange(context.Background(), req, testCred, time.Now().Add(30*time.Second))
+	require.NoError(t, err)
+	require.Equal(t, req, recvBounded(t, got))
+}
+
+// Test 6a: a Request and a Reply of exactly MaxFrameBytes with a full 6 MiB
+// payload are both carried byte-identical.
+func TestControl_AtBoundRequestAndReplyBothFit(t *testing.T) {
+	name := controlName(t)
+	want := replyOfSize(t, MaxPayloadBytes, int(MaxFrameBytes))
+	got := make(chan []byte, 1)
+	fullServe(t, fullListen(t, name), func(_ context.Context, req []byte, _ Credential) Reply {
+		got <- req
+		return want
+	})
+	req := requestOfSize(t, MaxPayloadBytes, int(MaxFrameBytes))
+	rep, err := fullClient(name).Exchange(context.Background(), req, testCred, time.Now().Add(60*time.Second))
+	require.NoError(t, err)
+	require.Equal(t, req, recvBounded(t, got))
+	require.Equal(t, want, rep)
+}
+
+// Test 6a: an at-bound reply with no payload, whose payload field the encoder
+// omits, still fits.
+func TestControl_AtBoundReplyWithoutPayloadFits(t *testing.T) {
+	name := controlName(t)
+	want := replyOfSize(t, 0, int(MaxFrameBytes))
+	fullServe(t, fullListen(t, name), func(context.Context, []byte, Credential) Reply { return want })
+	rep, err := fullClient(name).Exchange(context.Background(), []byte(`{"attempt_id":"a"}`), testCred, time.Now().Add(60*time.Second))
+	require.NoError(t, err)
+	require.Equal(t, want, rep)
+}
+
+// Test 6b: the real server answers a handler's oversized payload, and a reply
+// encoding over MaxFrameBytes, with the small unknown reply.
+func TestControl_ServerReplacesOversizedRepliesWithUnknown(t *testing.T) {
+	over := Reply{Outcome: OutcomeDone, Payload: make([]byte, MaxPayloadBytes+1)}
+	for name, rep := range map[string]Reply{
+		"payload over 6 MiB":        over,
+		"encoding over frame bound": replyOfSize(t, MaxPayloadBytes, int(MaxFrameBytes)+1),
+	} {
+		t.Run(name, func(t *testing.T) {
+			sock := controlName(t)
+			fullServe(t, fullListen(t, sock), func(context.Context, []byte, Credential) Reply { return rep })
+			got, err := fullClient(sock).Exchange(context.Background(), []byte(`{"attempt_id":"a"}`), testCred, time.Now().Add(60*time.Second))
+			require.NoError(t, err)
+			require.Equal(t, Reply{Outcome: OutcomeUnknown, Error: "result too large for the test path"}, got)
+		})
+	}
+	// The oversized payload is refused before the reply is encoded: one
+	// exchange allocates far less than an 8 MiB base64 encoding.
+	sock := controlName(t)
+	fullServe(t, fullListen(t, sock), func(context.Context, []byte, Credential) Reply { return over })
+	var err error
+	alloc := allocatedDuring(func() {
+		_, err = fullClient(sock).Exchange(context.Background(), []byte(`{"attempt_id":"a"}`), testCred, time.Now().Add(60*time.Second))
+	})
+	require.NoError(t, err)
+	require.Less(t, alloc, uint64(4<<20), "the oversized reply was encoded before the payload check")
+}
+
+// directPeer serves one connection by reading the request and credential,
+// then writing with body; the peer's own error is returned on the channel.
+func directPeer(t *testing.T, name string, body func(net.Conn) error) chan error {
+	t.Helper()
+	ln := fullListen(t, name)
+	t.Cleanup(func() { _ = ln.Close() })
+	peerErr := make(chan error, 1)
+	go func() {
+		peerErr <- func() error {
+			c, err := ln.AcceptUnix()
+			if err != nil {
+				return err
+			}
+			defer func() { _ = c.Close() }()
+			if _, err := readFrame(c, MaxFrameBytes); err != nil {
+				return err
+			}
+			if _, err := readFrame(c, MaxCredentialBytes); err != nil {
+				return err
+			}
+			return body(c)
+		}()
+	}()
+	return peerErr
+}
+
+// Test 6: replies from a direct peer daemon over the payload bound or the
+// frame bound are errors after sending, so the gateway reports unknown.
+func TestControl_ClientRejectsOversizedReplies(t *testing.T) {
+	overPayload, err := json.Marshal(Reply{Outcome: OutcomeDone, Payload: make([]byte, MaxPayloadBytes+1)})
+	require.NoError(t, err)
+	require.LessOrEqual(t, int64(len(overPayload)), MaxFrameBytes)
+	cases := map[string]struct {
+		send func(net.Conn) error
+		want string
+	}{
+		"payload over 6 MiB": {func(c net.Conn) error { return writeFrame(c, overPayload) }, "reply payload of 6291457 bytes exceeds 6291456"},
+		"frame over bound": {func(c net.Conn) error {
+			var hdr [4]byte
+			binary.BigEndian.PutUint32(hdr[:], uint32(MaxFrameBytes+1))
+			_, err := c.Write(hdr[:])
+			return err
+		}, "outside 1.."},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			sock := controlName(t)
+			peerErr := directPeer(t, sock, tc.send)
+			_, err := fullClient(sock).Exchange(context.Background(), []byte(`{"attempt_id":"a"}`), testCred, time.Now().Add(30*time.Second))
+			require.NoError(t, recvBounded(t, peerErr), "the peer read the request and wrote its reply")
+			require.ErrorContains(t, err, tc.want)
+			require.NotErrorIs(t, err, ErrNotSent)
+		})
+	}
+}
+
+// Test 12: the credential bound is unchanged beside a full-size request.
+func TestControl_CredentialOverBoundRefusedWithAFullRequest(t *testing.T) {
+	cred := testCred
+	b, err := json.Marshal(cred)
+	require.NoError(t, err)
+	cred.SessionToken += strings.Repeat("t", MaxCredentialBytes+1-len(b))
+	b, err = json.Marshal(cred)
+	require.NoError(t, err)
+	require.Len(t, b, MaxCredentialBytes+1)
+
+	name := controlName(t)
+	s, _ := serveControl(t, name, os.Getuid(), nil)
+	_, err = fullClient(name).Exchange(context.Background(), requestOfSize(t, MaxPayloadBytes, int(MaxFrameBytes)), cred, time.Now().Add(30*time.Second))
+	require.ErrorIs(t, err, ErrNotSent)
+	require.Zero(t, s.calls.Load())
+}
+
+// pausedHandlers counts concurrent Handle calls and blocks each until released.
+type pausedHandlers struct {
+	in, max atomic.Int32
+	calls   atomic.Int32
+	entered chan []byte
+	release chan struct{}
+	once    sync.Once
+}
+
+func newPaused(t *testing.T) *pausedHandlers {
+	p := &pausedHandlers{entered: make(chan []byte, 64), release: make(chan struct{})}
+	t.Cleanup(p.free)
+	return p
+}
+
+func (p *pausedHandlers) free() { p.once.Do(func() { close(p.release) }) }
+
+func (p *pausedHandlers) handle(ctx context.Context, req []byte, _ Credential) Reply {
+	n := p.in.Add(1)
+	for {
+		m := p.max.Load()
+		if n <= m || p.max.CompareAndSwap(m, n) {
+			break
+		}
+	}
+	p.calls.Add(1)
+	p.entered <- req
+	select {
+	case <-p.release:
+	case <-ctx.Done():
+	}
+	p.in.Add(-1)
+	return Reply{Outcome: OutcomeDone}
+}
+
+func (p *pausedHandlers) waitEntered(t *testing.T, n int) {
+	t.Helper()
+	for range n {
+		select {
+		case <-p.entered:
+		case <-time.After(30 * time.Second):
+			t.Fatal("handler not entered")
+		}
+	}
+}
+
+// serverConns reads the kernel's socket table for the abstract name: server
+// ends Accept returned are connected and held by this process; a connection
+// still in the listen backlog is connecting, with no inode until accepted.
+func serverConns(t *testing.T, name string) (accepted, backlog int) {
+	t.Helper()
+	held := map[string]bool{}
+	fds, err := os.ReadDir("/proc/self/fd")
+	require.NoError(t, err)
+	for _, fd := range fds {
+		if target, err := os.Readlink("/proc/self/fd/" + fd.Name()); err == nil && strings.HasPrefix(target, "socket:[") {
+			held[strings.TrimSuffix(strings.TrimPrefix(target, "socket:["), "]")] = true
+		}
+	}
+	table, err := os.ReadFile("/proc/net/unix")
+	require.NoError(t, err)
+	for _, line := range strings.Split(string(table), "\n")[1:] {
+		f := strings.Fields(line)
+		// Num RefCount Protocol Flags Type St Inode Path.
+		if len(f) != 8 || f[7] != "@"+name {
+			continue
+		}
+		switch {
+		case f[5] == "03" && held[f[6]]:
+			accepted++
+		case f[5] == "02" && f[6] == "0":
+			backlog++
+		}
+	}
+	return accepted, backlog
+}
+
+// Test 8: eight full-size requests at once; with two admitted connections
+// paused in Handle, the kernel shows two accepted and six still in the
+// backlog, and at most two Handle calls ever run at once.
+func TestControl_AdmissionHoldsAtMostTwoFullSizeRequests(t *testing.T) {
+	name := controlName(t)
+	p := newPaused(t)
+	fullServe(t, fullListen(t, name), p.handle)
+	req := requestOfSize(t, MaxPayloadBytes, int(MaxFrameBytes))
+
+	var wg sync.WaitGroup
+	errs := make(chan error, 8)
+	for range 8 {
+		wg.Go(func() {
+			_, err := fullClient(name).Exchange(context.Background(), req, testCred, time.Now().Add(90*time.Second))
+			errs <- err
+		})
+	}
+	p.waitEntered(t, MaxAdmitted)
+	deadline := time.Now().Add(30 * time.Second)
+	for {
+		accepted, backlog := serverConns(t, name)
+		require.LessOrEqual(t, accepted, MaxAdmitted, "a third connection was accepted")
+		if backlog == 8-MaxAdmitted {
+			break
+		}
+		require.True(t, time.Now().Before(deadline), "the other clients never reached the backlog: %d accepted, %d waiting", accepted, backlog)
+		runtime.Gosched()
+	}
+	for range 20 {
+		accepted, backlog := serverConns(t, name)
+		require.Equal(t, MaxAdmitted, accepted, "only the admitted connections are accepted")
+		require.Equal(t, 8-MaxAdmitted, backlog, "the other six stay unaccepted")
+		<-time.After(50 * time.Millisecond)
+	}
+	require.Equal(t, int32(MaxAdmitted), p.calls.Load())
+
+	p.free()
+	waitGroupBounded(t, &wg, 90*time.Second)
+	close(errs)
+	for err := range errs {
+		require.NoError(t, err)
+	}
+	require.Equal(t, int32(8), p.calls.Load())
+	require.LessOrEqual(t, p.max.Load(), int32(MaxAdmitted))
+}
+
+// Test 9: a gateway that wrote its whole request while unaccepted, then gave
+// up and closed, reports unknown; the daemon later admits it and Handle runs.
+func TestControl_GatewayGivesUpButBufferedRequestIsStillHandled(t *testing.T) {
+	name := controlName(t)
+	p := newPaused(t)
+	fullServe(t, fullListen(t, name), p.handle)
+	for range MaxAdmitted {
+		go func() {
+			_, _ = fullClient(name).Exchange(context.Background(), []byte(`{"attempt_id":"held"}`), testCred, time.Now().Add(60*time.Second))
+		}()
+	}
+	p.waitEntered(t, MaxAdmitted)
+
+	late := []byte(`{"attempt_id":"late"}`)
+	c := fullClient(name)
+	c.IOTimeout = 2 * time.Second
+	_, err := c.Exchange(context.Background(), late, testCred, time.Now().Add(2*time.Second))
+	require.Error(t, err)
+	require.NotErrorIs(t, err, ErrNotSent, "the request was written, so the outcome is unknown")
+
+	p.free()
+	deadline := time.After(30 * time.Second)
+	for {
+		select {
+		case req := <-p.entered:
+			if bytes.Equal(req, late) {
+				return // Handle ran for a request whose gateway reported unknown.
+			}
+		case <-deadline:
+			t.Fatal("the buffered request was never handled")
+		}
+	}
+}
+
+// smallBacklogListener binds name with a listen backlog of one.
+func smallBacklogListener(t *testing.T, name string) *net.UnixListener {
+	t.Helper()
+	fd, err := unix.Socket(unix.AF_UNIX, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0)
+	require.NoError(t, err)
+	require.NoError(t, unix.Bind(fd, &unix.SockaddrUnix{Name: "@" + name}))
+	require.NoError(t, unix.Listen(fd, 1))
+	f := os.NewFile(uintptr(fd), name)
+	defer func() { _ = f.Close() }()
+	l, err := net.FileListener(f)
+	require.NoError(t, err)
+	return l.(*net.UnixListener)
+}
+
+// Test 10: with both slots held and the backlog full, a gateway connect made
+// with no context deadline fails or stops at replyBy as ErrNotSent, and no
+// request reaches Handle.
+func TestControl_FullBacklogConnectIsNotSent(t *testing.T) {
+	name := controlName(t)
+	p := newPaused(t)
+	fullServe(t, smallBacklogListener(t, name), p.handle)
+	for range MaxAdmitted {
+		go func() {
+			_, _ = fullClient(name).Exchange(context.Background(), []byte(`{"attempt_id":"held"}`), testCred, time.Now().Add(60*time.Second))
+		}()
+	}
+	p.waitEntered(t, MaxAdmitted)
+
+	// Fill the backlog with raw connections that never write.
+	var fillers []net.Conn
+	t.Cleanup(func() {
+		for _, c := range fillers {
+			_ = c.Close()
+		}
+	})
+	for range 64 {
+		d := net.Dialer{Timeout: 200 * time.Millisecond}
+		c, err := d.Dial("unix", "@"+name)
+		if err != nil {
+			break
+		}
+		fillers = append(fillers, c)
+	}
+	require.Less(t, len(fillers), 64, "the backlog never filled")
+
+	began := time.Now()
+	_, err := fullClient(name).Exchange(context.Background(), []byte(`{"attempt_id":"x"}`), testCred, time.Now().Add(2*time.Second))
+	require.ErrorIs(t, err, ErrNotSent)
+	require.Less(t, time.Since(began), 10*time.Second)
+	require.Equal(t, int32(MaxAdmitted), p.calls.Load(), "no further request reached Handle")
+}
+
+// Test 15 (measurement, not a bound): two admitted full-size lifetimes, each
+// decoding and fit-checking its request as the daemon does and answering with
+// a 6 MiB reply, measured from a post-GC baseline just before the first dial
+// until both replies are read.
+func TestControl_MeasureTwoAdmittedFullSizeLifetimes(t *testing.T) {
+	name := controlName(t)
+	var mu sync.Mutex
+	entered := 0
+	both := make(chan struct{})
+	fitErrs := make(chan error, MaxAdmitted)
+	fullServe(t, fullListen(t, name), func(ctx context.Context, b []byte, cred Credential) Reply {
+		var req Request
+		if err := json.Unmarshal(b, &req); err != nil {
+			return Reply{Outcome: OutcomeRefused, Error: err.Error()}
+		}
+		req.cred = cred
+		fitErrs <- hostlink.CheckInvokeFits(guestRequest(req, 0))
+		mu.Lock()
+		entered++
+		if entered == MaxAdmitted {
+			close(both)
+		}
+		mu.Unlock()
+		select {
+		case <-both:
+		case <-ctx.Done():
+			return Reply{Outcome: OutcomeUnknown}
+		}
+		return Reply{Outcome: OutcomeDone, Payload: bytes.Repeat([]byte{9}, MaxPayloadBytes)}
+	})
+	req := requestOfSize(t, MaxPayloadBytes, int(MaxFrameBytes))
+
+	var wg sync.WaitGroup
+	var reps [MaxAdmitted]Reply
+	var errs [MaxAdmitted]error
+	var before, after runtime.MemStats
+	runtime.GC()
+	runtime.ReadMemStats(&before)
+
+	// HeapAlloc is sampled every 2 ms from the baseline until both replies are
+	// read, so its maximum is a sampled process-wide figure, not a true peak.
+	var heapMax uint64
+	samples := 0
+	stopSampling, sampled := make(chan struct{}), make(chan struct{})
+	go func() {
+		defer close(sampled)
+		tick := time.NewTicker(2 * time.Millisecond)
+		defer tick.Stop()
+		for {
+			var m runtime.MemStats
+			runtime.ReadMemStats(&m)
+			heapMax, samples = max(heapMax, m.HeapAlloc), samples+1
+			select {
+			case <-tick.C:
+			case <-stopSampling:
+				return
+			}
+		}
+	}()
+	var stopOnce sync.Once
+	stop := func() {
+		stopOnce.Do(func() { close(stopSampling) })
+		<-sampled
+	}
+	t.Cleanup(stop)
+
+	for i := range MaxAdmitted {
+		wg.Go(func() {
+			reps[i], errs[i] = fullClient(name).Exchange(context.Background(), req, testCred, time.Now().Add(60*time.Second))
+		})
+	}
+	waitGroupBounded(t, &wg, 60*time.Second)
+	stop()
+	runtime.ReadMemStats(&after)
+
+	for i := range MaxAdmitted {
+		require.NoError(t, errs[i])
+		require.Equal(t, OutcomeDone, reps[i].Outcome)
+		require.Len(t, reps[i].Payload, MaxPayloadBytes, "each lifetime ends with its full-size reply read")
+		require.NoError(t, recvBounded(t, fitErrs), "the full-size request fits the guest link")
+	}
+	// Excludes the executor's marshaling, HostSend, the guest and QEMU.
+	t.Logf("two admitted full-size lifetimes, process-wide (client and server): %d bytes in %d allocations; sampled max HeapAlloc %d bytes over %d samples",
+		after.TotalAlloc-before.TotalAlloc, after.Mallocs-before.Mallocs, heapMax, samples)
+}

spinifex/runtime/compute/lambda/testlink/control_daemon_test.go

@@ -0,0 +1,96 @@
+//go:build lambdatestlink
+
+//test:in-package — reads the daemon's unexported slot and owner fixtures
+package testlink
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/execution"
+)
+
+// A direct peer's request whose raw payload is 6 MiB + 1 but whose frame is
+// within MaxFrameBytes passes the frame check, reaches the real daemon, and
+// is refused there before Start with no address or tap taken.
+func TestControlDaemon_RawPayloadOverBoundRefusedBeforeStart(t *testing.T) {
+	code := []byte("zip bytes")
+	srv := codeServer(t, code)
+	g := grant(t, code)
+	ips, taps := &fakeIPs{}, &fakeTaps{}
+	d := g.daemon(&Daemon{IPs: ips, Taps: taps, HTTP: srv.Client(), WorkDir: t.TempDir()})
+
+	req := baseRequest(srv.URL, code)
+	req.Grant, req.ImageDigest = g.att.Grant, g.image.Digest
+	req.Payload = make([]byte, MaxPayloadBytes+1)
+	b, err := json.Marshal(req)
+	require.NoError(t, err)
+	require.LessOrEqual(t, int64(len(b)), MaxFrameBytes, "the frame alone would admit it")
+
+	name := controlName(t)
+	fullServe(t, fullListen(t, name), d.Handle)
+	rep, err := fullClient(name).Exchange(context.Background(), b, testCred, time.Now().Add(30*time.Second))
+	require.NoError(t, err)
+	require.Equal(t, OutcomeRefused, rep.Outcome, rep.Error)
+	require.Contains(t, rep.Error, "payload")
+	require.Empty(t, ips.allocated)
+	require.Empty(t, taps.created)
+	require.Equal(t, execution.StateGranted, g.state(t))
+}
+
+// With one real attempt started and held in its code fetch, a second request
+// through the real control server is refused before Start as busy.
+func TestControlDaemon_SecondRequestRefusedWhileAnAttemptIsHeld(t *testing.T) {
+	code := []byte("zip bytes")
+	fetching, release := make(chan struct{}), make(chan struct{})
+	var fetchOnce, releaseOnce sync.Once
+	free := func() { releaseOnce.Do(func() { close(release) }) }
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fetchOnce.Do(func() { close(fetching) })
+		select {
+		case <-release:
+		case <-r.Context().Done():
+			return
+		}
+		_, _ = w.Write(code)
+	}))
+	// Cleanups run last-registered first: free the held fetch before Close.
+	t.Cleanup(srv.Close)
+	t.Cleanup(free)
+	g := grant(t, code)
+	ips, taps := &fakeIPs{}, &fakeTaps{}
+	d := g.daemon(&Daemon{IPs: ips, Taps: taps, HTTP: srv.Client(), WorkDir: t.TempDir()})
+	name := controlName(t)
+	fullServe(t, fullListen(t, name), d.Handle)
+
+	first := make(chan error, 1)
+	go func() {
+		_, err := fullClient(name).Exchange(context.Background(), g.request(srv.URL, code), testCred, time.Now().Add(60*time.Second))
+		first <- err
+	}()
+	// On every path the held attempt is released and its exchange joined.
+	t.Cleanup(func() {
+		free()
+		select {
+		case <-first:
+		case <-time.After(30 * time.Second):
+			t.Error("the held exchange did not return")
+		}
+	})
+	recvBounded(t, fetching)
+	require.Equal(t, execution.StateRunning, g.state(t), "the first attempt started")
+
+	rep, err := fullClient(name).Exchange(context.Background(), g.request(srv.URL, code), testCred, time.Now().Add(30*time.Second))
+	require.NoError(t, err)
+	require.Equal(t, OutcomeRefused, rep.Outcome)
+	require.Contains(t, rep.Error, "another")
+	require.Empty(t, ips.allocated, "nothing was allocated for either attempt yet")
+	require.Empty(t, taps.created)
+}