fix(ci): give the nightly's single-node service suites node SSH to their own VM (#1653)

* fix(eks): alarm on a control-plane VM with no cluster record instead of terminating it

* chore(eks): order test imports

* feat(eks): add spx admin eks orphans and share the cluster-record judgement with the billable reaper

* fix(eks): list orphan control planes through an EC2-owned inventory capability

* test(eks): cover the orphan report ENI reader and the inventory fail-closed path

* fix(eks): report only running control planes and treat a missing ENI as gone

* fix(eks): open DNS watch buckets attach-only so a deleted account bucket is not recreated

* fix(eks): read control-plane ENIs from their record so a corrupt one is unreadable, and report the zone

.github/workflows/e2e-nightly.yml

@@ -578,6 +578,8 @@ jobs:
              echo \"::group::suite ${SUITE}\"; \
              date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-${SUITE}.start'; \
              GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+               SPINIFEX_NODES='${{ steps.vm.outputs.wan_ip }}' SPINIFEX_SSH_USER=tf-user \
+               SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
                ARTIFACT_DIR='${ARTIFACT_DIR}' \
                ./${SUITE}.test -test.v -test.timeout=${TMO} \
                2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log'; \

cmd/lambda-agent/sdk_config_testlink.go

@@ -0,0 +1,65 @@
+//go:build lambdatestlink
+
+package main
+
+import (
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"net"
+	"net/url"
+	"os"
+	"path/filepath"
+)
+
+const (
+	sdkEndpointEntry = "test-sdk-endpoint"
+	sdkCAEntry       = "test-sdk-ca"
+	sdkBundlePath    = "/run/lambda-sdk/ca-bundle.pem"
+	systemBundlePath = "/etc/ssl/certs/ca-certificates.crt"
+)
+
+type sdkConfig struct {
+	endpoint string
+	ca       []byte
+}
+
+// readSDKConfig reads optional public settings before the boot config is sealed.
+func readSDKConfig(dir string) (*sdkConfig, error) {
+	endpoint, endpointErr := os.ReadFile(filepath.Join(dir, sdkEndpointEntry, "raw"))
+	ca, caErr := os.ReadFile(filepath.Join(dir, sdkCAEntry, "raw"))
+	if errors.Is(endpointErr, os.ErrNotExist) && errors.Is(caErr, os.ErrNotExist) {
+		return nil, nil
+	}
+	if endpointErr != nil || caErr != nil {
+		return nil, errors.New("incomplete SDK gateway settings")
+	}
+	u, err := url.Parse(string(endpoint))
+	if err != nil || u.Scheme != "https" || u.User != nil || u.Path != "" || u.RawQuery != "" || u.Fragment != "" ||
+		net.ParseIP(u.Hostname()) == nil || u.Port() == "" {
+		return nil, errors.New("invalid SDK gateway endpoint")
+	}
+	if len(ca) > 64<<10 || !x509.NewCertPool().AppendCertsFromPEM(ca) {
+		return nil, errors.New("invalid SDK gateway CA")
+	}
+	return &sdkConfig{endpoint: string(endpoint), ca: ca}, nil
+}
+
+// writeBundle combines the system roots with the cluster's public root.
+func (c *sdkConfig) writeBundle(path, systemPath string) error {
+	system, err := os.ReadFile(systemPath)
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("read system CA bundle: %w", err)
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		return fmt.Errorf("create SDK CA directory: %w", err)
+	}
+	if err := os.Chmod(filepath.Dir(path), 0o755); err != nil {
+		return fmt.Errorf("make SDK CA directory readable: %w", err)
+	}
+	bundle := append(append(system, '\n'), c.ca...)
+	if err := os.WriteFile(path, bundle, 0o644); err != nil {
+		return fmt.Errorf("write SDK CA bundle: %w", err)
+	}
+	return os.Chmod(path, 0o644)
+}

cmd/lambda-agent/sdk_config_testlink_test.go

@@ -0,0 +1,106 @@
+//go:build lambdatestlink
+
+//test:in-package exercises the unexported boot configuration reader
+package main
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSDKConfig_OptionalAndInvalid(t *testing.T) {
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
+	defer server.Close()
+	ca := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw})
+	for _, tc := range []struct {
+		name     string
+		endpoint []byte
+		ca       []byte
+		wantErr  bool
+		want     bool
+	}{
+		{name: "absent"},
+		{name: "valid", endpoint: []byte(server.URL), ca: ca, want: true},
+		{name: "missing ca", endpoint: []byte(server.URL), wantErr: true},
+		{name: "missing endpoint", ca: ca, wantErr: true},
+		{name: "invalid endpoint", endpoint: []byte("http://127.0.0.1:1234"), ca: ca, wantErr: true},
+		{name: "invalid ca", endpoint: []byte(server.URL), ca: []byte("not a certificate"), wantErr: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			dir := t.TempDir()
+			for name, value := range map[string][]byte{sdkEndpointEntry: tc.endpoint, sdkCAEntry: tc.ca} {
+				if value != nil {
+					require.NoError(t, os.MkdirAll(filepath.Join(dir, name), 0o700))
+					require.NoError(t, os.WriteFile(filepath.Join(dir, name, "raw"), value, 0o600))
+				}
+			}
+			cfg, err := readSDKConfig(dir)
+			if tc.wantErr {
+				require.Error(t, err)
+				require.Nil(t, cfg)
+				return
+			}
+			require.NoError(t, err)
+			if !tc.want {
+				require.Nil(t, cfg)
+				return
+			}
+			bundle := filepath.Join(t.TempDir(), "sdk", "ca.pem")
+			require.NoError(t, cfg.writeBundle(bundle, filepath.Join(t.TempDir(), "absent")))
+			data, err := os.ReadFile(bundle)
+			require.NoError(t, err)
+			roots := x509.NewCertPool()
+			require.True(t, roots.AppendCertsFromPEM(data))
+			client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: roots}}}
+			resp, err := client.Get(cfg.endpoint)
+			require.NoError(t, err)
+			defer resp.Body.Close()
+			assert.Equal(t, http.StatusOK, resp.StatusCode)
+		})
+	}
+}
+
+func TestSDKConfig_BundleKeepsSystemRoots(t *testing.T) {
+	cluster := httptest.NewTLSServer(nil)
+	defer cluster.Close()
+	clusterCA := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cluster.Certificate().Raw})
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+	template := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test system root"},
+		NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour),
+		IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign}
+	der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
+	require.NoError(t, err)
+	systemCA := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+	dir := t.TempDir()
+	systemPath, bundlePath := filepath.Join(dir, "system.pem"), filepath.Join(dir, "sdk", "ca.pem")
+	require.NoError(t, os.WriteFile(systemPath, systemCA, 0o600))
+	require.NoError(t, (&sdkConfig{ca: clusterCA}).writeBundle(bundlePath, systemPath))
+	bundle, err := os.ReadFile(bundlePath)
+	require.NoError(t, err)
+	roots := x509.NewCertPool()
+	require.True(t, roots.AppendCertsFromPEM(bundle))
+	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: roots}}}
+	resp, err := client.Get(cluster.URL)
+	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
+	systemCert, err := x509.ParseCertificate(der)
+	require.NoError(t, err)
+	_, err = systemCert.Verify(x509.VerifyOptions{Roots: roots})
+	require.NoError(t, err)
+}

cmd/lambda-agent/start_testlink.go

@@ -12,6 +12,7 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
+	"strings"
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/agent"
@@ -31,13 +32,17 @@ type testlink struct {
 	pkg            taskPackage
 	runtimeAddr    string
 	startBootstrap func(ctx context.Context, env []string) (*exec.Cmd, error)
+	caBundlePath   string
+	systemCAPath   string
 }
 
 func start(ctx context.Context) int {
 	t := testlink{
-		startup:     defaultStartup(),
-		pkg:         devTaskPackage(),
-		runtimeAddr: runtimeAddr,
+		startup:      defaultStartup(),
+		pkg:          devTaskPackage(),
+		runtimeAddr:  runtimeAddr,
+		caBundlePath: sdkBundlePath,
+		systemCAPath: systemBundlePath,
 		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
 			return agent.StartBootstrap(ctx, taskRoot+"/bootstrap", env, os.Stdout, os.Stderr)
 		},
@@ -53,6 +58,7 @@ func (t testlink) run(parent context.Context) (int, error) {
 	// The package values are fw_cfg entries, so read them before connect seals it.
 	spec, specErr := readPackageSpec(t.fwCfgDir)
 	grant, grantErr := readGrant(t.fwCfgDir)
+	sdk, sdkErr := readSDKConfig(t.fwCfgDir)
 	conn, scope, code, err := t.connect(parent)
 	if err != nil {
 		return code, errors.Join(err, specErr, grantErr)
@@ -85,6 +91,15 @@ func (t testlink) run(parent context.Context) (int, error) {
 	}
 	cred := *first.Credential
 	first.Credential = nil
+	if sdkErr != nil {
+		slog.Warn("lambda-agent: SDK gateway settings unavailable", "err", sdkErr)
+	}
+	if sdk != nil {
+		if err := sdk.writeBundle(t.caBundlePath, t.systemCAPath); err != nil {
+			slog.Warn("lambda-agent: SDK gateway settings unavailable", "err", err)
+			sdk = nil
+		}
+	}
 	if err := t.pkg.install(spec); err != nil {
 		return exitRefused, fmt.Errorf("install package: %w", err)
 	}
@@ -100,7 +115,8 @@ func (t testlink) run(parent context.Context) (int, error) {
 
 	ctx, cancel := context.WithCancelCause(parent)
 	defer cancel(nil)
-	cmd, err := t.startBootstrap(ctx, bootstrapEnv(ln.Addr().String(), spec.handler, cred))
+	region, _, _ := strings.Cut(string(scope), "/")
+	cmd, err := t.startBootstrap(ctx, bootstrapEnv(ln.Addr().String(), spec.handler, cred, region, sdk, t.caBundlePath))
 	if err != nil {
 		return exitLink, fmt.Errorf("start bootstrap: %w", err)
 	}
@@ -124,10 +140,9 @@ func (t testlink) run(parent context.Context) (int, error) {
 	return exitClosed, nil
 }
 
-// bootstrapEnv is the bootstrap's whole environment: the runtime variables and
-// the execution role's three credential variables.
-func bootstrapEnv(runtimeAPI, handler string, cred hostlink.Credential) []string {
-	return []string{
+// bootstrapEnv supplies the runtime, execution-role and SDK settings.
+func bootstrapEnv(runtimeAPI, handler string, cred hostlink.Credential, region string, sdk *sdkConfig, caPath string) []string {
+	env := []string{
 		"AWS_LAMBDA_RUNTIME_API=" + runtimeAPI,
 		"_HANDLER=" + handler,
 		"LAMBDA_TASK_ROOT=" + taskRoot,
@@ -135,7 +150,13 @@ func bootstrapEnv(runtimeAPI, handler string, cred hostlink.Credential) []string
 		"AWS_ACCESS_KEY_ID=" + cred.AccessKeyID,
 		"AWS_SECRET_ACCESS_KEY=" + cred.SecretAccessKey,
 		"AWS_SESSION_TOKEN=" + cred.SessionToken,
+		"AWS_REGION=" + region,
+		"AWS_DEFAULT_REGION=" + region,
 	}
+	if sdk != nil {
+		env = append(env, "AWS_ENDPOINT_URL="+sdk.endpoint, "AWS_CA_BUNDLE="+caPath)
+	}
+	return env
 }
 
 // shutdownGrace is how long past the invocation deadline the agent waits for

cmd/lambda-agent/start_testlink_test.go

@@ -7,8 +7,10 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
+	"encoding/pem"
 	"io"
 	"net/http"
+	"net/http/httptest"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -70,6 +72,7 @@ type bootstrapStarts struct {
 	sealed  bool
 	// handlerOut receives the _HANDLER the bootstrap process itself saw.
 	handlerOut string
+	caBundle   string
 	// env is the environment the agent gave the bootstrap.
 	env []string
 }
@@ -94,6 +97,7 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 	b.started = make(chan struct{}, 1)
 	b.task = t.TempDir()
 	b.handlerOut = filepath.Join(t.TempDir(), "handler")
+	b.caBundle = filepath.Join(t.TempDir(), "sdk", "ca-bundle.pem")
 	return testlink{
 		startup: g.startup(),
 		pkg: taskPackage{
@@ -102,7 +106,9 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 			mountDir:   func(string) error { return nil },
 			sealDir:    func(string) error { b.sealed = true; return nil },
 		},
-		runtimeAddr: "127.0.0.1:0",
+		runtimeAddr:  "127.0.0.1:0",
+		caBundlePath: b.caBundle,
+		systemCAPath: filepath.Join(t.TempDir(), "system-ca.pem"),
 		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
 			b.n++
 			b.env = env
@@ -176,10 +182,45 @@ func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
 		"AWS_ACCESS_KEY_ID=" + testCredential.AccessKeyID,
 		"AWS_SECRET_ACCESS_KEY=" + testCredential.SecretAccessKey,
 		"AWS_SESSION_TOKEN=" + testCredential.SessionToken,
+		"AWS_REGION=ap-southeast-2",
+		"AWS_DEFAULT_REGION=ap-southeast-2",
 	}, b.env, "the bootstrap gets exactly the runtime and credential variables")
 	assert.NotEmpty(t, envValue(b.env, "AWS_LAMBDA_RUNTIME_API"))
 }
 
+func TestTestlinkProvidesSDKSettingsToBootstrap(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
+	defer server.Close()
+	h := newHost(t)
+	h.accept(ctx)
+	entries := h.entries()
+	entries[sdkEndpointEntry] = []byte(server.URL)
+	entries[sdkCAEntry] = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw})
+	var b bootstrapStarts
+	done := runTestlinkAsync(ctx, b.testlink(t, entries))
+	conn := h.guest(t)
+	defer conn.Close()
+	hs := hostlink.NewTestStream(conn)
+	req := hostlink.Request{Credential: &testCredential, GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: []byte(`{}`)}
+	require.NoError(t, hs.HostSend(req))
+	_, err := hs.HostResult(ctx)
+	require.NoError(t, err)
+	res := wait(t, done, "agent to stop")
+	require.NoError(t, res.err)
+	assert.Equal(t, server.URL, envValue(b.env, "AWS_ENDPOINT_URL"))
+	assert.Equal(t, b.caBundle, envValue(b.env, "AWS_CA_BUNDLE"))
+	assert.Equal(t, "ap-southeast-2", envValue(b.env, "AWS_REGION"))
+	assert.Equal(t, "ap-southeast-2", envValue(b.env, "AWS_DEFAULT_REGION"))
+	ca, err := os.ReadFile(b.caBundle)
+	require.NoError(t, err)
+	assert.Contains(t, string(ca), string(entries[sdkCAEntry]))
+	info, err := os.Stat(b.caBundle)
+	require.NoError(t, err)
+	assert.Equal(t, os.FileMode(0o644), info.Mode().Perm())
+}
+
 var testCredential = hostlink.Credential{AccessKeyID: "ASIAGUESTKEY", SecretAccessKey: "guest-secret", SessionToken: "guest-token"}
 
 func envValue(env []string, name string) string {

spinifex/domains/ec2/inventory/inventory.go

@@ -0,0 +1,67 @@
+// Package inventory is the EC2-owned read capability for an account's complete
+// instance list, for callers outside the EC2 domain.
+package inventory
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	ec2instanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/instance"
+	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// Instances lists every non-terminated instance an account owns, or returns an
+// error. It never returns a shorter list than the truth.
+type Instances interface {
+	ListAccountInstances(ctx context.Context, accountID string) (*ec2.DescribeInstancesOutput, error)
+}
+
+// NATSInstances reads the complete list across nodeIDs over NATS.
+type NATSInstances struct {
+	nc      *nats.Conn
+	nodeIDs []string
+}
+
+var _ Instances = (*NATSInstances)(nil)
+
+// NewNATSInstances returns a reader over the configured node set.
+func NewNATSInstances(nc *nats.Conn, nodeIDs []string) *NATSInstances {
+	return &NATSInstances{nc: nc, nodeIDs: nodeIDs}
+}
+
+// ListAccountInstances passes through the strict listing unchanged: any node
+// that did not answer, or any record that could not be read, is an error.
+func (n *NATSInstances) ListAccountInstances(ctx context.Context, accountID string) (*ec2.DescribeInstancesOutput, error) {
+	return ec2instanceapi.DescribeAccountInstancesStrict(ctx, n.nc, n.nodeIDs, accountID)
+}
+
+// NewENITagReader returns a reader of one ENI's tags straight from its stored
+// record, so an ENI that is absent and one that cannot be read are told apart:
+// a missing key is found=false, a failed read or an undecodable record is an
+// error. accountID is the account the ENI is stored under.
+func NewENITagReader(js jetstream.JetStream, accountID string) func(ctx context.Context, eniID string) (map[string]string, bool, error) {
+	return func(ctx context.Context, eniID string) (map[string]string, bool, error) {
+		kv, err := js.KeyValue(ctx, ec2vpc.KVBucketENIs)
+		if err != nil {
+			return nil, false, fmt.Errorf("open ENI bucket: %w", err)
+		}
+		entry, err := kv.Get(ctx, kvutil.AccountKey(accountID, eniID))
+		if errors.Is(err, jetstream.ErrKeyNotFound) {
+			return nil, false, nil
+		}
+		if err != nil {
+			return nil, false, fmt.Errorf("read ENI record %s: %w", eniID, err)
+		}
+		var record ec2vpc.ENIRecord
+		if err := json.Unmarshal(entry.Value(), &record); err != nil {
+			return nil, false, fmt.Errorf("decode ENI record %s: %w", eniID, err)
+		}
+		return record.Tags, true, nil
+	}
+}

spinifex/domains/ec2/inventory/inventory_test.go

@@ -0,0 +1,49 @@
+package inventory
+
+import (
+	"encoding/json"
+	"testing"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestListAccountInstancesRefusesAnUnknownNodeSet(t *testing.T) {
+	out, err := NewNATSInstances(nil, nil).ListAccountInstances(t.Context(), "000000000000")
+	require.Error(t, err)
+	assert.Nil(t, out, "an unknown node set must not yield a list")
+}
+
+func TestENITagReaderSeparatesGoneFromUnreadable(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	js := testutil.NewJetStream(t, nc)
+	read := NewENITagReader(js, "000000000000")
+
+	_, _, err := read(t.Context(), "eni-1")
+	require.Error(t, err, "a missing ENI bucket is a read failure, not an absent ENI")
+
+	enis, err := kvutil.GetOrCreateBucket(t.Context(), js, ec2vpc.KVBucketENIs, 10)
+	require.NoError(t, err)
+	good, err := json.Marshal(ec2vpc.ENIRecord{NetworkInterfaceId: "eni-good", Tags: map[string]string{"k": "v"}})
+	require.NoError(t, err)
+	_, err = enis.Put(t.Context(), kvutil.AccountKey("000000000000", "eni-good"), good)
+	require.NoError(t, err)
+	_, err = enis.Put(t.Context(), kvutil.AccountKey("000000000000", "eni-corrupt"), []byte("{not json"))
+	require.NoError(t, err)
+
+	tags, found, err := read(t.Context(), "eni-good")
+	require.NoError(t, err)
+	assert.True(t, found)
+	assert.Equal(t, map[string]string{"k": "v"}, tags)
+
+	_, found, err = read(t.Context(), "eni-absent")
+	require.NoError(t, err)
+	assert.False(t, found, "a missing key is the ENI gone")
+
+	_, found, err = read(t.Context(), "eni-corrupt")
+	require.Error(t, err, "an undecodable record is a read failure")
+	assert.False(t, found)
+}

spinifex/domains/eks/cluster/cp_orphans.go

@@ -0,0 +1,158 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"sort"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// ClusterRecordState is what a read of a cluster's record established.
+type ClusterRecordState int
+
+const (
+	ClusterRecordPresent ClusterRecordState = iota
+	ClusterRecordMissing
+	ClusterRecordUnreadable
+)
+
+// ClusterRecordVerdict is the outcome of looking a cluster record up without
+// creating anything. Reason names a missing or unreadable record; Err is set
+// only when the record could not be read.
+type ClusterRecordVerdict struct {
+	State  ClusterRecordState
+	Reason string
+	Err    error
+}
+
+// JudgeClusterRecord reads the cluster record read-only. A missing bucket or a
+// missing key is Missing, any other failure is Unreadable, and neither is ever
+// repaired. The billable reaper and the orphan report both use it, so they
+// judge a record the same way.
+func JudgeClusterRecord(ctx context.Context, js jetstream.JetStream, account, cluster string) ClusterRecordVerdict {
+	kv, err := OpenAccountBucket(ctx, js, account)
+	switch {
+	case errors.Is(err, jetstream.ErrBucketNotFound):
+		return ClusterRecordVerdict{State: ClusterRecordMissing, Reason: "account bucket missing"}
+	case err != nil:
+		return ClusterRecordVerdict{State: ClusterRecordUnreadable, Reason: "account bucket lookup failed", Err: err}
+	}
+	_, err = GetClusterMeta(ctx, kv, cluster)
+	switch {
+	case err == nil:
+		return ClusterRecordVerdict{State: ClusterRecordPresent}
+	case errors.Is(err, ErrClusterNotFound):
+		return ClusterRecordVerdict{State: ClusterRecordMissing, Reason: "cluster record missing"}
+	}
+	return ClusterRecordVerdict{State: ClusterRecordUnreadable, Reason: "cluster record read failed", Err: err}
+}
+
+// ENITagReader returns the tags on a control-plane ENI. found is false when
+// the ENI is gone.
+type ENITagReader func(ctx context.Context, eniID string) (tagSet map[string]string, found bool, err error)
+
+// ControlPlaneRow is one running EKS control-plane VM the report could not tie
+// to a live cluster record. Unreadable rows carry Err and are never omitted.
+type ControlPlaneRow struct {
+	InstanceID       string
+	Cluster          string
+	Account          string
+	Reason           string
+	Age              time.Duration
+	AvailabilityZone string
+	Unreadable       bool
+	Err              error
+}
+
+// FindControlPlaneOrphans classifies the instances in reservations. An instance
+// with no ENI, or whose ENI is gone or not tagged as an EKS control plane, is
+// skipped. A failed ENI read is an unreadable row, never a silent drop. A
+// control plane whose cluster record is present is omitted; one whose bucket or
+// record is missing is an orphan row; any other read failure is unreadable. It
+// only reads.
+func FindControlPlaneOrphans(ctx context.Context, js jetstream.JetStream, reservations []*ec2.Reservation, eni ENITagReader, now time.Time) []ControlPlaneRow {
+	var rows []ControlPlaneRow
+	for _, res := range reservations {
+		if res == nil {
+			continue
+		}
+		for _, inst := range res.Instances {
+			if inst == nil || !instanceRunning(inst) || len(inst.NetworkInterfaces) == 0 || inst.NetworkInterfaces[0] == nil {
+				continue
+			}
+			eniID := aws.StringValue(inst.NetworkInterfaces[0].NetworkInterfaceId)
+			if eniID == "" {
+				continue
+			}
+			row := ControlPlaneRow{InstanceID: aws.StringValue(inst.InstanceId)}
+			if inst.Placement != nil {
+				row.AvailabilityZone = aws.StringValue(inst.Placement.AvailabilityZone)
+			}
+			if inst.LaunchTime != nil {
+				row.Age = now.Sub(*inst.LaunchTime).Truncate(time.Second)
+			}
+
+			tagSet, found, err := eni(ctx, eniID)
+			if err != nil {
+				row.Unreadable, row.Err, row.Reason = true, err, "control-plane ENI read failed"
+				rows = append(rows, row)
+				continue
+			}
+			if !found || tagSet[tags.ManagedByKey] != tags.ManagedByEKS || tagSet[clusterEKSRoleTagKey] != clusterEKSRoleControlPlane {
+				continue
+			}
+			row.Cluster, row.Account = tagSet[clusterEKSClusterTagKey], tagSet[clusterEKSAccountTagKey]
+			if row.Cluster == "" || row.Account == "" {
+				continue
+			}
+
+			verdict := JudgeClusterRecord(ctx, js, row.Account, row.Cluster)
+			switch verdict.State {
+			case ClusterRecordPresent:
+				continue
+			case ClusterRecordUnreadable:
+				row.Unreadable, row.Err = true, verdict.Err
+			}
+			row.Reason = verdict.Reason
+			rows = append(rows, row)
+		}
+	}
+	sort.Slice(rows, func(i, j int) bool { return rows[i].InstanceID < rows[j].InstanceID })
+	return rows
+}
+
+// ReportControlPlaneOrphans writes the orphan report. list must return every
+// non-terminated system instance or an error: a failed or partial listing is
+// returned as an error and nothing is printed, so a short list is never taken
+// for a complete one. Unreadable rows are printed and counted separately.
+func ReportControlPlaneOrphans(ctx context.Context, w io.Writer, js jetstream.JetStream, list func(context.Context) (*ec2.DescribeInstancesOutput, error), eni ENITagReader, now time.Time) (orphans, unreadable int, err error) {
+	out, err := list(ctx)
+	if err != nil {
+		return 0, 0, fmt.Errorf("listing incomplete: %w", err)
+	}
+	if out == nil {
+		return 0, 0, errors.New("listing incomplete: no result")
+	}
+	rows := FindControlPlaneOrphans(ctx, js, out.Reservations, eni, now)
+	for _, r := range rows {
+		if r.Unreadable {
+			unreadable++
+			fmt.Fprintf(w, "UNREADABLE  %s  cluster=%s account=%s az=%s age=%s  %s: %v\n", r.InstanceID, r.Cluster, r.Account, r.AvailabilityZone, r.Age, r.Reason, r.Err)
+			continue
+		}
+		orphans++
+		fmt.Fprintf(w, "ORPHAN      %s  cluster=%s account=%s az=%s age=%s  %s\n", r.InstanceID, r.Cluster, r.Account, r.AvailabilityZone, r.Age, r.Reason)
+	}
+	return orphans, unreadable, nil
+}
+
+func instanceRunning(inst *ec2.Instance) bool {
+	return inst.State != nil && aws.StringValue(inst.State.Name) == ec2.InstanceStateNameRunning
+}

spinifex/domains/eks/cluster/cp_orphans_test.go

@@ -0,0 +1,198 @@
+package cluster
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	ec2inventory "github.com/mulgadc/spinifex/spinifex/domains/ec2/inventory"
+	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func orphanInstance(id, eni string) *ec2.Instance {
+	return &ec2.Instance{
+		InstanceId:        aws.String(id),
+		State:             &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)},
+		LaunchTime:        aws.Time(time.Unix(0, 0)),
+		Placement:         &ec2.Placement{AvailabilityZone: aws.String("ap-southeast-2a")},
+		NetworkInterfaces: []*ec2.InstanceNetworkInterface{{NetworkInterfaceId: aws.String(eni)}},
+	}
+}
+
+func cpTags(cluster, account string) map[string]string {
+	return map[string]string{
+		tags.ManagedByKey:       tags.ManagedByEKS,
+		clusterEKSClusterTagKey: cluster,
+		clusterEKSAccountTagKey: account,
+		clusterEKSRoleTagKey:    clusterEKSRoleControlPlane,
+	}
+}
+
+func workerTags(cluster, account string) map[string]string {
+	m := cpTags(cluster, account)
+	m[clusterEKSRoleTagKey] = "worker"
+	return m
+}
+
+func TestFindControlPlaneOrphansUnreadableRecordIsNotAnOrphan(t *testing.T) {
+	f := newEKSServiceFixture(t)
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+	ctx, cancel := context.WithCancel(t.Context())
+	cancel()
+	eni := func(context.Context, string) (map[string]string, bool, error) {
+		return cpTags("gone", testAccountID), true, nil
+	}
+	res := []*ec2.Reservation{{Instances: []*ec2.Instance{orphanInstance("i-1", "eni-1")}}}
+
+	rows := FindControlPlaneOrphans(ctx, js, res, eni, time.Unix(60, 0))
+
+	require.Len(t, rows, 1)
+	assert.True(t, rows[0].Unreadable, "a read that failed is not a missing record")
+	assert.Error(t, rows[0].Err)
+}
+
+func TestFindControlPlaneOrphans(t *testing.T) {
+	const lost = "999999999999"
+	f := newEKSServiceFixture(t)
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+	require.NoError(t, PutClusterMeta(t.Context(), f.kv, sampleClusterMeta("alive")))
+
+	eniTags := map[string]map[string]string{
+		"eni-orphan":   cpTags("gone", testAccountID),
+		"eni-live":     cpTags("alive", testAccountID),
+		"eni-nobucket": cpTags("lost", lost),
+		"eni-worker":   workerTags("gone", testAccountID),
+	}
+	eni := func(_ context.Context, id string) (map[string]string, bool, error) {
+		if id == "eni-broken" {
+			return nil, false, errors.New("vpc service down")
+		}
+		m, ok := eniTags[id]
+		return m, ok, nil
+	}
+	res := []*ec2.Reservation{{Instances: []*ec2.Instance{
+		orphanInstance("i-orphan", "eni-orphan"),
+		orphanInstance("i-live", "eni-live"),
+		orphanInstance("i-nobucket", "eni-nobucket"),
+		orphanInstance("i-worker", "eni-worker"),
+		orphanInstance("i-ghost", "eni-ghost"),
+		orphanInstance("i-broken", "eni-broken"),
+		{InstanceId: aws.String("i-noeni")},
+	}}}
+
+	rows := FindControlPlaneOrphans(t.Context(), js, res, eni, time.Unix(60, 0))
+
+	got := map[string]ControlPlaneRow{}
+	for _, r := range rows {
+		got[r.InstanceID] = r
+	}
+	assert.Len(t, got, 3, "only orphan, nobucket and the unreadable ENI are reported")
+	assert.Equal(t, "cluster record missing", got["i-orphan"].Reason)
+	assert.False(t, got["i-orphan"].Unreadable)
+	assert.Equal(t, "account bucket missing", got["i-nobucket"].Reason)
+	assert.True(t, got["i-broken"].Unreadable, "a failed ENI read is never dropped")
+	assert.Error(t, got["i-broken"].Err)
+	assert.NotContains(t, got, "i-live", "a present record is omitted")
+	assert.NotContains(t, got, "i-worker")
+	assert.NotContains(t, got, "i-ghost")
+	assert.NotContains(t, got, "i-noeni")
+}
+
+func TestReportControlPlaneOrphansIncompleteListingPrintsNothing(t *testing.T) {
+	f := newEKSServiceFixture(t)
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+	var buf bytes.Buffer
+	list := func(context.Context) (*ec2.DescribeInstancesOutput, error) {
+		return nil, errors.New("ServiceUnavailable")
+	}
+	orphans, unreadable, err := ReportControlPlaneOrphans(t.Context(), &buf, js, list, nil, time.Now())
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "listing incomplete")
+	assert.Zero(t, orphans+unreadable)
+	assert.Empty(t, buf.String(), "no partial list on a failed listing")
+}
+
+func TestReportControlPlaneOrphansPrintsRows(t *testing.T) {
+	f := newEKSServiceFixture(t)
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+	var buf bytes.Buffer
+	list := func(context.Context) (*ec2.DescribeInstancesOutput, error) {
+		return &ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{{Instances: []*ec2.Instance{
+			orphanInstance("i-orphan", "eni-orphan"), orphanInstance("i-broken", "eni-broken"),
+		}}}}, nil
+	}
+	eni := func(_ context.Context, id string) (map[string]string, bool, error) {
+		if id == "eni-broken" {
+			return nil, false, errors.New("boom")
+		}
+		return cpTags("gone", testAccountID), true, nil
+	}
+	orphans, unreadable, err := ReportControlPlaneOrphans(t.Context(), &buf, js, list, eni, time.Unix(60, 0))
+	require.NoError(t, err)
+	assert.Equal(t, 1, orphans)
+	assert.Equal(t, 1, unreadable)
+	assert.Contains(t, buf.String(), "ORPHAN      i-orphan")
+	assert.Contains(t, buf.String(), "UNREADABLE  i-broken")
+	assert.Equal(t, 2, strings.Count(buf.String(), "az=ap-southeast-2a"), "both row kinds print the zone")
+}
+
+func TestFindControlPlaneOrphansCorruptENIRecordIsUnreadable(t *testing.T) {
+	f := newEKSServiceFixture(t)
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+	enis, err := kvutil.GetOrCreateBucket(t.Context(), js, ec2vpc.KVBucketENIs, 10)
+	require.NoError(t, err)
+	good, err := json.Marshal(ec2vpc.ENIRecord{NetworkInterfaceId: "eni-good", Tags: cpTags("gone", testAccountID)})
+	require.NoError(t, err)
+	_, err = enis.Put(t.Context(), kvutil.AccountKey("000000000000", "eni-good"), good)
+	require.NoError(t, err)
+	_, err = enis.Put(t.Context(), kvutil.AccountKey("000000000000", "eni-corrupt"), []byte("{not json"))
+	require.NoError(t, err)
+	res := []*ec2.Reservation{{Instances: []*ec2.Instance{
+		orphanInstance("i-good", "eni-good"), orphanInstance("i-corrupt", "eni-corrupt"), orphanInstance("i-absent", "eni-absent"),
+	}}}
+
+	rows := FindControlPlaneOrphans(t.Context(), js, res, ec2inventory.NewENITagReader(js, "000000000000"), time.Unix(60, 0))
+
+	require.Len(t, rows, 2, "an absent ENI is skipped, the other two are reported")
+	assert.Equal(t, "i-corrupt", rows[0].InstanceID)
+	assert.True(t, rows[0].Unreadable, "a corrupt ENI record is unreadable, never gone")
+	assert.Equal(t, "i-good", rows[1].InstanceID)
+	assert.False(t, rows[1].Unreadable)
+}
+
+func TestFindControlPlaneOrphansOnlyReportsRunningInstances(t *testing.T) {
+	f := newEKSServiceFixture(t)
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+	eni := func(context.Context, string) (map[string]string, bool, error) {
+		return cpTags("gone", testAccountID), true, nil
+	}
+	mk := func(id, state string) *ec2.Instance {
+		i := orphanInstance(id, "eni-"+id)
+		i.State = &ec2.InstanceState{Name: aws.String(state)}
+		return i
+	}
+	noState := orphanInstance("i-nostate", "eni-nostate")
+	noState.State = nil
+	res := []*ec2.Reservation{{Instances: []*ec2.Instance{
+		mk("i-running", ec2.InstanceStateNameRunning),
+		mk("i-stopped", ec2.InstanceStateNameStopped),
+		mk("i-pending", ec2.InstanceStateNamePending),
+		noState,
+	}}}
+
+	rows := FindControlPlaneOrphans(t.Context(), js, res, eni, time.Unix(60, 0))
+
+	require.Len(t, rows, 1)
+	assert.Equal(t, "i-running", rows[0].InstanceID)
+}

spinifex/domains/eks/cluster/eks_billable_reaper.go

@@ -2,7 +2,6 @@ package cluster
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"log/slog"
 
@@ -13,13 +12,10 @@ import (
 	"github.com/nats-io/nats.go/jetstream"
 )
 
-// EKSBillableReaper is the meta-independent billable cleanup from historic PROP-LIFECYCLE-006 §5 (behaviour per ADR-0003:S2), built on
-// the resource lifecycle GC backstop. A normal DeleteCluster can leave a
-// running control-plane VM behind (terminate completes async; a daemon restart
-// re-launches it from saved running-state) after the cluster meta is already
-// swept — a billable orphan with no owner to drive a retry. This reaper finds a
-// running EKS control-plane VM whose cluster meta is DEFINITIVELY GONE and
-// terminates it. It is node-local: each node reaps the orphans it runs.
+// EKSBillableReaper raises the data-safety alarm for a running EKS control-plane
+// VM whose cluster record cannot be found. It never terminates anything: a
+// missing record cannot tell a completed DeleteCluster from a lost bucket or an
+// out-of-band delete, so removal is an operator decision. It is node-local.
 type EKSBillableReaper struct {
 	svc *EKSServiceImpl
 	// running returns this node's running VMs. The reaper filters to the EKS
@@ -38,10 +34,11 @@ func (s *EKSServiceImpl) NewBillableReaper(running func() ([]*vm.VM, error)) *EK
 func (r *EKSBillableReaper) Class() string         { return "eks-billable" }
 func (r *EKSBillableReaper) Scope() vm.ReaperScope { return vm.ScopeNodeLocal }
 
-// Sweep terminates every running EKS control-plane VM whose cluster meta is
-// absent. A VM whose cluster still exists (CREATING or DELETING) is left to the
-// cluster's own teardown/retry; an ENI that is gone or untagged is skipped, so
-// the reaper never reaps on uncertainty.
+// Sweep alarms on every running EKS control-plane VM whose cluster record, or
+// whole account bucket, is missing, and removes nothing. A VM whose cluster
+// exists is left to the cluster's own teardown; an ENI that is gone or untagged
+// is skipped, and a read that fails is logged and skipped. It returns 0 because
+// it never removes a VM.
 func (r *EKSBillableReaper) Sweep(ctx context.Context) (int, error) {
 	vms, err := r.running()
 	if err != nil {
@@ -53,11 +50,10 @@ func (r *EKSBillableReaper) Sweep(ctx context.Context) (int, error) {
 		return 0, fmt.Errorf("jetstream: %w", err)
 	}
 
-	reaped := 0
 	for _, v := range vms {
 		select {
 		case <-ctx.Done():
-			return reaped, ctx.Err()
+			return 0, ctx.Err()
 		default:
 		}
 		if v == nil || v.ManagedBy != tags.ManagedByEKS || v.ENIId == "" {
@@ -66,47 +62,23 @@ func (r *EKSBillableReaper) Sweep(ctx context.Context) (int, error) {
 
 		clusterName, clusterAccount, ok := r.clusterRefFromENI(ctx, v)
 		if !ok {
-			continue // ENI gone or untagged: cannot confirm orphan-hood, never reap
+			continue // ENI gone or untagged: cannot tie the VM to a cluster
 		}
 
-		acctKV, err := GetOrCreateAccountBucket(ctx, js, clusterAccount)
-		if err != nil {
-			slog.WarnContext(ctx, "eks-billable: account bucket lookup failed", "account", clusterAccount, "err", err)
-			continue
-		}
-		if _, err := GetClusterMeta(ctx, acctKV, clusterName); !errors.Is(err, ErrClusterNotFound) {
-			// Meta present (live/CREATING/DELETING) or unreadable: not a definitive
-			// orphan — leave it to the cluster's own teardown/retry.
+		verdict := JudgeClusterRecord(ctx, js, clusterAccount, clusterName)
+		switch verdict.State {
+		case ClusterRecordPresent:
+			continue // cluster exists (live, CREATING or DELETING)
+		case ClusterRecordUnreadable:
+			slog.WarnContext(ctx, "eks-billable: "+verdict.Reason, "cluster", clusterName, "account", clusterAccount, "err", verdict.Err)
 			continue
 		}
+		reason := verdict.Reason
 
-		// ALARM + reap: the cluster is gone but its control-plane VM still runs.
-		slog.WarnContext(ctx, "DATA-SAFETY ALARM: orphaned EKS control-plane VM reaped (cluster meta gone)",
-			"instanceId", v.ID, "cluster", clusterName, "account", clusterAccount, "node", v.LastNode)
-		if err := TerminateK3sServerVM(ctx, r.svc.deps.VPCK3s, r.svc.deps.Instance, v.AccountID, v.ID, v.ENIId); err != nil {
-			slog.WarnContext(ctx, "eks-billable: failed to terminate orphan CP VM", "instanceId", v.ID, "err", err)
-			continue
-		}
-		reaped++
-		r.reclaimOrphanInfra(ctx, v.AccountID, clusterName)
-	}
-	return reaped, nil
-}
-
-// reclaimOrphanInfra tears down the billable infra a reaped orphan's swept meta
-// no longer anchors: the NLB front-end NAT and the managed CP VPC's NAT-GW EIP.
-// Both are keyed by cluster name and idempotent, so a clean prior delete is a
-// no-op. Best-effort — the costliest resource (the VM) is already gone, and
-// DeleteClusterCPVPC releases the NAT-GW EIP before the VPC delete, so the
-// billable address is reclaimed even if the now-empty VPC delete trails. account
-// is the infra account (system account for the managed CP VPC topology).
-func (r *EKSBillableReaper) reclaimOrphanInfra(ctx context.Context, account, clusterName string) {
-	if err := DeleteClusterNLB(ctx, r.svc.deps.NLB, account, clusterName); err != nil {
-		slog.WarnContext(ctx, "eks-billable: reclaim orphan NLB failed", "cluster", clusterName, "err", err)
-	}
-	if err := DeleteClusterCPVPC(ctx, r.svc.cpVPCDeps(), account, clusterName, nil); err != nil {
-		slog.WarnContext(ctx, "eks-billable: reclaim orphan CP VPC (NAT-GW EIP) failed", "cluster", clusterName, "err", err)
+		slog.WarnContext(ctx, "DATA-SAFETY ALARM: EKS control-plane VM has no cluster record; retained for an operator",
+			"reason", reason, "instanceId", v.ID, "cluster", clusterName, "account", clusterAccount, "node", v.LastNode)
 	}
+	return 0, nil
 }
 
 // clusterRefFromENI reads the cluster name + customer account from the VM's

spinifex/domains/eks/cluster/eks_billable_reaper_test.go

@@ -1,14 +1,19 @@
 package cluster
 
 import (
+	"bytes"
 	"context"
+	"log/slog"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/elbv2"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -33,53 +38,119 @@ func cpVM(id, eniID string) *vm.VM {
 	return &vm.VM{ID: id, ManagedBy: tags.ManagedByEKS, ENIId: eniID, AccountID: reaperSysAccount, LastNode: "node-1"}
 }
 
-// TestRLC5_EKSBillableReaperTerminatesOrphanCPVM exercises the historic PROP-LIFECYCLE-006 §5 (behaviour per ADR-0003:S2)
-// meta-independent billable cleanup: a running EKS control-plane VM whose
-// cluster meta is DEFINITIVELY GONE is a billable orphan and must be terminated
-// by the GC backstop — the real fix for the orphan CP VM surviving a
-// daemon restart after DeleteCluster swept the meta.
-func TestRLC5_EKSBillableReaperTerminatesOrphanCPVM(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	f.vpc.describeByENI = map[string]*ec2.NetworkInterface{
-		"eni-orphan": cpENI("eni-orphan", "gone-cluster", testAccountID),
+// ADR-0003:S7. A running control-plane VM whose cluster record cannot be found
+// is retained and alarmed, never terminated and never has its NLB or VPC torn
+// down: a missing record cannot tell a completed DeleteCluster from a lost
+// bucket or an out-of-band delete.
+func TestS7_EKSBillableReaperNeverTerminatesAControlPlane(t *testing.T) {
+	const lostAccount = "999999999999"
+
+	cases := []struct {
+		name      string
+		account   string
+		prepare   func(t *testing.T, f *eksServiceFixture)
+		wantAlarm string
+		wantLog   string
+	}{
+		{
+			name:      "account bucket missing",
+			account:   lostAccount,
+			prepare:   func(*testing.T, *eksServiceFixture) {},
+			wantAlarm: "account bucket missing",
+		},
+		{
+			name:      "bucket has no meta key",
+			account:   testAccountID,
+			prepare:   func(*testing.T, *eksServiceFixture) {},
+			wantAlarm: "cluster record missing",
+		},
+		{
+			name:    "meta removed by the teardown revision-checked delete",
+			account: testAccountID,
+			prepare: func(t *testing.T, f *eksServiceFixture) {
+				require.NoError(t, PutClusterMeta(t.Context(), f.kv, sampleClusterMeta("orphan")))
+				_, err := kvutil.DeleteIf(t.Context(), f.kv, ClusterMetaKey("orphan"), kvutil.CASConfig{
+					Attempts: maxClusterStateCASRetries,
+					NotFound: ErrClusterNotFound,
+				}, func(*ClusterMeta) error { return nil })
+				require.NoError(t, err)
+			},
+			wantAlarm: "cluster record missing",
+		},
+		{
+			name:    "meta removed out of band",
+			account: testAccountID,
+			prepare: func(t *testing.T, f *eksServiceFixture) {
+				require.NoError(t, PutClusterMeta(t.Context(), f.kv, sampleClusterMeta("orphan")))
+				require.NoError(t, f.kv.Delete(t.Context(), ClusterMetaKey("orphan")))
+			},
+			wantAlarm: "cluster record missing",
+		},
+		{
+			name:    "meta unreadable",
+			account: testAccountID,
+			prepare: func(t *testing.T, f *eksServiceFixture) {
+				_, err := f.kv.Put(t.Context(), ClusterMetaKey("orphan"), []byte("{not json"))
+				require.NoError(t, err)
+			},
+			wantLog: "cluster record read failed",
+		},
 	}
-	orphan := cpVM("i-orphan", "eni-orphan")
 
-	// Seed the orphan cluster's name/tag-driven billable infra the swept meta no
-	// longer anchors: an NLB front-end and a tagged NAT gateway holding a billable
-	// EIP. The reaper's post-terminate reclaim must sweep both.
-	nlbName := ClusterNLBName("gone-cluster")
-	f.nlb.lbByName[nlbName] = &elbv2.LoadBalancer{
-		LoadBalancerArn:  aws.String("arn:lb-orphan"),
-		LoadBalancerName: aws.String(nlbName),
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			// slog.SetDefault is process-wide, so this test is not parallel.
+			var buf bytes.Buffer
+			restore := slog.Default()
+			slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo})))
+			t.Cleanup(func() { slog.SetDefault(restore) })
+
+			f := newEKSServiceFixture(t)
+			f.vpc.describeByENI = map[string]*ec2.NetworkInterface{
+				"eni-cp": cpENI("eni-cp", "orphan", tc.account),
+			}
+			nlbName := ClusterNLBName("orphan")
+			f.nlb.lbByName[nlbName] = &elbv2.LoadBalancer{
+				LoadBalancerArn:  aws.String("arn:lb-orphan"),
+				LoadBalancerName: aws.String(nlbName),
+			}
+			f.ngw.gws = []*fakeCPNatGateway{{
+				id:    "nat-orphan",
+				tags:  map[string]string{clusterEKSClusterTagKey: "orphan", clusterEKSRoleTagKey: cpVPCRoles.NatGW},
+				state: "available",
+				addrs: []*ec2.NatGatewayAddress{{AllocationId: aws.String("eipalloc-orphan")}},
+			}}
+			tc.prepare(t, f)
+
+			reaper := f.svc.NewBillableReaper(func() ([]*vm.VM, error) { return []*vm.VM{cpVM("i-cp", "eni-cp")}, nil })
+			for range 2 {
+				removed, err := reaper.Sweep(context.Background())
+				require.NoError(t, err)
+				assert.Zero(t, removed, "ADR-0003:S7: the reaper removes nothing")
+			}
+
+			assert.Empty(t, f.inst.terminateCalls, "ADR-0003:S7: a reaper's judgement is never the decision to terminate")
+			assert.Empty(t, f.vpc.deleteCalls, "the control-plane ENI must stay")
+			assert.Contains(t, f.nlb.lbByName, nlbName, "the NLB must stay")
+			assert.Empty(t, f.eip.releaseCalls, "no EIP may be released")
+
+			js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+			if tc.account == lostAccount {
+				_, err := OpenAccountBucket(t.Context(), js, lostAccount)
+				assert.ErrorIs(t, err, jetstream.ErrBucketNotFound, "a sweep must not create the bucket it found missing")
+			}
+
+			out := buf.String()
+			if tc.wantAlarm != "" {
+				assert.Contains(t, out, "DATA-SAFETY ALARM")
+				assert.Contains(t, out, tc.wantAlarm)
+				assert.Contains(t, out, "i-cp")
+			} else {
+				assert.NotContains(t, out, "DATA-SAFETY ALARM", "an unreadable record is not a missing one")
+				assert.Contains(t, out, tc.wantLog)
+			}
+		})
 	}
-	f.ngw.gws = []*fakeCPNatGateway{{
-		id:    "nat-orphan",
-		tags:  map[string]string{clusterEKSClusterTagKey: "gone-cluster", clusterEKSRoleTagKey: cpVPCRoles.NatGW},
-		state: "available",
-		addrs: []*ec2.NatGatewayAddress{{AllocationId: aws.String("eipalloc-orphan")}},
-	}}
-
-	reaper := f.svc.NewBillableReaper(func() ([]*vm.VM, error) { return []*vm.VM{orphan}, nil })
-
-	reaped, err := reaper.Sweep(context.Background())
-	require.NoError(t, err)
-	assert.Equal(t, 1, reaped, "ADR-0003:S2 behaviour; historic PROP-LIFECYCLE-006 §5 mechanism: a CP VM whose cluster meta is gone must be reaped")
-	assert.Contains(t, f.inst.terminateCalls, "i-orphan", "the orphan CP VM must be terminated")
-	require.Len(t, f.vpc.deleteCalls, 1, "the orphan CP ENI must be deleted")
-	assert.Equal(t, "eni-orphan", aws.StringValue(f.vpc.deleteCalls[0].NetworkInterfaceId))
-
-	// The orphan's billable NLB + NAT-GW EIP must also be reclaimed.
-	assert.NotContains(t, f.nlb.lbByName, nlbName, "the orphan NLB front-end must be reclaimed")
-	require.Len(t, f.eip.releaseCalls, 1, "the orphan NAT-GW EIP must be released")
-	assert.Equal(t, "eipalloc-orphan", aws.StringValue(f.eip.releaseCalls[0].AllocationId))
-
-	// Idempotent: a repeat sweep over the same (still meta-absent) VM is safe —
-	// the terminate + reclaim paths tolerate already-gone infra (no second release).
-	_, err = reaper.Sweep(context.Background())
-	require.NoError(t, err)
-	assert.Len(t, f.eip.releaseCalls, 1, "reclaim must not double-release an already-freed EIP")
 }
 
 // TestRLC5_EKSBillableReaperSpareLiveAndUncertain enforces the reaper's no-false-

spinifex/domains/eks/cluster/store.go

@@ -185,7 +185,8 @@ func AccountWatchBuckets(ctx context.Context, nc *nats.Conn) ([]*kvstore.Bucket,
 	}
 	buckets := make([]*kvstore.Bucket, 0, len(names))
 	for _, name := range names {
-		buckets = append(buckets, kvstore.NewBucket(js, kvstore.Config{Name: name, History: 1}))
+		// Attach only: a watch on a deleted bucket must fail and retry, not recreate it.
+		buckets = append(buckets, kvstore.NewBucket(js, kvstore.Config{Name: name, History: 1, AttachOnly: true}))
 	}
 	return buckets, nil
 }
@@ -206,6 +207,13 @@ func GetOrCreateAccountBucket(ctx context.Context, js jetstream.JetStream, accou
 	return kv, nil
 }
 
+// OpenAccountBucket returns the per-account KV bucket for accountID without
+// creating it. A bucket that does not exist is jetstream.ErrBucketNotFound, so a
+// read-only caller never turns a lost bucket into an empty one.
+func OpenAccountBucket(ctx context.Context, js jetstream.JetStream, accountID string) (jetstream.KeyValue, error) {
+	return js.KeyValue(ctx, AccountBucketName(accountID))
+}
+
 // InitLeaderBucket creates (or attaches to) the shared spinifex-eks-leader
 // bucket used for per-cluster reconciler leader-lease CAS locks.
 func InitLeaderBucket(ctx context.Context, js jetstream.JetStream) (jetstream.KeyValue, error) {

spinifex/domains/eks/cluster/watch_source_test.go

@@ -7,6 +7,7 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit"
 	ekscluster "github.com/mulgadc/spinifex/spinifex/domains/eks/cluster"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -79,3 +80,26 @@ func TestAccountWatchBuckets_EnumerationErrorPropagates(t *testing.T) {
 	_, err := ekscluster.AccountWatchBuckets(ctx, nc)
 	require.Error(t, err)
 }
+
+// TestAccountWatchBuckets_WatchOnDeletedBucketDoesNotRecreateIt pins attach-only
+// handles: a DNS watch that outlives its bucket must fail and retry, because
+// recreating it hides a deleted account bucket behind an empty one.
+func TestAccountWatchBuckets_WatchOnDeletedBucketDoesNotRecreateIt(t *testing.T) {
+	t.Parallel()
+	_, nc, _ := testutil.StartTestJetStream(t)
+	js := testutil.NewJetStream(t, nc)
+	_, err := ekscluster.GetOrCreateAccountBucket(t.Context(), js, "111111111111")
+	require.NoError(t, err)
+
+	buckets, err := ekscluster.AccountWatchBuckets(t.Context(), nc)
+	require.NoError(t, err)
+	require.Len(t, buckets, 1)
+
+	require.NoError(t, js.DeleteKeyValue(t.Context(), ekscluster.AccountBucketName("111111111111")))
+
+	_, err = buckets[0].Watch(t.Context(), ">")
+	require.Error(t, err, "a watch on a deleted bucket must fail")
+
+	_, err = ekscluster.OpenAccountBucket(t.Context(), js, "111111111111")
+	assert.ErrorIs(t, err, jetstream.ErrBucketNotFound, "the watch must not have recreated the bucket")
+}

spinifex/domains/lambda/awsapi/eventsource.go

@@ -0,0 +1,277 @@
+package awsapi
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/lambda"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/eventsource"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// MappingStore is what the handlers need of the event-source mapping store.
+type MappingStore interface {
+	Create(ctx context.Context, m eventsource.Mapping) error
+	Get(ctx context.Context, uuid string) (eventsource.Mapping, uint64, error)
+	List(ctx context.Context) ([]eventsource.Entry, error)
+	Update(ctx context.Context, m eventsource.Mapping, revision uint64) (uint64, error)
+}
+
+var _ MappingStore = (*eventsource.Store)(nil)
+
+// SourceChecker reads an SQS queue's visibility timeout as the function's
+// execution role, so a queue the role cannot read is refused at create.
+type SourceChecker interface {
+	QueueVisibilityTimeout(ctx context.Context, rec function.Record, queueARN string) (int64, error)
+}
+
+// Batch size bounds for an SQS standard queue without a batching window.
+const (
+	defaultBatchSize = 10
+	maxBatchSize     = 10
+)
+
+// mappingCreateFields are the CreateEventSourceMapping members this build
+// accepts; every other member is refused rather than ignored.
+var mappingCreateFields = map[string]bool{
+	"FunctionName": true, "EventSourceArn": true, "BatchSize": true, "Enabled": true,
+	"MaximumBatchingWindowInSeconds": true,
+}
+
+var mappingUUIDRE = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`)
+
+func mappingNotFound() error {
+	return awserrors.Errorf(awserrors.ErrorResourceNotFoundException, "The resource you requested does not exist.")
+}
+
+// mappingARN is the IAM resource of one mapping, or "*" for a malformed UUID.
+func mappingARN(region, accountID, uuid string) string {
+	if region == "" || accountID == "" || !mappingUUIDRE.MatchString(uuid) {
+		return "*"
+	}
+	return fmt.Sprintf("arn:aws:lambda:%s:%s:event-source-mapping:%s", region, accountID, uuid)
+}
+
+type mappingCreate struct {
+	FunctionName, QueueARN string
+	BatchSize              int64
+}
+
+// decodeMappingCreate reads the body strictly and refuses every option this
+// build does not implement.
+func decodeMappingCreate(body []byte) (*mappingCreate, error) {
+	var raw map[string]json.RawMessage
+	dec := json.NewDecoder(bytes.NewReader(body))
+	if err := dec.Decode(&raw); err != nil {
+		return nil, invalid("The request body could not be parsed as JSON.")
+	}
+	for k := range raw {
+		if !mappingCreateFields[k] {
+			return nil, unsupported(k)
+		}
+	}
+	var in struct {
+		FunctionName                   *string `json:"FunctionName"`
+		EventSourceArn                 *string `json:"EventSourceArn"`
+		BatchSize                      *int64  `json:"BatchSize"`
+		Enabled                        *bool   `json:"Enabled"`
+		MaximumBatchingWindowInSeconds *int64  `json:"MaximumBatchingWindowInSeconds"`
+	}
+	if err := json.Unmarshal(body, &in); err != nil {
+		return nil, invalid("The request body could not be parsed as JSON.")
+	}
+	if in.FunctionName == nil || *in.FunctionName == "" {
+		return nil, invalid("1 validation error detected: Value null at 'functionName' failed to satisfy constraint: Member must not be null")
+	}
+	if in.EventSourceArn == nil || *in.EventSourceArn == "" {
+		return nil, invalid("Unrecognized event source, must be kinesis, dynamodb stream or sqs.")
+	}
+	if in.Enabled != nil && !*in.Enabled {
+		return nil, unsupported("Enabled false")
+	}
+	if in.MaximumBatchingWindowInSeconds != nil && *in.MaximumBatchingWindowInSeconds != 0 {
+		return nil, unsupported("MaximumBatchingWindowInSeconds")
+	}
+	m := &mappingCreate{FunctionName: *in.FunctionName, QueueARN: *in.EventSourceArn, BatchSize: defaultBatchSize}
+	if in.BatchSize != nil {
+		if *in.BatchSize < 1 || *in.BatchSize > maxBatchSize {
+			return nil, invalid("BatchSize must be between 1 and %d for an SQS queue without a batching window.", maxBatchSize)
+		}
+		m.BatchSize = *in.BatchSize
+	}
+	return m, nil
+}
+
+// checkQueueARN refuses anything but a standard SQS queue in the caller's
+// own account and Region.
+func checkQueueARN(region, accountID, queueARN string) error {
+	parts := strings.Split(queueARN, ":")
+	if len(parts) != 6 || parts[0] != "arn" || parts[1] != "aws" {
+		return invalid("Unrecognized event source, must be kinesis, dynamodb stream or sqs.")
+	}
+	if parts[2] != "sqs" {
+		return unsupported("an event source other than SQS")
+	}
+	if parts[3] != region || parts[4] != accountID {
+		return unsupported("a queue in another account or Region")
+	}
+	if parts[5] == "" {
+		return invalid("Unrecognized event source, must be kinesis, dynamodb stream or sqs.")
+	}
+	if strings.HasSuffix(parts[5], ".fifo") {
+		return unsupported("a FIFO queue")
+	}
+	return nil
+}
+
+// mappingUUID derives one UUID per account, Region, function and queue, so the
+// keyed create itself refuses a duplicate with no inventory scan or second record.
+func mappingUUID(accountID, region, function, queueARN string) string {
+	b := sha256.Sum256([]byte(accountID + "\x00" + region + "\x00" + function + "\x00" + queueARN))
+	b[6] = b[6]&0x0f | 0x50
+	b[8] = b[8]&0x3f | 0x80
+	return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:16])
+}
+
+func mappingShape(m eventsource.Mapping, state string) *lambda.EventSourceMappingConfiguration {
+	return &lambda.EventSourceMappingConfiguration{
+		UUID:                           aws.String(m.UUID),
+		BatchSize:                      aws.Int64(m.BatchSize),
+		MaximumBatchingWindowInSeconds: aws.Int64(0),
+		EventSourceArn:                 aws.String(m.EventSourceARN),
+		FunctionArn:                    aws.String(m.FunctionARN),
+		LastModified:                   aws.Time(m.LastModified),
+		State:                          aws.String(state),
+		StateTransitionReason:          aws.String(m.StateTransitionReason),
+	}
+}
+
+func (d Deps) mappings(ctx context.Context) (MappingStore, error) {
+	if d.Mappings == nil {
+		return nil, serviceError("event-source mappings are unavailable")
+	}
+	return d.Mappings(ctx)
+}
+
+// createEventSourceMapping resolves the function, reads the queue as its
+// execution role, and records an Enabled mapping its poller picks up.
+func (d Deps) createEventSourceMapping(ctx context.Context, c *call) (reply, error) {
+	in, err := decodeMappingCreate(c.Body)
+	if err != nil {
+		return reply{}, err
+	}
+	if err := checkQueueARN(c.Region, c.AccountID, in.QueueARN); err != nil {
+		return reply{}, err
+	}
+	name, q, err := resolveFunctionName(c.Region, c.AccountID, in.FunctionName)
+	if err != nil {
+		return reply{}, err
+	}
+	if q != "" {
+		return reply{}, unsupported("a qualified FunctionName")
+	}
+	rec, _, err := d.Meta.Get(ctx, c.AccountID, c.Region, name)
+	if errors.Is(err, function.ErrNotFound) {
+		return reply{}, notFound(c.Region, c.AccountID, name)
+	}
+	if err != nil {
+		return reply{}, err
+	}
+	if d.Sources == nil {
+		return reply{}, serviceError("event-source mappings are unavailable")
+	}
+	visibility, err := d.Sources.QueueVisibilityTimeout(ctx, rec, in.QueueARN)
+	if err != nil {
+		return reply{}, err
+	}
+	if rec.Timeout > visibility {
+		return reply{}, invalid("Queue visibility timeout: %d seconds is less than Function timeout: %d seconds", visibility, rec.Timeout)
+	}
+	store, err := d.mappings(ctx)
+	if err != nil {
+		return reply{}, err
+	}
+	uuid := mappingUUID(c.AccountID, c.Region, name, in.QueueARN)
+	m := eventsource.Mapping{
+		FormatVersion: eventsource.FormatVersion, UUID: uuid, AccountID: c.AccountID, Region: c.Region,
+		FunctionName: name, FunctionARN: arn.FormatLambdaFunction(c.Region, c.AccountID, name),
+		EventSourceARN: in.QueueARN, BatchSize: in.BatchSize, State: eventsource.StateEnabled,
+		StateTransitionReason: eventsource.ReasonUserInitiated, LastModified: time.Now().UTC().Truncate(time.Millisecond),
+	}
+	if err := store.Create(ctx, m); err != nil {
+		if errors.Is(err, eventsource.ErrConflict) {
+			return reply{}, awserrors.Errorf(awserrors.ErrorResourceConflictException,
+				"An event source mapping with SQS arn (\" %s \") and function (\" %s \") already exists. Please update or delete the existing mapping with UUID %s",
+				in.QueueARN, name, uuid)
+		}
+		return reply{}, err
+	}
+	return reply{Status: http.StatusAccepted, Body: mappingShape(m, "Creating")}, nil
+}
+
+// ownMapping reads a mapping of the caller's account and Region.
+func (d Deps) ownMapping(ctx context.Context, c *call) (MappingStore, eventsource.Mapping, uint64, error) {
+	if !mappingUUIDRE.MatchString(c.Params[0]) {
+		return nil, eventsource.Mapping{}, 0, mappingNotFound()
+	}
+	store, err := d.mappings(ctx)
+	if err != nil {
+		return nil, eventsource.Mapping{}, 0, err
+	}
+	m, rev, err := store.Get(ctx, c.Params[0])
+	if errors.Is(err, eventsource.ErrNotFound) || (err == nil && (m.AccountID != c.AccountID || m.Region != c.Region)) {
+		return nil, eventsource.Mapping{}, 0, mappingNotFound()
+	}
+	return store, m, rev, err
+}
+
+func (d Deps) getEventSourceMapping(ctx context.Context, c *call) (reply, error) {
+	_, m, _, err := d.ownMapping(ctx, c)
+	if err != nil {
+		return reply{}, err
+	}
+	return reply{Status: http.StatusOK, Body: mappingShape(m, m.State)}, nil
+}
+
+// deleteEventSourceMapping records Deleting and returns; the poller stops its
+// worker and only then removes the record.
+func (d Deps) deleteEventSourceMapping(ctx context.Context, c *call) (reply, error) {
+	for range 3 {
+		store, m, rev, err := d.ownMapping(ctx, c)
+		if err != nil {
+			return reply{}, err
+		}
+		if m.State == eventsource.StateDeleting {
+			return reply{Status: http.StatusAccepted, Body: mappingShape(m, m.State)}, nil
+		}
+		m.State = eventsource.StateDeleting
+		m.StateTransitionReason = eventsource.ReasonUserInitiated
+		m.LastModified = time.Now().UTC().Truncate(time.Millisecond)
+		_, err = store.Update(ctx, m, rev)
+		if errors.Is(err, eventsource.ErrConflict) {
+			continue
+		}
+		if err != nil {
+			return reply{}, err
+		}
+		return reply{Status: http.StatusAccepted, Body: mappingShape(m, m.State)}, nil
+	}
+	return reply{}, serviceError("the mapping changed concurrently; retry the request")
+}
+
+// listEventSourceMappings is refused: the mapping store can only be listed
+// whole, and a request must not wait on the whole inventory.
+func (d Deps) listEventSourceMappings(_ context.Context, _ *call) (reply, error) {
+	return reply{}, invalid("ListEventSourceMappings is not supported in this build; use GetEventSourceMapping with the mapping UUID")
+}

spinifex/domains/lambda/awsapi/eventsource_test.go

@@ -0,0 +1,230 @@
+package awsapi_test
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"sync"
+	"testing"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/internal/testkit/gwtest"
+	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/eventsource"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type fakeSources struct {
+	mu         sync.Mutex
+	visibility int64
+	err        error
+	asRole     []string
+}
+
+func (f *fakeSources) QueueVisibilityTimeout(_ context.Context, rec function.Record, _ string) (int64, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.asRole = append(f.asRole, rec.Role)
+	return f.visibility, f.err
+}
+
+const mappingsPath = "/2015-03-31/event-source-mappings/"
+
+var queueARN = "arn:aws:sqs:" + gwtest.Region + ":" + gwtest.AccountID + ":orders"
+
+func newMappingEnv(t *testing.T, statements ...iamdomain.Statement) (*env, *eventsource.Store, *fakeSources) {
+	t.Helper()
+	e := newEnv(t, statements...)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) { b["Timeout"] = 30 }))
+	require.NoError(t, err)
+	_, _, js := testutil.StartTestJetStream(t)
+	store, err := eventsource.Open(t.Context(), js)
+	require.NoError(t, err)
+	src := &fakeSources{visibility: 30}
+	e.deps.Mappings = func(context.Context) (awsapi.MappingStore, error) { return store, nil }
+	e.deps.Sources = src
+	return e, store, src
+}
+
+func mappingBody(mutate func(map[string]any)) map[string]any {
+	b := map[string]any{"FunctionName": "hello", "EventSourceArn": queueARN}
+	if mutate != nil {
+		mutate(b)
+	}
+	return b
+}
+
+func decodeMapping(t *testing.T, raw []byte) map[string]any {
+	t.Helper()
+	var out map[string]any
+	require.NoError(t, json.Unmarshal(raw, &out))
+	return out
+}
+
+func TestEventSourceMapping_CreateGetDelete(t *testing.T) {
+	e, store, src := newMappingEnv(t)
+	rec, err := e.do(http.MethodPost, mappingsPath, mappingBody(func(b map[string]any) { b["BatchSize"] = 4 }))
+	require.NoError(t, err)
+	require.Equal(t, http.StatusAccepted, rec.Code)
+	created := decodeMapping(t, rec.Body.Bytes())
+	assert.Equal(t, "Creating", created["State"])
+	assert.EqualValues(t, 4, created["BatchSize"])
+	assert.Equal(t, queueARN, created["EventSourceArn"])
+	assert.Equal(t, []string{roleARN}, src.asRole, "the queue is read as the function's execution role")
+	uuid, _ := created["UUID"].(string)
+
+	stored, _, err := store.Get(t.Context(), uuid)
+	require.NoError(t, err)
+	assert.Equal(t, eventsource.StateEnabled, stored.State)
+
+	rec, err = e.do(http.MethodGet, mappingsPath+uuid, nil)
+	require.NoError(t, err)
+	assert.Equal(t, "Enabled", decodeMapping(t, rec.Body.Bytes())["State"])
+
+	_, err = e.do(http.MethodPost, mappingsPath, mappingBody(nil))
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
+
+	for range 2 {
+		rec, err = e.do(http.MethodDelete, mappingsPath+uuid, nil)
+		require.NoError(t, err)
+		require.Equal(t, http.StatusAccepted, rec.Code)
+		assert.Equal(t, "Deleting", decodeMapping(t, rec.Body.Bytes())["State"])
+	}
+	stored, _, err = store.Get(t.Context(), uuid)
+	require.NoError(t, err, "delete leaves the record for the poller to remove")
+	assert.Equal(t, eventsource.StateDeleting, stored.State)
+}
+
+func TestEventSourceMapping_ConcurrentDuplicateCreatesYieldOneMapping(t *testing.T) {
+	first, store, src := newMappingEnv(t)
+	const n = 8
+	gateways := []*env{first}
+	for len(gateways) < n {
+		e := newEnv(t)
+		_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) { b["Timeout"] = 30 }))
+		require.NoError(t, err)
+		e.deps.Mappings = first.deps.Mappings
+		e.deps.Sources = src
+		gateways = append(gateways, e)
+	}
+	codes := make(chan error, n)
+	var wg sync.WaitGroup
+	for _, e := range gateways {
+		wg.Go(func() {
+			_, err := e.do(http.MethodPost, mappingsPath, mappingBody(nil))
+			codes <- err
+		})
+	}
+	wg.Wait()
+	close(codes)
+	created := 0
+	for err := range codes {
+		if err == nil {
+			created++
+			continue
+		}
+		testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
+	}
+	assert.Equal(t, 1, created)
+	entries, err := store.List(t.Context())
+	require.NoError(t, err)
+	assert.Len(t, entries, 1)
+}
+
+func TestEventSourceMapping_ListIsRefusedWithoutReadingTheStore(t *testing.T) {
+	e, store, _ := newMappingEnv(t)
+	_, err := e.do(http.MethodPost, mappingsPath, mappingBody(nil))
+	require.NoError(t, err)
+	for _, path := range []string{mappingsPath, mappingsPath + "?FunctionName=hello", mappingsPath + "?MaxItems=2"} {
+		_, err = e.do(http.MethodGet, path, nil)
+		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+	}
+	entries, err := store.List(t.Context())
+	require.NoError(t, err)
+	assert.Len(t, entries, 1, "the refusal leaves the mapping in place")
+}
+
+func TestEventSourceMapping_RefusesWhatThisBuildDoesNotImplement(t *testing.T) {
+	e, store, _ := newMappingEnv(t)
+	cases := map[string]func(map[string]any){
+		"FIFO queue":      func(b map[string]any) { b["EventSourceArn"] = queueARN + ".fifo" },
+		"another account": func(b map[string]any) { b["EventSourceArn"] = "arn:aws:sqs:" + gwtest.Region + ":210987654321:orders" },
+		"another Region":  func(b map[string]any) { b["EventSourceArn"] = "arn:aws:sqs:us-east-1:" + gwtest.AccountID + ":orders" },
+		"not SQS": func(b map[string]any) {
+			b["EventSourceArn"] = "arn:aws:kinesis:" + gwtest.Region + ":" + gwtest.AccountID + ":stream/s"
+		},
+		"filter criteria":   func(b map[string]any) { b["FilterCriteria"] = map[string]any{"Filters": []any{}} },
+		"disabled":          func(b map[string]any) { b["Enabled"] = false },
+		"batching window":   func(b map[string]any) { b["MaximumBatchingWindowInSeconds"] = 5 },
+		"batch over 10":     func(b map[string]any) { b["BatchSize"] = 11 },
+		"batch zero":        func(b map[string]any) { b["BatchSize"] = 0 },
+		"qualified name":    func(b map[string]any) { b["FunctionName"] = "hello:1" },
+		"missing source":    func(b map[string]any) { delete(b, "EventSourceArn") },
+		"missing function":  func(b map[string]any) { delete(b, "FunctionName") },
+		"starting position": func(b map[string]any) { b["StartingPosition"] = "LATEST" },
+	}
+	for name, mutate := range cases {
+		t.Run(name, func(t *testing.T) {
+			_, err := e.do(http.MethodPost, mappingsPath, mappingBody(mutate))
+			testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+		})
+	}
+	entries, err := store.List(t.Context())
+	require.NoError(t, err)
+	assert.Empty(t, entries)
+}
+
+func TestEventSourceMapping_VisibilityTimeoutMustCoverTheFunctionTimeout(t *testing.T) {
+	e, store, src := newMappingEnv(t)
+	src.visibility = 29
+	_, err := e.do(http.MethodPost, mappingsPath, mappingBody(nil))
+	testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+
+	src.visibility, src.err = 0, awserrors.Errorf(awserrors.ErrorAccessDeniedException, "denied")
+	_, err = e.do(http.MethodPost, mappingsPath, mappingBody(nil))
+	testutil.RequireAWSCode(t, err, awserrors.ErrorAccessDeniedException)
+
+	entries, err := store.List(t.Context())
+	require.NoError(t, err)
+	assert.Empty(t, entries)
+
+	_, err = e.do(http.MethodPost, mappingsPath, mappingBody(func(b map[string]any) { b["FunctionName"] = "absent" }))
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+}
+
+func TestEventSourceMapping_AnotherAccountsMappingIsNotFound(t *testing.T) {
+	e, store, _ := newMappingEnv(t)
+	m := eventsource.Mapping{
+		FormatVersion: eventsource.FormatVersion, UUID: "0a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d", AccountID: "210987654321",
+		Region: gwtest.Region, FunctionName: "hello", EventSourceARN: "arn:aws:sqs:" + gwtest.Region + ":210987654321:orders",
+		BatchSize: 1, State: eventsource.StateEnabled,
+	}
+	require.NoError(t, store.Create(t.Context(), m))
+	_, err := e.do(http.MethodGet, mappingsPath+m.UUID, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+	_, err = e.do(http.MethodDelete, mappingsPath+m.UUID, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+	_, err = e.do(http.MethodGet, mappingsPath+"not-a-uuid", nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+}
+
+func TestEventSourceMapping_GetAndDeleteAuthorizeAgainstTheMappingARN(t *testing.T) {
+	const uuid = "0a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d"
+	mapping := "arn:aws:lambda:" + gwtest.Region + ":" + gwtest.AccountID + ":event-source-mapping:" + uuid
+	e, store, _ := newMappingEnv(t, gwtest.Statement("Allow", "*", "*"), gwtest.Statement("Deny", "lambda:*", mapping))
+	require.NoError(t, store.Create(t.Context(), eventsource.Mapping{
+		FormatVersion: eventsource.FormatVersion, UUID: uuid, AccountID: gwtest.AccountID, Region: gwtest.Region,
+		FunctionName: "hello", EventSourceARN: queueARN, BatchSize: 1, State: eventsource.StateEnabled,
+	}))
+	_, err := e.do(http.MethodGet, mappingsPath+uuid, nil)
+	gwtest.AssertDenied(t, err)
+	_, err = e.do(http.MethodDelete, mappingsPath+uuid, nil)
+	gwtest.AssertDenied(t, err)
+	m, _, err := store.Get(t.Context(), uuid)
+	require.NoError(t, err)
+	assert.Equal(t, eventsource.StateEnabled, m.State)
+}

spinifex/domains/lambda/awsapi/handler_test.go

@@ -765,8 +765,9 @@ func TestAuthorization(t *testing.T) {
 
 func TestRegistration_ResolvesActionsAndInventory(t *testing.T) {
 	reg := awsapi.NewRegistration(awsapi.Deps{})
-	assert.Equal(t, []string{"CreateFunction", "GetFunction", "GetFunctionCodeSigningConfig", "GetFunctionConfiguration", "Invoke", "ListVersionsByFunction"}, reg.Inventory.Registered)
+	assert.Equal(t, []string{"CreateEventSourceMapping", "CreateFunction", "DeleteEventSourceMapping", "GetEventSourceMapping", "GetFunction", "GetFunctionCodeSigningConfig", "GetFunctionConfiguration", "Invoke", "ListEventSourceMappings", "ListVersionsByFunction"}, reg.Inventory.Registered)
 	assert.Equal(t, "ListVersionsByFunction", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x/versions", nil)))
+	assert.Equal(t, "DeleteEventSourceMapping", reg.ResolveAction(httptest.NewRequest(http.MethodDelete, "/2015-03-31/event-source-mappings/x", nil)))
 	assert.Equal(t, "GetFunctionCodeSigningConfig", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2020-06-30/functions/x/code-signing-config", nil)))
 	assert.Equal(t, "CreateFunction", reg.ResolveAction(httptest.NewRequest(http.MethodPost, "/2015-03-31/functions", nil)))
 	assert.Equal(t, "GetFunction", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x", nil)))

spinifex/domains/lambda/awsapi/invoke.go

@@ -114,16 +114,32 @@ func (d Deps) invoke(ctx context.Context, c *call) (reply, error) {
 	if err != nil {
 		return reply{}, err
 	}
+	out, err := d.runSync(ctx, rec, revision, c.Invoke.Qualifier, c.Body)
+	if err != nil {
+		return reply{}, err
+	}
+	hdr := http.Header{}
+	hdr.Set("X-Amz-Executed-Version", out.ExecutedVersion)
+	if out.FunctionError != "" {
+		hdr.Set("X-Amz-Function-Error", out.FunctionError)
+	}
+	return reply{Status: http.StatusOK, Body: rawJSON(out.Payload), Header: hdr}, nil
+}
+
+// runSync runs one synchronous attempt on a resolved record and classifies
+// its result; every error is the AWS error Invoke returns. A nil error means
+// the result is valid, with or without a function error.
+func (d Deps) runSync(ctx context.Context, rec function.Record, revision uint64, qualifier string, payload []byte) (SyncResult, error) {
 	if rec.State != function.StateActive {
-		return reply{}, awserrors.Errorf(awserrors.ErrorResourceConflictException,
+		return SyncResult{}, awserrors.Errorf(awserrors.ErrorResourceConflictException,
 			"The operation cannot be performed at this time. The function is currently in the following state: %s", rec.State)
 	}
 	if d.Exec == nil || rec.Code.State != function.CodeStored {
-		return reply{}, serviceError("function execution is unavailable")
+		return SyncResult{}, serviceError("function execution is unavailable")
 	}
 
 	if err := ctx.Err(); err != nil {
-		return reply{}, err
+		return SyncResult{}, err
 	}
 	// Once accepted, a disconnecting caller does not cancel the attempt; only
 	// the wait is bounded, and a late result is discarded.
@@ -133,31 +149,37 @@ func (d Deps) invoke(ctx context.Context, c *call) (reply, error) {
 	out, err := d.Exec.InvokeSync(actx, SyncInvocation{
 		Function:  rec,
 		Revision:  revision,
-		Qualifier: c.Invoke.Qualifier,
-		Payload:   c.Body,
+		Qualifier: qualifier,
+		Payload:   payload,
 		WaitUntil: waitUntil,
 	})
 	if refusal, ok := errors.AsType[*RefusalError](err); ok {
 		if _, coded := awserrors.ResolveErrorCode(refusal.Err); coded {
-			return reply{}, refusal.Err
+			return SyncResult{}, refusal.Err
 		}
 		err = refusal
 	}
 	if err != nil {
 		slog.ErrorContext(ctx, "Lambda: synchronous attempt outcome unknown", "function_id", rec.FunctionID, "err", err)
-		return reply{}, serviceError("the invocation outcome is unknown")
+		return SyncResult{}, serviceError("the invocation outcome is unknown")
 	}
 	if out.ExecutedVersion == "" || len(out.Payload) > maxSyncPayload {
 		slog.ErrorContext(ctx, "Lambda: executor returned an invalid result", "function_id", rec.FunctionID,
 			"payload_bytes", len(out.Payload))
-		return reply{}, serviceError("the invocation outcome is unknown")
+		return SyncResult{}, serviceError("the invocation outcome is unknown")
 	}
-	hdr := http.Header{}
-	hdr.Set("X-Amz-Executed-Version", out.ExecutedVersion)
-	if out.FunctionError != "" {
-		hdr.Set("X-Amz-Function-Error", out.FunctionError)
+	return out, nil
+}
+
+// InvokeEventSource runs one event-source batch on $LATEST through the same
+// state, code and result checks as Invoke, without Invoke's IAM check: the
+// mapping was authorized when it was created.
+func (d Deps) InvokeEventSource(ctx context.Context, accountID, region, name string, payload []byte) (SyncResult, error) {
+	rec, revision, err := d.Meta.Get(ctx, accountID, region, name)
+	if err != nil {
+		return SyncResult{}, err
 	}
-	return reply{Status: http.StatusOK, Body: rawJSON(out.Payload), Header: hdr}, nil
+	return d.runSync(ctx, rec, revision, "", payload)
 }
 
 // checkInvokeOptions refuses every request option this build does not

spinifex/domains/lambda/awsapi/registration.go

@@ -54,6 +54,12 @@ type Deps struct {
 	AEAD cipher.AEAD
 	// Exec runs synchronous attempts; nil refuses every Invoke.
 	Exec Executor
+	// Mappings opens the event-source mapping store; nil refuses every
+	// mapping action.
+	Mappings func(ctx context.Context) (MappingStore, error)
+	// Sources reads a queue as a function's execution role; nil refuses
+	// CreateEventSourceMapping.
+	Sources SourceChecker
 }
 
 // NewRegistration returns the Lambda REST-JSON registration.
@@ -136,6 +142,8 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 		if name, _, err := resolveFunctionName(inv.Region, inv.AccountID, params[0]); err == nil {
 			resource = functionARN(inv.Region, inv.AccountID, name)
 		}
+	case "GetEventSourceMapping", "DeleteEventSourceMapping":
+		resource = mappingARN(inv.Region, inv.AccountID, params[0])
 	case "Invoke":
 		// Invoke validates deterministically before authorizing, and its
 		// resource comes from that same validated target. URL.Query drops
@@ -160,8 +168,7 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 		}
 	}
 	// Only CreateFunction and GetFunction touch the code store.
-	needsCodes := action != "GetFunctionConfiguration" && action != "GetFunctionCodeSigningConfig" &&
-		action != "ListVersionsByFunction" && action != "Invoke"
+	needsCodes := action == "CreateFunction" || action == "GetFunction"
 	if d.Meta == nil || (d.Codes == nil && needsCodes) {
 		return errors.New(awserrors.ErrorServiceException)
 	}

spinifex/domains/lambda/awsapi/routes.go

@@ -66,6 +66,30 @@ var routes = []route{
 		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
 			return d.invoke(ctx, c)
 		}},
+	{Method: "POST", Pattern: "/2015-03-31/event-source-mappings/", Action: "CreateEventSourceMapping",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.createEventSourceMapping(ctx, c)
+		}},
+	{Method: "POST", Pattern: "/2015-03-31/event-source-mappings", Action: "CreateEventSourceMapping",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.createEventSourceMapping(ctx, c)
+		}},
+	{Method: "GET", Pattern: "/2015-03-31/event-source-mappings/", Action: "ListEventSourceMappings",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.listEventSourceMappings(ctx, c)
+		}},
+	{Method: "GET", Pattern: "/2015-03-31/event-source-mappings", Action: "ListEventSourceMappings",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.listEventSourceMappings(ctx, c)
+		}},
+	{Method: "GET", Pattern: "/2015-03-31/event-source-mappings/{UUID}", Action: "GetEventSourceMapping",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.getEventSourceMapping(ctx, c)
+		}},
+	{Method: "DELETE", Pattern: "/2015-03-31/event-source-mappings/{UUID}", Action: "DeleteEventSourceMapping",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.deleteEventSourceMapping(ctx, c)
+		}},
 }
 
 var router = rest.NewRouter("lambda", routes)

spinifex/domains/lambda/eventsource/mapping.go

@@ -0,0 +1,170 @@
+// Package eventsource holds the Lambda event-source mapping record and its
+// JetStream KV store.
+package eventsource
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// FormatVersion is the only record layout this build reads or writes.
+const FormatVersion = 1
+
+// KVBucket holds every mapping, keyed by UUID; the UUID is global in AWS.
+const KVBucket = "lambda-event-source-mappings"
+
+// Mapping states this build records. A mapping is Enabled from creation and
+// Deleting from the delete request until its poller has stopped.
+const (
+	StateEnabled  = "Enabled"
+	StateDeleting = "Deleting"
+)
+
+// ReasonUserInitiated is the transition reason of a state the caller asked for.
+const ReasonUserInitiated = "USER_INITIATED"
+
+// ErrNotFound is returned for a UUID with no mapping.
+var ErrNotFound = errors.New("eventsource: mapping not found")
+
+// ErrConflict is returned when the record changed since it was read.
+var ErrConflict = errors.New("eventsource: mapping changed concurrently")
+
+// Mapping is one SQS event-source mapping.
+type Mapping struct {
+	FormatVersion  int    `json:"formatVersion"`
+	UUID           string `json:"uuid"`
+	AccountID      string `json:"accountId"`
+	Region         string `json:"region"`
+	FunctionName   string `json:"functionName"`
+	FunctionARN    string `json:"functionArn"`
+	EventSourceARN string `json:"eventSourceArn"`
+	BatchSize      int64  `json:"batchSize"`
+	State          string `json:"state"`
+	// StateTransitionReason is USER_INITIATED, or the last receive failure
+	// prefixed "PROBLEM: " while polling cannot proceed.
+	StateTransitionReason string    `json:"stateTransitionReason"`
+	LastModified          time.Time `json:"lastModified"`
+}
+
+// Store reads and writes mappings in KVBucket.
+type Store struct{ kv jetstream.KeyValue }
+
+// Open returns the store, creating its bucket on first use.
+func Open(ctx context.Context, js jetstream.KeyValueManager) (*Store, error) {
+	kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucket, 1)
+	if err != nil {
+		return nil, fmt.Errorf("eventsource: open bucket: %w", err)
+	}
+	return &Store{kv: kv}, nil
+}
+
+func key(uuid string) string { return "esm." + uuid }
+
+// Create writes a new mapping; a UUID already present is a conflict.
+func (s *Store) Create(ctx context.Context, m Mapping) error {
+	b, err := json.Marshal(m)
+	if err != nil {
+		return err
+	}
+	if _, err := s.kv.Create(ctx, key(m.UUID), b); err != nil {
+		if errors.Is(err, jetstream.ErrKeyExists) {
+			return ErrConflict
+		}
+		return fmt.Errorf("eventsource: create: %w", err)
+	}
+	return nil
+}
+
+// Get returns the mapping and its KV revision.
+func (s *Store) Get(ctx context.Context, uuid string) (Mapping, uint64, error) {
+	e, err := s.kv.Get(ctx, key(uuid))
+	if errors.Is(err, jetstream.ErrKeyNotFound) {
+		return Mapping{}, 0, ErrNotFound
+	}
+	if err != nil {
+		return Mapping{}, 0, fmt.Errorf("eventsource: get: %w", err)
+	}
+	m, err := decode(e.Value())
+	return m, e.Revision(), err
+}
+
+// Entry is one listed mapping with the revision it was read at.
+type Entry struct {
+	Mapping  Mapping
+	Revision uint64
+}
+
+// List returns every mapping in the bucket once.
+func (s *Store) List(ctx context.Context) ([]Entry, error) {
+	lister, err := s.kv.ListKeys(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("eventsource: list: %w", err)
+	}
+	defer func() { _ = lister.Stop() }()
+	var out []Entry
+	// A KV listing may repeat a key that changes while it is being listed.
+	seen := map[string]bool{}
+	for k := range lister.Keys() {
+		if seen[k] {
+			continue
+		}
+		seen[k] = true
+		e, err := s.kv.Get(ctx, k)
+		if errors.Is(err, jetstream.ErrKeyNotFound) {
+			continue
+		}
+		if err != nil {
+			return nil, fmt.Errorf("eventsource: list get: %w", err)
+		}
+		m, err := decode(e.Value())
+		if err != nil {
+			return nil, err
+		}
+		out = append(out, Entry{Mapping: m, Revision: e.Revision()})
+	}
+	return out, nil
+}
+
+// Update replaces the mapping only if it is still at revision.
+func (s *Store) Update(ctx context.Context, m Mapping, revision uint64) (uint64, error) {
+	b, err := json.Marshal(m)
+	if err != nil {
+		return 0, err
+	}
+	rev, err := s.kv.Update(ctx, key(m.UUID), b, revision)
+	if err != nil {
+		if errors.Is(err, jetstream.ErrKeyExists) {
+			return 0, ErrConflict
+		}
+		return 0, fmt.Errorf("eventsource: update: %w", err)
+	}
+	return rev, nil
+}
+
+// Remove deletes the mapping only if it is still at revision.
+func (s *Store) Remove(ctx context.Context, uuid string, revision uint64) error {
+	if err := s.kv.Delete(ctx, key(uuid), jetstream.LastRevision(revision)); err != nil {
+		if errors.Is(err, jetstream.ErrKeyExists) {
+			return ErrConflict
+		}
+		return fmt.Errorf("eventsource: remove: %w", err)
+	}
+	return nil
+}
+
+func decode(b []byte) (Mapping, error) {
+	var m Mapping
+	if err := json.Unmarshal(b, &m); err != nil {
+		return Mapping{}, fmt.Errorf("eventsource: decode: %w", err)
+	}
+	if m.FormatVersion != FormatVersion {
+		return Mapping{}, fmt.Errorf("eventsource: record format %d is not %d", m.FormatVersion, FormatVersion)
+	}
+	return m, nil
+}

spinifex/domains/lambda/eventsource/mapping_test.go

@@ -0,0 +1,42 @@
+package eventsource_test
+
+import (
+	"testing"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/eventsource"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestStore_RevisionsGuardEveryChange(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	s, err := eventsource.Open(t.Context(), js)
+	require.NoError(t, err)
+
+	entries, err := s.List(t.Context())
+	require.NoError(t, err)
+	assert.Empty(t, entries)
+
+	m := eventsource.Mapping{FormatVersion: eventsource.FormatVersion, UUID: "u1", State: eventsource.StateEnabled}
+	require.NoError(t, s.Create(t.Context(), m))
+	require.ErrorIs(t, s.Create(t.Context(), m), eventsource.ErrConflict)
+
+	got, rev, err := s.Get(t.Context(), "u1")
+	require.NoError(t, err)
+	assert.Equal(t, m, got)
+
+	m.State = eventsource.StateDeleting
+	newRev, err := s.Update(t.Context(), m, rev)
+	require.NoError(t, err)
+	_, err = s.Update(t.Context(), m, rev)
+	require.ErrorIs(t, err, eventsource.ErrConflict, "a stale revision never overwrites")
+	require.ErrorIs(t, s.Remove(t.Context(), "u1", rev), eventsource.ErrConflict, "a stale revision never removes")
+
+	require.NoError(t, s.Remove(t.Context(), "u1", newRev))
+	_, _, err = s.Get(t.Context(), "u1")
+	require.ErrorIs(t, err, eventsource.ErrNotFound)
+	entries, err = s.List(t.Context())
+	require.NoError(t, err)
+	assert.Empty(t, entries)
+}

spinifex/operator/cli/admin_eks.go

@@ -3,11 +3,16 @@ package cli
 import (
 	"context"
 	"fmt"
+	"maps"
 	"os"
+	"slices"
 	"time"
 
+	"github.com/aws/aws-sdk-go/service/ec2"
+	ec2inventory "github.com/mulgadc/spinifex/spinifex/domains/ec2/inventory"
 	ekscluster "github.com/mulgadc/spinifex/spinifex/domains/eks/cluster"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/spf13/cobra"
 )
 
@@ -75,3 +80,57 @@ func runAdminEksRestoreSnapshot(cmd *cobra.Command, _ []string) {
 	fmt.Printf("New control plane %s launched (snapshot %s)\n", out.NewInstanceID, out.Snapshot)
 	fmt.Printf("Status: %s\n", out.Status)
 }
+
+var adminEksOrphansCmd = &cobra.Command{
+	Use:   "orphans",
+	Short: "List running EKS control-plane VMs that have no cluster record",
+	Long: `orphans reports each running EKS control-plane VM whose cluster record, or whole
+account bucket, is missing (ORPHAN), and each it could not judge because a read
+failed (UNREADABLE). It only reads and never creates or removes anything.
+
+It lists the system account's instances through the strict inventory, so if any
+configured node does not answer it prints "listing incomplete" and exits non-zero
+without a partial list. It cannot say which node was silent. An instance with no
+ENI, or whose ENI is gone or not tagged as an EKS control plane, is skipped.
+
+Exit status is non-zero when the listing is incomplete or any row is ORPHAN or
+UNREADABLE.`,
+	Run: runAdminEksOrphans,
+}
+
+func init() {
+	adminEksCmd.AddCommand(adminEksOrphansCmd)
+}
+
+func runAdminEksOrphans(_ *cobra.Command, _ []string) {
+	cfg, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	defer nc.Close()
+
+	js, err := jetstream.New(nc)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	nodeIDs := slices.Sorted(maps.Keys(cfg.Nodes))
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	list := func(ctx context.Context) (*ec2.DescribeInstancesOutput, error) {
+		return ec2inventory.NewNATSInstances(nc, nodeIDs).ListAccountInstances(ctx, awsidentifiers.GlobalAccountID)
+	}
+	eni := ekscluster.ENITagReader(ec2inventory.NewENITagReader(js, awsidentifiers.GlobalAccountID))
+
+	orphans, unreadable, err := ekscluster.ReportControlPlaneOrphans(ctx, os.Stdout, js, list, eni, time.Now())
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	fmt.Printf("%d orphan(s), %d unreadable\n", orphans, unreadable)
+	if orphans+unreadable > 0 {
+		os.Exit(1)
+	}
+}

spinifex/runtime/compute/lambda/testlink/daemon.go

@@ -83,6 +83,9 @@ type Daemon struct {
 	MTU     int
 	WorkDir string
 	HTTP    *http.Client
+	// SDKEndpoint and SDKCA are public gateway connection settings for the guest.
+	SDKEndpoint string
+	SDKCA       []byte
 	// Image is the prepared runtime image every launch boots; the daemon
 	// prepares it once before serving and keeps it open for its whole life.
 	Image *Image
@@ -582,6 +585,10 @@ func (d *Daemon) prepare(req Request, code []byte, dir, tap, mac, ip string) (*v
 	entries["test-package-sha256"] = []byte(req.CodeSHA256)
 	entries["test-grant"] = grantEntry(req.Grant)
 	entries["test-handler"] = []byte(req.Handler)
+	if d.SDKEndpoint != "" && len(d.SDKCA) != 0 {
+		entries["test-sdk-endpoint"] = []byte(d.SDKEndpoint)
+		entries["test-sdk-ca"] = d.SDKCA
+	}
 	netcfgBytes := fmt.Appendf(nil, "NIC0_MAC=%s\nNIC0_CIDR=%s/24\nNIC0_DEFAULT=0\n", mac, ip)
 	if d.Firecracker != "" {
 		archive := make(map[string][]byte, len(entries)+1)

spinifex/runtime/compute/lambda/testlink/testlink_test.go

@@ -368,6 +368,36 @@ func TestPrepare_ScopeAndGrantEntryComeFromTheGrant(t *testing.T) {
 	require.Equal(t, "ap-southeast-2/123456789012/fn1/env-x/3", strings.TrimSpace(string(expect)))
 }
 
+func TestPrepare_DeliversSDKSettingsForEachVMM(t *testing.T) {
+	for _, firecracker := range []bool{false, true} {
+		t.Run(fmt.Sprintf("firecracker=%t", firecracker), func(t *testing.T) {
+			req := baseRequest("https://example.invalid/code", []byte("zip"))
+			req.Grant = execution.Grant{GrantID: "g-7", AttemptID: req.AttemptID, EnvironmentID: "env-x", Generation: 3, Node: testNode}
+			d := &Daemon{HostIP: "127.0.0.1", SDKEndpoint: "https://10.15.8.1:9999", SDKCA: []byte("public-ca")}
+			if firecracker {
+				d.Firecracker = "/test/firecracker"
+			}
+			dir := filepath.Join(t.TempDir(), "attempt")
+			_, _, ln, err := d.prepare(req, []byte("zip"), dir, "tap0", "02:00:00:00:00:01", "10.0.0.2")
+			require.NoError(t, err)
+			defer ln.Close()
+			endpointPath, caPath := filepath.Join(dir, "fwcfg-test-sdk-endpoint"), filepath.Join(dir, "fwcfg-test-sdk-ca")
+			if firecracker {
+				root := filepath.Join(t.TempDir(), "config")
+				extract(t, filepath.Join(dir, configArchive), root)
+				entryDir := filepath.Join(root, "by_name", lambdav1.FwCfgPrefix)
+				endpointPath, caPath = filepath.Join(entryDir, "test-sdk-endpoint", "raw"), filepath.Join(entryDir, "test-sdk-ca", "raw")
+			}
+			endpoint, err := os.ReadFile(endpointPath)
+			require.NoError(t, err)
+			ca, err := os.ReadFile(caPath)
+			require.NoError(t, err)
+			require.Equal(t, d.SDKEndpoint, string(endpoint))
+			require.Equal(t, d.SDKCA, ca)
+		})
+	}
+}
+
 // elfHeader is a bare 64-bit little-endian shared-object header for machine.
 func elfHeader(machine elf.Machine) []byte {
 	h := make([]byte, 64)

spinifex/runtime/daemon/daemon_lambda_testlink.go

@@ -7,6 +7,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/hex"
+	"encoding/pem"
 	"errors"
 	"fmt"
 	"io"
@@ -82,15 +83,21 @@ func (d *Daemon) startLambdaTestLink() {
 		return
 	}
 	httpClient, trustErr := d.lambdaTestLinkHTTP()
+	sdkEndpoint, sdkCA, sdkErr := d.lambdaTestLinkSDKConfig()
+	if sdkErr != nil {
+		slog.Warn("Lambda test link: SDK gateway settings unavailable", "err", sdkErr)
+	}
 	imageDir := filepath.Join(microvmDir, "lambda-testlink")
 	tl := &testlink.Daemon{
-		Node:    d.node,
-		Grants:  grants,
-		Taps:    taps,
-		IPs:     d.mgmtIPAllocator,
-		Bridge:  d.mgmtBridgeName(),
-		HostIP:  d.mgmtBridgeIP,
-		WorkDir: workDir,
+		Node:        d.node,
+		Grants:      grants,
+		Taps:        taps,
+		IPs:         d.mgmtIPAllocator,
+		Bridge:      d.mgmtBridgeName(),
+		HostIP:      d.mgmtBridgeIP,
+		WorkDir:     workDir,
+		SDKEndpoint: sdkEndpoint,
+		SDKCA:       sdkCA,
 
 		Initializer:       filepath.Join(imageDir, "hardkill.so"),
 		InitializerSHA256: lambdaHardKillSHA256,
@@ -135,7 +142,6 @@ func (d *Daemon) startLambdaTestLink() {
 // missed, by a crash or an earlier refusal, waits for the next pass.
 const lambdaActivationInterval = 30 * time.Second
 
-
 // lambdaTestLinkReady runs the serving checks and, only once they pass,
 // prepares the pinned image into tl; a refusal leaves tl.Image nil.
 func lambdaTestLinkReady(tl *testlink.Daemon, imageDir, imagePin string, trustErr error) error {
@@ -201,4 +207,32 @@ func (d *Daemon) lambdaTestLinkHTTP() (*http.Client, error) {
 	}}, nil
 }
 
+// lambdaTestLinkSDKConfig returns public settings for the guest's AWS SDK.
+func (d *Daemon) lambdaTestLinkSDKConfig() (string, []byte, error) {
+	if d.mgmtBridgeIP == "" || d.config.NATS.CACert == "" {
+		return "", nil, errors.New("management address or cluster CA not configured")
+	}
+	ca, err := os.ReadFile(d.config.NATS.CACert)
+	if err != nil {
+		return "", nil, errors.New("cluster CA unreadable")
+	}
+	if len(ca) > 64<<10 || !x509.NewCertPool().AppendCertsFromPEM(ca) {
+		return "", nil, errors.New("cluster CA invalid or too large")
+	}
+	var public []byte
+	for rest := ca; len(rest) != 0; {
+		block, next := pem.Decode(rest)
+		if block == nil {
+			break
+		}
+		rest = next
+		if block.Type == "CERTIFICATE" {
+			if _, err := x509.ParseCertificate(block.Bytes); err == nil {
+				public = append(public, pem.EncodeToMemory(block)...)
+			}
+		}
+	}
+	return d.resolveSystemGatewayBaseURL(), public, nil
+}
+
 var _ testlink.Taps = (*host.OVSPlumber)(nil)

spinifex/runtime/daemon/daemon_mgmt_ip.go

@@ -154,8 +154,17 @@ func (a *MgmtIPAllocator) BindKV(jsManager *JetStreamManager, node string) {
 // an address is free cluster-wide on its own — and is refused otherwise.
 func (a *MgmtIPAllocator) Allocate(instanceID string) (string, error) {
 	a.mu.Lock()
+	// Addresses this node's own instances hold stay off the table even when the
+	// cluster record has lost their entries.
+	held := make([]MgmtIPEntry, 0, len(a.allocated))
+	for id, ip := range a.allocated {
+		held = append(held, MgmtIPEntry{IP: ip, InstanceID: id})
+	}
 	if ip, ok := a.allocated[instanceID]; ok {
 		a.mu.Unlock()
+		if other := heldByOther(held, ip, instanceID); other != "" {
+			return "", fmt.Errorf("mgmt IP %s for %s is also held locally by %s", ip, instanceID, other)
+		}
 		return ip, nil
 	}
 	jsManager, node, subnet := a.jsManager, a.node, a.subnet
@@ -171,29 +180,7 @@ func (a *MgmtIPAllocator) Allocate(instanceID string) (string, error) {
 	// CAS write that follows is a harmless no-op against the same revision.
 	var allocatedIP string
 	var mutateErr error
-	_, err := updateMgmtIPAMWithRetry(jsManager, subnet, func(r *MgmtIPRecord) {
-		if r.Subnet == "" {
-			r.Subnet = subnet
-		}
-
-		// Idempotent: an instance that already holds an address (allocated
-		// by this node earlier, or by another node before a restart lost
-		// the local cache) keeps it rather than taking a second one.
-		for _, e := range r.Allocated {
-			if e.InstanceID == instanceID {
-				allocatedIP = e.IP
-				return
-			}
-		}
-
-		ip, findErr := nextFreeIP(a.baseIP, a.rangeMin, a.rangeMax, r.Allocated)
-		if findErr != nil {
-			mutateErr = findErr
-			return
-		}
-		r.Allocated = append(r.Allocated, MgmtIPEntry{IP: ip, InstanceID: instanceID, Node: node})
-		allocatedIP = ip
-	}, true)
+	_, err := updateMgmtIPAMWithRetry(jsManager, subnet, a.allocMutate(instanceID, node, subnet, held, &allocatedIP, &mutateErr), true)
 	if err != nil {
 		return "", fmt.Errorf("mgmt IP CAS update for subnet %s: %w", subnet, err)
 	}
@@ -209,6 +196,16 @@ func (a *MgmtIPAllocator) Allocate(instanceID string) (string, error) {
 	return allocatedIP, nil
 }
 
+// heldByOther names a cached instance other than id holding ip, or "".
+func heldByOther(held []MgmtIPEntry, ip, id string) string {
+	for _, e := range held {
+		if e.IP == ip && e.InstanceID != id {
+			return e.InstanceID
+		}
+	}
+	return ""
+}
+
 // nextFreeIP scans rangeMin-rangeMax on the /24 rooted at baseIP, skipping
 // every IP already present in allocated regardless of which entry's Node it
 // belongs to — the range is shared
@@ -511,3 +508,37 @@ func (d *Daemon) primeMgmtNeighEntry(instanceID, mgmtIP, mgmtMAC string) {
 			"instanceId", instanceID, "mgmtIP", mgmtIP, "mgmtMAC", mgmtMAC, "bridge", bridge, "err", err)
 	}
 }
+
+// allocMutate returns the CAS callback for Allocate. Results go out through
+// ipOut and errOut because casUpdate's mutate has no return value.
+func (a *MgmtIPAllocator) allocMutate(instanceID, node, subnet string, held []MgmtIPEntry, ipOut *string, errOut *error) func(*MgmtIPRecord) {
+	return func(r *MgmtIPRecord) {
+		// Each CAS attempt starts clean, so a retry never returns an earlier attempt's result.
+		*ipOut, *errOut = "", nil
+		if r.Subnet == "" {
+			r.Subnet = subnet
+		}
+
+		// Idempotent: an instance that already holds an address (allocated
+		// by this node earlier, or by another node before a restart lost
+		// the local cache) keeps it rather than taking a second one.
+		for _, e := range r.Allocated {
+			if e.InstanceID == instanceID {
+				if other := heldByOther(held, e.IP, instanceID); other != "" {
+					*errOut = fmt.Errorf("mgmt IP %s recorded for %s is held locally by %s", e.IP, instanceID, other)
+					return
+				}
+				*ipOut = e.IP
+				return
+			}
+		}
+
+		ip, findErr := nextFreeIP(a.baseIP, a.rangeMin, a.rangeMax, append(held, r.Allocated...))
+		if findErr != nil {
+			*errOut = findErr
+			return
+		}
+		r.Allocated = append(r.Allocated, MgmtIPEntry{IP: ip, InstanceID: instanceID, Node: node})
+		*ipOut = ip
+	}
+}

spinifex/runtime/daemon/daemon_mgmt_ip_test.go

@@ -472,6 +472,76 @@ func TestMgmtIPAllocator_Allocate_IdempotentAcrossNodes(t *testing.T) {
 	}
 }
 
+// An instance whose cluster-record entry has gone (the record lost it while the
+// instance kept running) still holds its address on this node, so a new
+// allocation must not be handed that address.
+func TestMgmtIPAllocator_Allocate_SkipsLocallyHeldWhenRecordLost(t *testing.T) {
+	t.Parallel()
+	jsm := newTestMgmtJSM(t)
+
+	a, err := NewMgmtIPAllocator("10.41.8.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	a.BindKV(jsm, "node-1")
+	cleanupMgmtIPAM(t, jsm, a)
+
+	held, err := a.Allocate("i-system")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := jsm.clusterKV.Delete(context.Background(), mgmtIPAMKeyPrefix+a.subnet); err != nil {
+		t.Fatalf("drop record: %v", err)
+	}
+
+	got, err := a.Allocate("lambda-attempt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got == held {
+		t.Fatalf("new allocation reused %s, still held by i-system", got)
+	}
+	if again, err := a.Allocate("i-system"); err != nil || again != held {
+		t.Errorf("holder re-allocate = %q, %v; want %q", again, err, held)
+	}
+}
+
+// Allocate refuses a recorded entry whose address another local instance holds.
+func TestMgmtIPAllocator_Allocate_RefusesRecordedConflictWithLocalHolder(t *testing.T) {
+	t.Parallel()
+	jsm := newTestMgmtJSM(t)
+
+	a, err := NewMgmtIPAllocator("10.42.8.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	a.BindKV(jsm, "node-1")
+	cleanupMgmtIPAM(t, jsm, a)
+
+	held, err := a.Allocate("i-system")
+	if err != nil {
+		t.Fatal(err)
+	}
+	other, err := NewMgmtIPAllocator("10.42.8.2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	other.BindKV(jsm, "node-2")
+	if err := jsm.clusterKV.Delete(context.Background(), mgmtIPAMKeyPrefix+a.subnet); err != nil {
+		t.Fatalf("drop record: %v", err)
+	}
+	// The record now names the held address for a different instance.
+	other.Claim("lambda-attempt", held)
+
+	if got, err := a.Allocate("lambda-attempt"); err == nil {
+		t.Fatalf("Allocate returned %s, which i-system holds locally", got)
+	}
+	a.Claim("i-dup", held)
+	if got, err := a.Allocate("i-dup"); err == nil {
+		t.Fatalf("cached duplicate returned %s, which i-system holds locally", got)
+	}
+}
+
 // TestMgmtIPAllocator_Release_FreesAddressClusterWide proves Release's
 // effect is visible cluster-wide: a second allocator, not the one that
 // released the address, can immediately take it.
@@ -868,8 +938,9 @@ func TestMgmtIPAllocator_ClaimLetsTheNewHolderReleaseItLater(t *testing.T) {
 		t.Errorf("the holder could not release its own address: entry still %+v", entry)
 	}
 
-	// And the address is genuinely back, not merely unrecorded.
-	reused, err := node1.Allocate("i-next")
+	// And the address is genuinely back, not merely unrecorded. node-1's cache
+	// still names its dropped copy, so the holder that released it allocates.
+	reused, err := node2.Allocate("i-next")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1110,3 +1181,34 @@ func TestMgmtIPAllocator_ReleaseDropsLocalOnForeignHolderAndKVFailure(t *testing
 		t.Errorf("local entries after release with KV down = %d, want 0", node3.AllocatedCount())
 	}
 }
+
+// A CAS retry that sees the conflicting entry gone must report its own
+// result, not the refusal left by the earlier attempt.
+func TestMgmtIPAllocator_AllocMutate_RetryClearsEarlierConflict(t *testing.T) {
+	a, err := NewMgmtIPAllocator("10.30.8.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	held := []MgmtIPEntry{{IP: "10.30.8.10", InstanceID: "i-sqs"}}
+	var ip string
+	var mutateErr error
+	mutate := a.allocMutate("i-fn", "node1", "10.30.8.0/24", held, &ip, &mutateErr)
+
+	mutate(&MgmtIPRecord{Allocated: []MgmtIPEntry{{IP: "10.30.8.10", InstanceID: "i-fn", Node: "node1"}}})
+	if mutateErr == nil || ip != "" {
+		t.Fatalf("conflicted attempt: ip=%q err=%v, want refusal", ip, mutateErr)
+	}
+
+	clean := &MgmtIPRecord{}
+	mutate(clean)
+	if mutateErr != nil {
+		t.Fatalf("retry kept the earlier refusal: %v", mutateErr)
+	}
+	if ip == "" || ip == "10.30.8.10" {
+		t.Fatalf("retry allocated %q, want a free address other than the held one", ip)
+	}
+	want := MgmtIPEntry{IP: ip, InstanceID: "i-fn", Node: "node1"}
+	if len(clean.Allocated) != 1 || clean.Allocated[0] != want {
+		t.Fatalf("record = %+v, want [%+v]", clean.Allocated, want)
+	}
+}

spinifex/runtime/daemon/lambda_sdk_config_test.go

@@ -0,0 +1,44 @@
+//go:build lambdatestlink
+
+//test:in-package exercises daemon-owned gateway configuration
+package daemon
+
+import (
+	"encoding/pem"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLambdaSDKConfigUsesManagementGatewayAndClusterCA(t *testing.T) {
+	server := httptest.NewTLSServer(nil)
+	defer server.Close()
+	ca := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw})
+	path := filepath.Join(t.TempDir(), "ca.pem")
+	require.NoError(t, os.WriteFile(path, ca, 0o600))
+	d := &Daemon{mgmtBridgeIP: "10.15.8.1", config: &config.Config{
+		NATS: config.NATSConfig{CACert: path}, AWSGW: config.AWSGWConfig{Host: "0.0.0.0:9999"},
+	}}
+	endpoint, gotCA, err := d.lambdaTestLinkSDKConfig()
+	require.NoError(t, err)
+	assert.Equal(t, "https://10.15.8.1:9999", endpoint)
+	assert.Equal(t, ca, gotCA)
+
+	withKey := append(append([]byte{}, ca...), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: []byte("private")})...)
+	withKey = append(withKey, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: []byte("not a certificate")})...)
+	require.NoError(t, os.WriteFile(path, withKey, 0o600))
+	_, gotCA, err = d.lambdaTestLinkSDKConfig()
+	require.NoError(t, err)
+	assert.Equal(t, ca, gotCA, "only public certificates reach the guest")
+
+	require.NoError(t, os.WriteFile(path, []byte("bad CA"), 0o600))
+	_, _, err = d.lambdaTestLinkSDKConfig()
+	require.Error(t, err)
+	_, err = d.lambdaTestLinkHTTP()
+	require.Error(t, err, "configured invalid download trust must still refuse serving")
+}

spinifex/runtime/roles/awsgw/awsgw.go

@@ -393,7 +393,9 @@ func launchService(config *config.ClusterConfig) error {
 	}
 	// Built but routed only in test-link builds: Lambda stays out of the public
 	// router until its runtime image exists.
-	lambdaControlPlane := lambdaawsapi.NewRegistration(lambdaawsapi.Deps{
+	lambdaMappings := lambdaMappingStore(natsConn)
+	lambdaEventSources := newESMRuntime(lambdaMappings, stsService, nodeConfig.Region)
+	lambdaDeps := lambdaawsapi.Deps{
 		Meta: lambdafunction.NewNATSClient(natsConn),
 		Codes: func(ctx context.Context) (lambdaawsapi.CodeStore, error) {
 			store, err := lambdaCodeStore.Store(ctx)
@@ -406,7 +408,12 @@ func launchService(config *config.ClusterConfig) error {
 		Trusts: lambdaRoleTrusts(iamService),
 		AEAD:   lambdaKey.AEAD,
 		Exec:   lambdaTestExecutor(natsConn, config.Node, lambdaCodeStore, stsService),
-	})
+		Mappings: func(ctx context.Context) (lambdaawsapi.MappingStore, error) {
+			return lambdaMappings(ctx)
+		},
+		Sources: lambdaEventSources.sources(),
+	}
+	lambdaControlPlane := lambdaawsapi.NewRegistration(lambdaDeps)
 
 	// ECR auth bridge: load (or first-run create) the ES256 signing key from the
 	// cluster-replicated awsgw-keys KV bucket, then build the token issuer
@@ -779,6 +786,7 @@ func launchService(config *config.ClusterConfig) error {
 	go quotaSvc.RunBedrockRPMSync(janitorCtx, js)
 
 	handler := gw.SetupRoutes()
+	lambdaEventSources.start(janitorCtx, handler, lambdaDeps, sqsEndpoint(registryHost, registryPort))
 
 	// Load TLS certificate
 	cert, err := tls.LoadX509KeyPair(nodeConfig.AWSGW.TLSCert, nodeConfig.AWSGW.TLSKey)

spinifex/runtime/roles/awsgw/lambda_environment_role.go

@@ -4,7 +4,10 @@ import (
 	"context"
 
 	"github.com/aws/aws-sdk-go/service/sts"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/eventsource"
 	stsdomain "github.com/mulgadc/spinifex/spinifex/domains/sts"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
 )
 
 // lambdaEnvironmentRoleIssuer is the in-process STS entry point the test-link
@@ -15,3 +18,14 @@ type lambdaEnvironmentRoleIssuer interface {
 }
 
 var _ lambdaEnvironmentRoleIssuer = (*stsdomain.STSServiceImpl)(nil)
+
+// lambdaMappingStore opens the event-source mapping bucket on first use.
+func lambdaMappingStore(nc *nats.Conn) func(ctx context.Context) (*eventsource.Store, error) {
+	return func(ctx context.Context) (*eventsource.Store, error) {
+		js, err := jetstream.New(nc)
+		if err != nil {
+			return nil, err
+		}
+		return eventsource.Open(ctx, js)
+	}
+}

spinifex/runtime/roles/awsgw/lambda_testlink_esm.go

@@ -0,0 +1,593 @@
+//go:build lambdatestlink
+
+package awsgw
+
+import (
+	"context"
+	"crypto/rand"
+	"crypto/tls"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/sqs"
+	"github.com/aws/aws-sdk-go/service/sqs/sqsiface"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/eventsource"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	stsdomain "github.com/mulgadc/spinifex/spinifex/domains/sts"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// Poller timing. The receive is a long poll, so an idle queue costs one
+// request per esmWaitSeconds; a session is reissued well before it expires.
+const (
+	esmWaitSeconds    = 20
+	esmReconcileEvery = 5 * time.Second
+	esmMaxBackoff     = 60 * time.Second
+	esmSessionSeconds = 900
+	esmRenewBefore    = 10 * time.Minute
+	esmDeleteTimeout  = 15 * time.Second
+	// esmMaxEvent is the synchronous request bound one event must fit.
+	esmMaxEvent = 6 * 1024 * 1024
+	// esmMaxReason bounds a recorded failure reason.
+	esmMaxReason = 512
+)
+
+// esmInvoker runs one event through Invoke's checks, without Invoke's IAM check.
+type esmInvoker interface {
+	InvokeEventSource(ctx context.Context, accountID, region, name string, payload []byte) (awsapi.SyncResult, error)
+}
+
+var _ esmInvoker = awsapi.Deps{}
+
+// esmStore is what the poller needs of the mapping store.
+type esmStore interface {
+	Get(ctx context.Context, uuid string) (eventsource.Mapping, uint64, error)
+	List(ctx context.Context) ([]eventsource.Entry, error)
+	Update(ctx context.Context, m eventsource.Mapping, revision uint64) (uint64, error)
+	Remove(ctx context.Context, uuid string, revision uint64) error
+}
+
+var _ esmStore = (*eventsource.Store)(nil)
+
+// esmRuntime polls every enabled SQS mapping on this gateway. It is one loop
+// per gateway with no cross-node lease, so it is correct on a single node only.
+type esmRuntime struct {
+	store    func(ctx context.Context) (esmStore, error)
+	sessions lambdaEnvironmentRoleIssuer
+	region   string
+	meta     function.Client
+	invoke   esmInvoker
+	// queue builds an SQS client signed as one session; nil means the
+	// gateway handler, set by start.
+	queue func(cred *credentials.Credentials) sqsiface.SQSAPI
+	now   func() time.Time
+	// interval and wait pace reconcile and back-off; zero uses the defaults.
+	interval time.Duration
+	backoff  time.Duration
+
+	mu      sync.Mutex
+	handler http.Handler
+	// endpoint is the public SQS base URL queue URLs are built on.
+	endpoint string
+}
+
+func newESMRuntime(open func(ctx context.Context) (*eventsource.Store, error), sessions lambdaEnvironmentRoleIssuer, region string) *esmRuntime {
+	store := func(ctx context.Context) (esmStore, error) { return open(ctx) }
+	return &esmRuntime{store: store, sessions: sessions, region: region, now: time.Now}
+}
+
+func (r *esmRuntime) sources() awsapi.SourceChecker { return r }
+
+// start runs the poller until ctx ends; deps is the composed Lambda control plane.
+func (r *esmRuntime) start(ctx context.Context, h http.Handler, deps awsapi.Deps, sqsEndpoint string) {
+	r.mu.Lock()
+	r.handler = h
+	r.endpoint = sqsEndpoint
+	if r.queue == nil {
+		r.queue = r.handlerQueue
+	}
+	r.meta = deps.Meta
+	r.invoke = deps
+	r.mu.Unlock()
+	go r.run(ctx)
+}
+
+// handlerTransport carries a signed SDK request straight into the gateway
+// handler. It never leaves the TLS-terminating gateway, so it is marked as
+// received over TLS and is authorized exactly as one from a client.
+type handlerTransport struct{ h http.Handler }
+
+func (t handlerTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	in := req.Clone(req.Context())
+	in.TLS = &tls.ConnectionState{Version: tls.VersionTLS13, HandshakeComplete: true}
+	in.RemoteAddr = "127.0.0.1:0"
+	in.RequestURI = req.URL.RequestURI()
+	if in.Host == "" {
+		in.Host = req.URL.Host
+	}
+	rec := httptest.NewRecorder()
+	t.h.ServeHTTP(rec, in)
+	return rec.Result(), nil
+}
+
+func (r *esmRuntime) handlerQueue(cred *credentials.Credentials) sqsiface.SQSAPI {
+	r.mu.Lock()
+	h, endpoint := r.handler, r.endpoint
+	r.mu.Unlock()
+	sess := session.Must(session.NewSession(&aws.Config{
+		Region:      aws.String(r.region),
+		Endpoint:    aws.String(endpoint),
+		Credentials: cred,
+		HTTPClient:  &http.Client{Transport: handlerTransport{h: h}},
+		MaxRetries:  aws.Int(0),
+	}))
+	return sqs.New(sess)
+}
+
+// esmSession is one execution-role session the poller signs with.
+type esmSession struct {
+	cred    *credentials.Credentials
+	keyID   string
+	role    string
+	expires time.Time
+	scope   stsdomain.LambdaEnvironmentSessionScope
+}
+
+func (r *esmRuntime) issue(ctx context.Context, rec function.Record) (*esmSession, error) {
+	b := make([]byte, 6)
+	if _, err := rand.Read(b); err != nil {
+		return nil, err
+	}
+	scope := stsdomain.LambdaEnvironmentSessionScope{
+		AccountID:     rec.AccountID,
+		FunctionID:    rec.FunctionID,
+		FunctionARN:   arn.FormatLambdaFunction(rec.Region, rec.AccountID, rec.FunctionName),
+		EnvironmentID: "esm-" + hex.EncodeToString(b),
+	}
+	out, err := r.sessions.AssumeRoleForLambdaEnvironment(ctx, stsdomain.LambdaEnvironmentSessionRequest{
+		AccountID: scope.AccountID, RoleARN: rec.Role, FunctionARN: scope.FunctionARN,
+		FunctionID: scope.FunctionID, EnvironmentID: scope.EnvironmentID, MaxLifetimeSeconds: esmSessionSeconds,
+	})
+	if err != nil {
+		return nil, err
+	}
+	c := out.Credentials
+	return &esmSession{
+		cred: credentials.NewStaticCredentials(aws.StringValue(c.AccessKeyId), aws.StringValue(c.SecretAccessKey),
+			aws.StringValue(c.SessionToken)),
+		keyID: aws.StringValue(c.AccessKeyId), role: rec.Role, expires: aws.TimeValue(c.Expiration), scope: scope,
+	}, nil
+}
+
+func (r *esmRuntime) revoke(s *esmSession) {
+	if s == nil {
+		return
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), revokeTimeout)
+	defer cancel()
+	if err := r.sessions.RevokeLambdaEnvironmentSession(ctx, s.keyID, s.scope); err != nil {
+		slog.Warn("Lambda event source: poller session not revoked; it stays valid until expiry",
+			"environment_id", s.scope.EnvironmentID, "err", err)
+	}
+}
+
+// fresh returns s while it is the function's role and outlives need;
+// otherwise it issues a replacement and revokes s, which no request holds.
+func (r *esmRuntime) fresh(ctx context.Context, rec function.Record, s *esmSession, need time.Duration) (*esmSession, error) {
+	if s != nil && s.role == rec.Role && s.expires.Sub(r.now()) > need {
+		return s, nil
+	}
+	next, err := r.issue(ctx, rec)
+	if err != nil {
+		return s, err
+	}
+	r.revoke(s)
+	return next, nil
+}
+
+// eventClient renews *s before each event unless it outlives that event's
+// invocation and delete, so no acknowledgement runs on expired credentials.
+func (r *esmRuntime) eventClient(ctx context.Context, rec function.Record, s **esmSession) func() (sqsiface.SQSAPI, error) {
+	need := time.Duration(rec.Timeout)*time.Second + esmDeleteTimeout + time.Minute
+	return func() (sqsiface.SQSAPI, error) {
+		next, err := r.fresh(ctx, rec, *s, need)
+		*s = next
+		if err != nil {
+			return nil, err
+		}
+		return r.queue(next.cred), nil
+	}
+}
+
+func queueName(queueARN string) (account, name string) {
+	parts := strings.Split(queueARN, ":")
+	if len(parts) != 6 {
+		return "", ""
+	}
+	return parts[4], parts[5]
+}
+
+// queueURL builds the queue's URL rather than asking GetQueueUrl, which the
+// execution role is not required to allow.
+func (r *esmRuntime) queueURL(queueARN string) (string, error) {
+	r.mu.Lock()
+	endpoint := r.endpoint
+	r.mu.Unlock()
+	account, name := queueName(queueARN)
+	if endpoint == "" || name == "" {
+		return "", errors.New("SQS is not served on this gateway")
+	}
+	return endpoint + "/" + account + "/" + name, nil
+}
+
+// QueueVisibilityTimeout reads the queue as the function's execution role
+// with a short-lived session it revokes before returning.
+func (r *esmRuntime) QueueVisibilityTimeout(ctx context.Context, rec function.Record, queueARN string) (int64, error) {
+	s, err := r.issue(ctx, rec)
+	if err != nil {
+		if code, ok := awserrors.ResolveErrorCode(err); ok && code == awserrors.ErrorAccessDenied {
+			return 0, awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, "The role defined for the function cannot be assumed by Lambda.")
+		}
+		return 0, err
+	}
+	defer r.revoke(s)
+	q := r.queue(s.cred)
+	url, err := r.queueURL(queueARN)
+	if err != nil {
+		return 0, awserrors.Errorf(awserrors.ErrorServiceException, "%s", err.Error())
+	}
+	out, err := q.GetQueueAttributesWithContext(ctx, &sqs.GetQueueAttributesInput{
+		QueueUrl: aws.String(url), AttributeNames: aws.StringSlice([]string{sqs.QueueAttributeNameVisibilityTimeout}),
+	})
+	if err != nil {
+		return 0, sourceError(err)
+	}
+	var v int64
+	if _, err := fmt.Sscan(aws.StringValue(out.Attributes[sqs.QueueAttributeNameVisibilityTimeout]), &v); err != nil {
+		return 0, fmt.Errorf("queue visibility timeout: %w", err)
+	}
+	return v, nil
+}
+
+// sourceError reports a queue the execution role cannot read as the
+// caller's fault, as Lambda does.
+func sourceError(err error) error {
+	var ae awserr.Error
+	if !errors.As(err, &ae) {
+		return err
+	}
+	switch ae.Code() {
+	case "AccessDenied", "AccessDeniedException":
+		return awserrors.Errorf(awserrors.ErrorInvalidParameterValueException,
+			"The function execution role does not have permissions to call GetQueueAttributes on SQS")
+	case sqs.ErrCodeQueueDoesNotExist, "QueueDoesNotExist":
+		return awserrors.Errorf(awserrors.ErrorInvalidParameterValueException,
+			"Error occurred while ReceiveMessage. SQS Error Code: AWS.SimpleQueueService.NonExistentQueue. SQS Error Message: The specified queue does not exist.")
+	}
+	return err
+}
+
+type esmWorker struct {
+	cancel context.CancelFunc
+	done   chan struct{}
+}
+
+func (w *esmWorker) stop() {
+	w.cancel()
+	<-w.done
+}
+
+func (r *esmRuntime) run(ctx context.Context) {
+	workers := map[string]*esmWorker{}
+	defer func() {
+		for _, w := range workers {
+			w.stop()
+		}
+	}()
+	every := r.interval
+	if every == 0 {
+		every = esmReconcileEvery
+	}
+	t := time.NewTicker(every)
+	defer t.Stop()
+	for {
+		r.reconcile(ctx, workers)
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+		}
+	}
+}
+
+// reconcile starts a worker for each enabled mapping, including those
+// written before a restart, and finishes each delete only after its worker
+// has stopped, so no invoke or message delete follows it.
+func (r *esmRuntime) reconcile(ctx context.Context, workers map[string]*esmWorker) {
+	store, err := r.store(ctx)
+	if err != nil {
+		slog.Warn("Lambda event source: mapping store unavailable", "err", err)
+		return
+	}
+	entries, err := store.List(ctx)
+	if err != nil {
+		slog.Warn("Lambda event source: list mappings failed", "err", err)
+		return
+	}
+	seen := map[string]bool{}
+	for _, e := range entries {
+		m := e.Mapping
+		seen[m.UUID] = true
+		switch m.State {
+		case eventsource.StateEnabled:
+			if workers[m.UUID] == nil {
+				wctx, cancel := context.WithCancel(ctx)
+				w := &esmWorker{cancel: cancel, done: make(chan struct{})}
+				workers[m.UUID] = w
+				go func() {
+					defer close(w.done)
+					r.work(wctx, m)
+				}()
+			}
+		case eventsource.StateDeleting:
+			if w := workers[m.UUID]; w != nil {
+				w.stop()
+				delete(workers, m.UUID)
+			}
+			if err := r.finishDelete(ctx, store, m.UUID); err != nil {
+				slog.Warn("Lambda event source: mapping not removed; retrying", "uuid", m.UUID, "err", err)
+			}
+		}
+	}
+	for id, w := range workers {
+		if !seen[id] {
+			w.stop()
+			delete(workers, id)
+		}
+	}
+}
+
+func (r *esmRuntime) finishDelete(ctx context.Context, store esmStore, id string) error {
+	m, rev, err := store.Get(ctx, id)
+	if errors.Is(err, eventsource.ErrNotFound) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	if m.State != eventsource.StateDeleting {
+		return nil
+	}
+	return store.Remove(ctx, id, rev)
+}
+
+// setReason records a receive failure on an enabled mapping, or clears it
+// back to USER_INITIATED with reason "".
+func (r *esmRuntime) setReason(ctx context.Context, id, reason string) {
+	if reason == "" {
+		reason = eventsource.ReasonUserInitiated
+	} else {
+		reason = "PROBLEM: " + reason
+		if len(reason) > esmMaxReason {
+			reason = reason[:esmMaxReason]
+		}
+	}
+	store, err := r.store(ctx)
+	if err != nil {
+		return
+	}
+	m, rev, err := store.Get(ctx, id)
+	if err != nil || m.State != eventsource.StateEnabled || m.StateTransitionReason == reason {
+		return
+	}
+	m.StateTransitionReason = reason
+	if _, err := store.Update(ctx, m, rev); err != nil && !errors.Is(err, eventsource.ErrConflict) {
+		slog.Warn("Lambda event source: reason not recorded", "uuid", id, "err", err)
+	}
+}
+
+func (r *esmRuntime) wait(ctx context.Context, d time.Duration) {
+	t := time.NewTimer(d)
+	defer t.Stop()
+	select {
+	case <-ctx.Done():
+	case <-t.C:
+	}
+}
+
+// work polls one mapping until ctx ends. A failure keeps the mapping, records
+// why, and retries after a back-off.
+func (r *esmRuntime) work(ctx context.Context, m eventsource.Mapping) {
+	var s *esmSession
+	defer func() { r.revoke(s) }()
+	base := r.backoff
+	if base == 0 {
+		base = time.Second
+	}
+	backoff := base
+	fail := func(reason string) {
+		r.setReason(context.WithoutCancel(ctx), m.UUID, reason)
+		r.wait(ctx, backoff)
+		backoff = min(2*backoff, esmMaxBackoff)
+	}
+	for ctx.Err() == nil {
+		rec, _, err := r.meta.Get(ctx, m.AccountID, m.Region, m.FunctionName)
+		if err != nil {
+			fail("Function lookup failed: " + err.Error())
+			continue
+		}
+		next, err := r.fresh(ctx, rec, s, esmRenewBefore)
+		s = next
+		if err != nil {
+			fail("The execution role could not be assumed: " + err.Error())
+			continue
+		}
+		q := r.queue(s.cred)
+		url, err := r.queueURL(m.EventSourceARN)
+		if err != nil {
+			fail("Error occurred while ReceiveMessage: " + err.Error())
+			continue
+		}
+		out, err := q.ReceiveMessageWithContext(ctx, &sqs.ReceiveMessageInput{
+			QueueUrl:              aws.String(url),
+			MaxNumberOfMessages:   aws.Int64(m.BatchSize),
+			WaitTimeSeconds:       aws.Int64(esmWaitSeconds),
+			AttributeNames:        aws.StringSlice([]string{"All"}),
+			MessageAttributeNames: aws.StringSlice([]string{"All"}),
+		})
+		if err != nil {
+			if ctx.Err() == nil {
+				fail("Error occurred while ReceiveMessage: " + err.Error())
+			}
+			continue
+		}
+		r.setReason(ctx, m.UUID, "")
+		if err := r.process(ctx, m, r.eventClient(ctx, rec, &s), url, out.Messages); err != nil {
+			slog.Warn("Lambda event source: batch not processed; messages stay on the queue", "uuid", m.UUID, "err", err)
+			r.wait(ctx, backoff)
+			backoff = min(2*backoff, esmMaxBackoff)
+			continue
+		}
+		backoff = base
+	}
+}
+
+// sqsRecord and sqsAttribute are the AWS SQS event record shape.
+type sqsRecord struct {
+	MessageID         string                  `json:"messageId"`
+	ReceiptHandle     string                  `json:"receiptHandle"`
+	Body              string                  `json:"body"`
+	Attributes        map[string]string       `json:"attributes"`
+	MessageAttributes map[string]sqsAttribute `json:"messageAttributes"`
+	MD5OfBody         string                  `json:"md5OfBody"`
+	EventSource       string                  `json:"eventSource"`
+	EventSourceARN    string                  `json:"eventSourceARN"`
+	AWSRegion         string                  `json:"awsRegion"`
+}
+
+type sqsAttribute struct {
+	StringValue      *string  `json:"stringValue,omitempty"`
+	BinaryValue      []byte   `json:"binaryValue,omitempty"`
+	StringListValues []string `json:"stringListValues"`
+	BinaryListValues [][]byte `json:"binaryListValues"`
+	DataType         string   `json:"dataType"`
+}
+
+func eventRecord(m eventsource.Mapping, msg *sqs.Message) sqsRecord {
+	rec := sqsRecord{
+		MessageID: aws.StringValue(msg.MessageId), ReceiptHandle: aws.StringValue(msg.ReceiptHandle),
+		Body: aws.StringValue(msg.Body), Attributes: map[string]string{}, MessageAttributes: map[string]sqsAttribute{},
+		MD5OfBody: aws.StringValue(msg.MD5OfBody), EventSource: "aws:sqs", EventSourceARN: m.EventSourceARN, AWSRegion: m.Region,
+	}
+	for k, v := range msg.Attributes {
+		rec.Attributes[k] = aws.StringValue(v)
+	}
+	for k, v := range msg.MessageAttributes {
+		rec.MessageAttributes[k] = sqsAttribute{
+			StringValue: v.StringValue, BinaryValue: v.BinaryValue, StringListValues: []string{},
+			BinaryListValues: [][]byte{}, DataType: aws.StringValue(v.DataType),
+		}
+	}
+	return rec
+}
+
+// esmEvent is one invocation's payload and the messages it carries.
+type esmEvent struct {
+	Payload  []byte
+	Messages []*sqs.Message
+}
+
+// splitEvents groups messages into events whose encoding fits esmMaxEvent.
+// A message too large for an event of its own is returned in oversized and
+// is neither invoked nor deleted.
+func splitEvents(m eventsource.Mapping, msgs []*sqs.Message) (events []esmEvent, oversized []*sqs.Message) {
+	const frame = len(`{"Records":[]}`)
+	var recs []json.RawMessage
+	var batch []*sqs.Message
+	size := frame
+	flush := func() {
+		if len(recs) == 0 {
+			return
+		}
+		payload, _ := json.Marshal(struct {
+			Records []json.RawMessage `json:"Records"`
+		}{recs})
+		events = append(events, esmEvent{Payload: payload, Messages: batch})
+		recs, batch, size = nil, nil, frame
+	}
+	for _, msg := range msgs {
+		enc, err := json.Marshal(eventRecord(m, msg))
+		if err != nil || frame+len(enc) > esmMaxEvent {
+			oversized = append(oversized, msg)
+			continue
+		}
+		extra := len(enc)
+		if len(recs) > 0 {
+			extra++
+		}
+		if size+extra > esmMaxEvent {
+			flush()
+			extra = len(enc)
+		}
+		recs = append(recs, enc)
+		batch = append(batch, msg)
+		size += extra
+	}
+	flush()
+	return events, oversized
+}
+
+// process invokes each event and deletes its messages only on a classified
+// success with no function error; anything else leaves them to reappear.
+// process invokes and acknowledges each event in turn. It returns the first
+// failure, so the worker backs off before receiving again.
+func (r *esmRuntime) process(ctx context.Context, m eventsource.Mapping, client func() (sqsiface.SQSAPI, error), url string, msgs []*sqs.Message) error {
+	events, oversized := splitEvents(m, msgs)
+	if len(oversized) > 0 {
+		r.setReason(ctx, m.UUID, fmt.Sprintf("%d message(s) exceed the %d-byte event limit and are not delivered", len(oversized), esmMaxEvent))
+	}
+	for _, ev := range events {
+		// A stopped worker invokes nothing further; delete waits for it.
+		if ctx.Err() != nil {
+			return nil
+		}
+		q, err := client()
+		if err != nil {
+			return fmt.Errorf("execution role session: %w", err)
+		}
+		res, err := r.invoke.InvokeEventSource(ctx, m.AccountID, m.Region, m.FunctionName, ev.Payload)
+		if err != nil {
+			return err
+		}
+		if res.FunctionError != "" {
+			return fmt.Errorf("function error %s", res.FunctionError)
+		}
+		entries := make([]*sqs.DeleteMessageBatchRequestEntry, len(ev.Messages))
+		for i, msg := range ev.Messages {
+			entries[i] = &sqs.DeleteMessageBatchRequestEntry{Id: aws.String(fmt.Sprint(i)), ReceiptHandle: msg.ReceiptHandle}
+		}
+		dctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), esmDeleteTimeout)
+		out, err := q.DeleteMessageBatchWithContext(dctx, &sqs.DeleteMessageBatchInput{QueueUrl: aws.String(url), Entries: entries})
+		cancel()
+		if err != nil {
+			slog.Warn("Lambda event source: processed messages not deleted; they will be delivered again", "uuid", m.UUID, "err", err)
+		} else if len(out.Failed) > 0 {
+			slog.Warn("Lambda event source: some processed messages not deleted", "uuid", m.UUID, "failed", len(out.Failed))
+		}
+	}
+	return nil
+}

spinifex/runtime/roles/awsgw/lambda_testlink_esm_off.go

@@ -0,0 +1,23 @@
+//go:build !lambdatestlink
+
+package awsgw
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/eventsource"
+)
+
+// Without the test link no mapping is polled and CreateEventSourceMapping has
+// no queue reader, so it refuses; Lambda is unrouted there anyway.
+type esmRuntime struct{}
+
+func newESMRuntime(func(context.Context) (*eventsource.Store, error), lambdaEnvironmentRoleIssuer, string) *esmRuntime {
+	return &esmRuntime{}
+}
+
+func (*esmRuntime) sources() awsapi.SourceChecker { return nil }
+
+func (*esmRuntime) start(context.Context, http.Handler, awsapi.Deps, string) {}

spinifex/runtime/roles/awsgw/lambda_testlink_esm_test.go

@@ -0,0 +1,517 @@
+//go:build lambdatestlink
+
+//test:in-package — the poller, its session handling and its in-process
+// transport are unexported test-link internals.
+package awsgw
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/service/sqs"
+	"github.com/aws/aws-sdk-go/service/sqs/sqsiface"
+	"github.com/aws/aws-sdk-go/service/sts"
+	"github.com/mulgadc/bluebottle/pkg/iampolicy"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/eventsource"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	stsdomain "github.com/mulgadc/spinifex/spinifex/domains/sts"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/credential"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/testlink"
+	"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/policyauth"
+	"github.com/stretchr/testify/require"
+)
+
+const esmQueueARN = "arn:aws:sqs:ap-southeast-2:123456789012:in"
+
+// esmQueue is an SQS fake: Receive hands out queued messages once, Delete
+// records what was deleted.
+type esmQueue struct {
+	sqsiface.SQSAPI
+	mu         sync.Mutex
+	pending    []*sqs.Message
+	deleted    []string
+	receives   int
+	receiveErr error
+	urls       []string
+	// redeliver keeps received messages pending, as an expired visibility would.
+	redeliver bool
+}
+
+func (q *esmQueue) ReceiveMessageWithContext(ctx aws.Context, in *sqs.ReceiveMessageInput, _ ...request.Option) (*sqs.ReceiveMessageOutput, error) {
+	q.mu.Lock()
+	q.receives++
+	q.urls = append(q.urls, aws.StringValue(in.QueueUrl))
+	err := q.receiveErr
+	n := min(int(aws.Int64Value(in.MaxNumberOfMessages)), len(q.pending))
+	msgs := q.pending[:n]
+	if !q.redeliver {
+		q.pending = q.pending[n:]
+	}
+	q.mu.Unlock()
+	if err != nil {
+		return nil, err
+	}
+	if len(msgs) == 0 {
+		t := time.NewTimer(20 * time.Millisecond)
+		defer t.Stop()
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case <-t.C:
+		}
+	}
+	return &sqs.ReceiveMessageOutput{Messages: msgs}, nil
+}
+
+func (q *esmQueue) DeleteMessageBatchWithContext(_ aws.Context, in *sqs.DeleteMessageBatchInput, _ ...request.Option) (*sqs.DeleteMessageBatchOutput, error) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	for _, e := range in.Entries {
+		q.deleted = append(q.deleted, aws.StringValue(e.ReceiptHandle))
+	}
+	return &sqs.DeleteMessageBatchOutput{}, nil
+}
+
+func (q *esmQueue) state() (deleted []string, receives int) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	return append([]string(nil), q.deleted...), q.receives
+}
+
+// esmInvokes records each event and returns the configured outcome.
+type esmInvokes struct {
+	mu       sync.Mutex
+	payloads []string
+	result   awsapi.SyncResult
+	err      error
+	// started, when set, is closed as the first call begins, which then
+	// blocks until release is closed.
+	started, release chan struct{}
+	// during, when set, runs inside every invocation.
+	during func()
+}
+
+func (f *esmInvokes) InvokeEventSource(_ context.Context, _, _, _ string, payload []byte) (awsapi.SyncResult, error) {
+	f.mu.Lock()
+	f.payloads = append(f.payloads, string(payload))
+	started, release, during := f.started, f.release, f.during
+	f.started = nil
+	f.mu.Unlock()
+	if during != nil {
+		during()
+	}
+	if started != nil {
+		close(started)
+		<-release
+	}
+	return f.result, f.err
+}
+
+func (f *esmInvokes) count() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return len(f.payloads)
+}
+
+// esmSessions issues numbered sessions with a fixed lifetime.
+type esmSessions struct {
+	mu       sync.Mutex
+	life     time.Duration
+	issued   int
+	revoked  []string
+	issueErr error
+}
+
+func (f *esmSessions) AssumeRoleForLambdaEnvironment(_ context.Context, _ stsdomain.LambdaEnvironmentSessionRequest) (*sts.AssumeRoleOutput, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.issueErr != nil {
+		return nil, f.issueErr
+	}
+	f.issued++
+	id := "ASIA" + strings.Repeat("K", f.issued)
+	return &sts.AssumeRoleOutput{Credentials: &sts.Credentials{
+		AccessKeyId: aws.String(id), SecretAccessKey: aws.String("s"), SessionToken: aws.String("t"),
+		Expiration: aws.Time(time.Now().Add(f.life)),
+	}}, nil
+}
+
+func (f *esmSessions) RevokeLambdaEnvironmentSession(_ context.Context, keyID string, _ stsdomain.LambdaEnvironmentSessionScope) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.revoked = append(f.revoked, keyID)
+	return nil
+}
+
+type esmMeta struct{ function.Client }
+
+func (esmMeta) Get(_ context.Context, accountID, region, name string) (function.Record, uint64, error) {
+	return function.Record{FunctionID: "fn1", AccountID: accountID, Region: region, FunctionName: name,
+		Role: "arn:aws:iam::123456789012:role/fn-exec", Timeout: 3}, 1, nil
+}
+
+func esmMapping(id string) eventsource.Mapping {
+	return eventsource.Mapping{
+		FormatVersion: eventsource.FormatVersion, UUID: id, AccountID: "123456789012", Region: "ap-southeast-2",
+		FunctionName: "f", EventSourceARN: esmQueueARN, BatchSize: 10, State: eventsource.StateEnabled,
+		StateTransitionReason: eventsource.ReasonUserInitiated,
+	}
+}
+
+func msg(id, body string) *sqs.Message {
+	return &sqs.Message{MessageId: aws.String(id), ReceiptHandle: aws.String("rh-" + id), Body: aws.String(body)}
+}
+
+func newTestESM(t *testing.T, q *esmQueue, inv *esmInvokes) (*esmRuntime, *eventsource.Store, *esmSessions) {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	store, err := eventsource.Open(context.Background(), js)
+	require.NoError(t, err)
+	sessions := &esmSessions{life: time.Hour}
+	r := newESMRuntime(func(context.Context) (*eventsource.Store, error) { return store, nil }, sessions, "ap-southeast-2")
+	r.meta = esmMeta{}
+	r.invoke = inv
+	r.queue = func(*credentials.Credentials) sqsiface.SQSAPI { return q }
+	r.endpoint = "https://gw.example:9999"
+	r.interval = 20 * time.Millisecond
+	r.backoff = 10 * time.Millisecond
+	return r, store, sessions
+}
+
+// Verifies: ADR-0009:S16
+func TestESM_DeletesOnlyOnClassifiedSuccess(t *testing.T) {
+	cases := []struct {
+		name   string
+		result awsapi.SyncResult
+		err    error
+		delete bool
+	}{
+		{name: "success", result: awsapi.SyncResult{ExecutedVersion: "$LATEST", Payload: []byte(`{}`)}, delete: true},
+		{name: "function error", result: awsapi.SyncResult{ExecutedVersion: "$LATEST", FunctionError: "Unhandled"}},
+		{name: "refusal", err: &awsapi.RefusalError{Err: errors.New("node busy")}},
+		{name: "unknown outcome", err: errors.New("the invocation outcome is unknown")},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			q := &esmQueue{}
+			inv := &esmInvokes{result: tc.result, err: tc.err}
+			r, _, _ := newTestESM(t, q, inv)
+			err := r.process(context.Background(), esmMapping("u1"), func() (sqsiface.SQSAPI, error) { return q, nil }, "url",
+				[]*sqs.Message{msg("1", "a"), msg("2", "b")})
+			require.Equal(t, tc.delete, err == nil, "only a processed batch reports success")
+			require.Equal(t, 1, inv.count())
+			deleted, _ := q.state()
+			if tc.delete {
+				require.Equal(t, []string{"rh-1", "rh-2"}, deleted)
+			} else {
+				require.Empty(t, deleted)
+			}
+		})
+	}
+}
+
+func TestESM_EventShapeAndSizeSplit(t *testing.T) {
+	m := esmMapping("u1")
+	quoted := strings.Repeat(`"`, 700*1024) // each byte encodes as two
+	msgs := []*sqs.Message{}
+	for i := range 6 {
+		msgs = append(msgs, msg(string(rune('a'+i)), quoted))
+	}
+	events, oversized := splitEvents(m, msgs)
+	require.Empty(t, oversized)
+	require.Len(t, events, 2, "8.4 MiB of encoded records must split under the 6 MiB bound")
+	total := 0
+	for _, ev := range events {
+		require.LessOrEqual(t, len(ev.Payload), esmMaxEvent)
+		total += len(ev.Messages)
+	}
+	require.Equal(t, 6, total)
+	require.Contains(t, string(events[0].Payload), `"eventSource":"aws:sqs"`)
+	require.Contains(t, string(events[0].Payload), `"eventSourceARN":"`+esmQueueARN+`"`)
+	require.Contains(t, string(events[0].Payload), `"messageAttributes":{}`)
+
+	huge := strings.Repeat("\x00", esmMaxEvent/6+1) // each byte encodes as \u0000
+	events, oversized = splitEvents(m, []*sqs.Message{msg("big", huge), msg("ok", "small")})
+	require.Len(t, oversized, 1)
+	require.Equal(t, "big", aws.StringValue(oversized[0].MessageId))
+	require.Len(t, events, 1)
+
+	q := &esmQueue{}
+	inv := &esmInvokes{result: awsapi.SyncResult{ExecutedVersion: "$LATEST"}}
+	r, _, _ := newTestESM(t, q, inv)
+	require.NoError(t, r.process(context.Background(), m, func() (sqsiface.SQSAPI, error) { return q, nil }, "url", []*sqs.Message{msg("big", huge)}))
+	require.Zero(t, inv.count(), "an oversized message is never invoked")
+	deleted, _ := q.state()
+	require.Empty(t, deleted, "an oversized message is never deleted")
+}
+
+// A mapping written before the gateway starts is polled, and a Deleting
+// record is removed only after its worker stopped; nothing follows it.
+func TestESM_RestartReloadAndDeleteStopsWorker(t *testing.T) {
+	q := &esmQueue{pending: []*sqs.Message{msg("1", "hello")}}
+	inv := &esmInvokes{result: awsapi.SyncResult{ExecutedVersion: "$LATEST"}}
+	r, store, _ := newTestESM(t, q, inv)
+	ctx := t.Context()
+	require.NoError(t, store.Create(ctx, esmMapping("u1")))
+
+	go r.run(ctx)
+	require.Eventually(t, func() bool { d, _ := q.state(); return len(d) == 1 }, 5*time.Second, 10*time.Millisecond)
+	_, receives := q.state()
+	require.Positive(t, receives)
+	q.mu.Lock()
+	firstURL := q.urls[0]
+	q.mu.Unlock()
+	require.Equal(t, "https://gw.example:9999/123456789012/in", firstURL)
+
+	m, rev, err := store.Get(ctx, "u1")
+	require.NoError(t, err)
+	m.State = eventsource.StateDeleting
+	_, err = store.Update(ctx, m, rev)
+	require.NoError(t, err)
+	require.Eventually(t, func() bool {
+		_, _, err := store.Get(ctx, "u1")
+		return errors.Is(err, eventsource.ErrNotFound)
+	}, 5*time.Second, 10*time.Millisecond)
+
+	_, after := q.state()
+	q.mu.Lock()
+	q.pending = []*sqs.Message{msg("2", "late")}
+	q.mu.Unlock()
+	time.Sleep(200 * time.Millisecond)
+	deleted, receivesNow := q.state()
+	require.Equal(t, after, receivesNow, "no receive after the delete completed")
+	require.Equal(t, []string{"rh-1"}, deleted)
+	require.Equal(t, 1, inv.count())
+}
+
+// A delete requested while a batch is running completes only after that batch
+// finished, so its message delete precedes the record's removal.
+func TestESM_DeleteWaitsForInFlightBatch(t *testing.T) {
+	q := &esmQueue{pending: []*sqs.Message{msg("1", "hello")}}
+	inv := &esmInvokes{result: awsapi.SyncResult{ExecutedVersion: "$LATEST"},
+		started: make(chan struct{}), release: make(chan struct{})}
+	started := inv.started
+	r, store, _ := newTestESM(t, q, inv)
+	ctx := t.Context()
+	require.NoError(t, store.Create(ctx, esmMapping("u4")))
+	go r.run(ctx)
+	<-started
+
+	m, rev, err := store.Get(ctx, "u4")
+	require.NoError(t, err)
+	m.State = eventsource.StateDeleting
+	_, err = store.Update(ctx, m, rev)
+	require.NoError(t, err)
+	require.Never(t, func() bool {
+		_, _, err := store.Get(ctx, "u4")
+		return errors.Is(err, eventsource.ErrNotFound)
+	}, 300*time.Millisecond, 10*time.Millisecond, "the delete must wait for the running batch")
+
+	close(inv.release)
+	require.Eventually(t, func() bool {
+		_, _, err := store.Get(ctx, "u4")
+		return errors.Is(err, eventsource.ErrNotFound)
+	}, 5*time.Second, 10*time.Millisecond)
+	deleted, _ := q.state()
+	require.Equal(t, []string{"rh-1"}, deleted, "the finished batch's message was deleted before removal")
+}
+
+// A Deleting record left by a crash before the gateway restarted is removed.
+func TestESM_InterruptedDeleteResumes(t *testing.T) {
+	q := &esmQueue{}
+	r, store, _ := newTestESM(t, q, &esmInvokes{})
+	ctx := t.Context()
+	m := esmMapping("u2")
+	m.State = eventsource.StateDeleting
+	require.NoError(t, store.Create(ctx, m))
+	go r.run(ctx)
+	require.Eventually(t, func() bool {
+		_, _, err := store.Get(ctx, "u2")
+		return errors.Is(err, eventsource.ErrNotFound)
+	}, 5*time.Second, 10*time.Millisecond)
+	_, receives := q.state()
+	require.Zero(t, receives)
+}
+
+func TestESM_SessionReissuedBeforeExpiry(t *testing.T) {
+	r, _, sessions := newTestESM(t, &esmQueue{}, &esmInvokes{})
+	rec, _, _ := esmMeta{}.Get(context.Background(), "123456789012", "ap-southeast-2", "f")
+	sessions.life = esmRenewBefore + time.Minute
+	s, err := r.fresh(context.Background(), rec, nil, esmRenewBefore)
+	require.NoError(t, err)
+	same, err := r.fresh(context.Background(), rec, s, esmRenewBefore)
+	require.NoError(t, err)
+	require.Same(t, s, same, "a session far from expiry is kept")
+
+	r.now = func() time.Time { return time.Now().Add(2 * time.Minute) }
+	next, err := r.fresh(context.Background(), rec, s, esmRenewBefore)
+	require.NoError(t, err)
+	require.NotSame(t, s, next)
+	require.Equal(t, []string{s.keyID}, sessions.revoked, "the replaced session is revoked")
+
+	rec.Role = "arn:aws:iam::123456789012:role/other"
+	r.now = time.Now
+	third, err := r.fresh(context.Background(), rec, next, esmRenewBefore)
+	require.NoError(t, err)
+	require.NotSame(t, next, third, "a changed execution role gets a new session")
+}
+
+// Time passes during each invocation; the second event of one receive must
+// run and be acknowledged on a renewed session, not the one about to expire.
+func TestESM_EachSplitEventRunsOnACoveringSession(t *testing.T) {
+	q := &esmQueue{}
+	var mu sync.Mutex
+	now := time.Now()
+	inv := &esmInvokes{result: awsapi.SyncResult{ExecutedVersion: "$LATEST"}, during: func() {
+		mu.Lock()
+		now = now.Add(29 * time.Minute)
+		mu.Unlock()
+	}}
+	r, _, sessions := newTestESM(t, q, inv)
+	r.now = func() time.Time { mu.Lock(); defer mu.Unlock(); return now }
+	sessions.life = 30 * time.Minute
+	rec, _, _ := esmMeta{}.Get(context.Background(), "123456789012", "ap-southeast-2", "f")
+	big := strings.Repeat("a", esmMaxEvent/2)
+	var s *esmSession
+	t.Cleanup(func() { r.revoke(s) })
+	err := r.process(context.Background(), esmMapping("u1"), r.eventClient(context.Background(), rec, &s), "url",
+		[]*sqs.Message{msg("1", big), msg("2", big)})
+	require.NoError(t, err)
+	require.Equal(t, 2, inv.count(), "the two messages cannot share one event")
+	require.Equal(t, 2, sessions.issued, "the second event renews the session")
+	require.Equal(t, []string{"ASIAK"}, sessions.revoked)
+	deleted, _ := q.state()
+	require.Len(t, deleted, 2)
+}
+
+// A failed invocation makes the worker back off before it receives again.
+func TestESM_InvocationFailureBacksOff(t *testing.T) {
+	q := &esmQueue{pending: []*sqs.Message{msg("1", "hello")}, redeliver: true}
+	inv := &esmInvokes{err: errors.New("TooManyRequestsException: busy")}
+	r, store, _ := newTestESM(t, q, inv)
+	r.backoff = 300 * time.Millisecond
+	require.NoError(t, store.Create(context.Background(), esmMapping("u1")))
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() { defer close(done); r.run(ctx) }()
+	time.Sleep(time.Second)
+	cancel()
+	<-done
+	_, receives := q.state()
+	require.LessOrEqual(t, receives, 4, "a failing batch is retried after a back-off, not in a tight loop")
+	deleted, _ := q.state()
+	require.Empty(t, deleted)
+}
+
+func TestESM_ReceiveFailureRecordedAndCleared(t *testing.T) {
+	q := &esmQueue{receiveErr: errors.New("AWS.SimpleQueueService.NonExistentQueue: The specified queue does not exist.")}
+	r, store, _ := newTestESM(t, q, &esmInvokes{})
+	ctx, cancel := context.WithCancel(t.Context())
+	defer cancel()
+	require.NoError(t, store.Create(ctx, esmMapping("u3")))
+	done := make(chan struct{})
+	go func() { defer close(done); r.work(ctx, esmMapping("u3")) }()
+	require.Eventually(t, func() bool {
+		m, _, err := store.Get(ctx, "u3")
+		return err == nil && strings.HasPrefix(m.StateTransitionReason, "PROBLEM: ") &&
+			strings.Contains(m.StateTransitionReason, "NonExistentQueue")
+	}, 5*time.Second, 10*time.Millisecond)
+	m, _, err := store.Get(ctx, "u3")
+	require.NoError(t, err)
+	require.Equal(t, eventsource.StateEnabled, m.State, "a receive failure keeps the mapping")
+
+	q.mu.Lock()
+	q.receiveErr = nil
+	q.mu.Unlock()
+	require.Eventually(t, func() bool {
+		m, _, err := store.Get(ctx, "u3")
+		return err == nil && m.StateTransitionReason == eventsource.ReasonUserInitiated
+	}, 5*time.Second, 10*time.Millisecond)
+	cancel()
+	<-done
+}
+
+func TestESM_HandlerTransportCarriesSignedRequest(t *testing.T) {
+	var got *http.Request
+	var body string
+	h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		got = r
+		b, _ := io.ReadAll(r.Body)
+		body = string(b)
+		w.Header().Set("Content-Type", "application/x-amz-json-1.0")
+		_, _ = w.Write([]byte(`{"Attributes":{"VisibilityTimeout":"45"}}`))
+	})
+	r, _, _ := newTestESM(t, &esmQueue{}, &esmInvokes{})
+	r.handler = h
+	q := r.handlerQueue(credentials.NewStaticCredentials("ASIAX", "secret", "token"))
+	out, err := q.GetQueueAttributes(&sqs.GetQueueAttributesInput{QueueUrl: aws.String("https://gw.example:9999/123456789012/in"),
+		AttributeNames: aws.StringSlice([]string{"VisibilityTimeout"})})
+	require.NoError(t, err)
+	require.Equal(t, "45", aws.StringValue(out.Attributes["VisibilityTimeout"]))
+	require.Contains(t, got.Header.Get("Authorization"), "Credential=ASIAX/")
+	require.Contains(t, got.Header.Get("Authorization"), "/ap-southeast-2/sqs/aws4_request")
+	require.Equal(t, "token", got.Header.Get("X-Amz-Security-Token"))
+	require.Equal(t, "gw.example:9999", got.Host)
+	require.Contains(t, body, `"QueueUrl":"https://gw.example:9999/123456789012/in"`)
+	keys := policyauth.RequestConditionKeys(got, credential.Principal{})
+	require.Equal(t, "true", keys[iampolicy.KeySecureTransport], "a role requiring TLS must still authorize the poller")
+}
+
+// activeMeta serves the invocation record as an Active function with code.
+type activeMeta struct{ function.Client }
+
+func (activeMeta) Get(_ context.Context, _, _, _ string) (function.Record, uint64, error) {
+	rec := invocation(nil).Function
+	rec.State = function.StateActive
+	rec.Code.State = function.CodeStored
+	return rec, 1, nil
+}
+
+// noVersionExec returns a result with no ExecutedVersion, which Invoke rejects.
+type noVersionExec struct{}
+
+func (noVersionExec) InvokeSync(context.Context, awsapi.SyncInvocation) (awsapi.SyncResult, error) {
+	return awsapi.SyncResult{Payload: []byte(`{}`)}, nil
+}
+
+// Both batches run through Deps.InvokeEventSource: an invalid executor result
+// and a real executor refused by a node that already holds an attempt.
+func TestESM_InvokeEventSourceFailuresDeleteNothing(t *testing.T) {
+	store := grantStore(t)
+	held := &fakeControl{max: 1 << 20, err: errors.New("no reply")}
+	_, err := executor(held, nil, store).InvokeSync(context.Background(), invocation(nil))
+	require.Error(t, err)
+	busy := &fakeControl{max: 1 << 20, reply: &testlink.Reply{Outcome: testlink.OutcomeDone}}
+
+	cases := []struct {
+		name string
+		exec awsapi.Executor
+	}{
+		{name: "missing ExecutedVersion", exec: noVersionExec{}},
+		{name: "node busy", exec: executor(busy, nil, store)},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			q := &esmQueue{}
+			r, _, _ := newTestESM(t, q, nil)
+			r.invoke = awsapi.Deps{Meta: activeMeta{}, Exec: tc.exec}
+			err := r.process(context.Background(), esmMapping("u1"), func() (sqsiface.SQSAPI, error) { return q, nil }, "url",
+				[]*sqs.Message{msg("1", "a")})
+			require.Error(t, err)
+			deleted, _ := q.state()
+			require.Empty(t, deleted)
+		})
+	}
+	require.Empty(t, busy.sent, "the busy node is sent nothing")
+}