fix(lambda): validate environment names before reserved keys

feat(lambda): deliver function environment to test-link guests
feat(lambda): provide test-link SDK gateway settings (#1655 )
2026-11-06 19:01:10 +00:00 · 2026-11-06 22:43:08 +10:00 · 2026-11-06 22:36:31 +10:00 · 2026-11-06 21:43:50 +10:00 · 2026-11-06 21:37:27 +10:00 · 2026-11-06 14:54:18 +10:00
51 changed files with 3564 additions and 227 deletions
@@ -0,0 +1,65 @@
+//go:build lambdatestlink
+
+package main
+
+import (
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"net"
+	"net/url"
+	"os"
+	"path/filepath"
+)
+
+const (
+	sdkEndpointEntry = "test-sdk-endpoint"
+	sdkCAEntry       = "test-sdk-ca"
+	sdkBundlePath    = "/run/lambda-sdk/ca-bundle.pem"
+	systemBundlePath = "/etc/ssl/certs/ca-certificates.crt"
+)
+
+type sdkConfig struct {
+	endpoint string
+	ca       []byte
+}
+
+// readSDKConfig reads optional public settings before the boot config is sealed.
+func readSDKConfig(dir string) (*sdkConfig, error) {
+	endpoint, endpointErr := os.ReadFile(filepath.Join(dir, sdkEndpointEntry, "raw"))
+	ca, caErr := os.ReadFile(filepath.Join(dir, sdkCAEntry, "raw"))
+	if errors.Is(endpointErr, os.ErrNotExist) && errors.Is(caErr, os.ErrNotExist) {
+		return nil, nil
+	}
+	if endpointErr != nil || caErr != nil {
+		return nil, errors.New("incomplete SDK gateway settings")
+	}
+	u, err := url.Parse(string(endpoint))
+	if err != nil || u.Scheme != "https" || u.User != nil || u.Path != "" || u.RawQuery != "" || u.Fragment != "" ||
+		net.ParseIP(u.Hostname()) == nil || u.Port() == "" {
+		return nil, errors.New("invalid SDK gateway endpoint")
+	}
+	if len(ca) > 64<<10 || !x509.NewCertPool().AppendCertsFromPEM(ca) {
+		return nil, errors.New("invalid SDK gateway CA")
+	}
+	return &sdkConfig{endpoint: string(endpoint), ca: ca}, nil
+}
+
+// writeBundle combines the system roots with the cluster's public root.
+func (c *sdkConfig) writeBundle(path, systemPath string) error {
+	system, err := os.ReadFile(systemPath)
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("read system CA bundle: %w", err)
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		return fmt.Errorf("create SDK CA directory: %w", err)
+	}
+	if err := os.Chmod(filepath.Dir(path), 0o755); err != nil {
+		return fmt.Errorf("make SDK CA directory readable: %w", err)
+	}
+	bundle := append(append(system, '\n'), c.ca...)
+	if err := os.WriteFile(path, bundle, 0o644); err != nil {
+		return fmt.Errorf("write SDK CA bundle: %w", err)
+	}
+	return os.Chmod(path, 0o644)
+}
@@ -0,0 +1,106 @@
+//go:build lambdatestlink
+
+//test:in-package exercises the unexported boot configuration reader
+package main
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSDKConfig_OptionalAndInvalid(t *testing.T) {
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
+	defer server.Close()
+	ca := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw})
+	for _, tc := range []struct {
+		name     string
+		endpoint []byte
+		ca       []byte
+		wantErr  bool
+		want     bool
+	}{
+		{name: "absent"},
+		{name: "valid", endpoint: []byte(server.URL), ca: ca, want: true},
+		{name: "missing ca", endpoint: []byte(server.URL), wantErr: true},
+		{name: "missing endpoint", ca: ca, wantErr: true},
+		{name: "invalid endpoint", endpoint: []byte("http://127.0.0.1:1234"), ca: ca, wantErr: true},
+		{name: "invalid ca", endpoint: []byte(server.URL), ca: []byte("not a certificate"), wantErr: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			dir := t.TempDir()
+			for name, value := range map[string][]byte{sdkEndpointEntry: tc.endpoint, sdkCAEntry: tc.ca} {
+				if value != nil {
+					require.NoError(t, os.MkdirAll(filepath.Join(dir, name), 0o700))
+					require.NoError(t, os.WriteFile(filepath.Join(dir, name, "raw"), value, 0o600))
+				}
+			}
+			cfg, err := readSDKConfig(dir)
+			if tc.wantErr {
+				require.Error(t, err)
+				require.Nil(t, cfg)
+				return
+			}
+			require.NoError(t, err)
+			if !tc.want {
+				require.Nil(t, cfg)
+				return
+			}
+			bundle := filepath.Join(t.TempDir(), "sdk", "ca.pem")
+			require.NoError(t, cfg.writeBundle(bundle, filepath.Join(t.TempDir(), "absent")))
+			data, err := os.ReadFile(bundle)
+			require.NoError(t, err)
+			roots := x509.NewCertPool()
+			require.True(t, roots.AppendCertsFromPEM(data))
+			client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: roots}}}
+			resp, err := client.Get(cfg.endpoint)
+			require.NoError(t, err)
+			defer resp.Body.Close()
+			assert.Equal(t, http.StatusOK, resp.StatusCode)
+		})
+	}
+}
+
+func TestSDKConfig_BundleKeepsSystemRoots(t *testing.T) {
+	cluster := httptest.NewTLSServer(nil)
+	defer cluster.Close()
+	clusterCA := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cluster.Certificate().Raw})
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+	template := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test system root"},
+		NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour),
+		IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign}
+	der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
+	require.NoError(t, err)
+	systemCA := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+	dir := t.TempDir()
+	systemPath, bundlePath := filepath.Join(dir, "system.pem"), filepath.Join(dir, "sdk", "ca.pem")
+	require.NoError(t, os.WriteFile(systemPath, systemCA, 0o600))
+	require.NoError(t, (&sdkConfig{ca: clusterCA}).writeBundle(bundlePath, systemPath))
+	bundle, err := os.ReadFile(bundlePath)
+	require.NoError(t, err)
+	roots := x509.NewCertPool()
+	require.True(t, roots.AppendCertsFromPEM(bundle))
+	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: roots}}}
+	resp, err := client.Get(cluster.URL)
+	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
+	systemCert, err := x509.ParseCertificate(der)
+	require.NoError(t, err)
+	_, err = systemCert.Verify(x509.VerifyOptions{Roots: roots})
+	require.NoError(t, err)
+}
@@ -12,11 +12,14 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
+	"sort"
+	"strings"
 	"time"

 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/agent"
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/runtimeapi"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/lambdaenv"
 )

 const (
@@ -31,13 +34,17 @@ type testlink struct {
 	pkg            taskPackage
 	runtimeAddr    string
 	startBootstrap func(ctx context.Context, env []string) (*exec.Cmd, error)
+	caBundlePath   string
+	systemCAPath   string
 }

 func start(ctx context.Context) int {
 	t := testlink{
-		startup:     defaultStartup(),
-		pkg:         devTaskPackage(),
-		runtimeAddr: runtimeAddr,
+		startup:      defaultStartup(),
+		pkg:          devTaskPackage(),
+		runtimeAddr:  runtimeAddr,
+		caBundlePath: sdkBundlePath,
+		systemCAPath: systemBundlePath,
 		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
 			return agent.StartBootstrap(ctx, taskRoot+"/bootstrap", env, os.Stdout, os.Stderr)
 		},
@@ -53,6 +60,7 @@ func (t testlink) run(parent context.Context) (int, error) {
 	// The package values are fw_cfg entries, so read them before connect seals it.
 	spec, specErr := readPackageSpec(t.fwCfgDir)
 	grant, grantErr := readGrant(t.fwCfgDir)
+	sdk, sdkErr := readSDKConfig(t.fwCfgDir)
 	conn, scope, code, err := t.connect(parent)
 	if err != nil {
 		return code, errors.Join(err, specErr, grantErr)
@@ -85,6 +93,18 @@ func (t testlink) run(parent context.Context) (int, error) {
 	}
 	cred := *first.Credential
 	first.Credential = nil
+	if err := lambdaenv.Validate(cred.Environment); err != nil {
+		return exitRefused, errors.New("first frame carries an invalid function environment")
+	}
+	if sdkErr != nil {
+		slog.Warn("lambda-agent: SDK gateway settings unavailable", "err", sdkErr)
+	}
+	if sdk != nil {
+		if err := sdk.writeBundle(t.caBundlePath, t.systemCAPath); err != nil {
+			slog.Warn("lambda-agent: SDK gateway settings unavailable", "err", err)
+			sdk = nil
+		}
+	}
 	if err := t.pkg.install(spec); err != nil {
 		return exitRefused, fmt.Errorf("install package: %w", err)
 	}
@@ -100,7 +120,8 @@ func (t testlink) run(parent context.Context) (int, error) {

 	ctx, cancel := context.WithCancelCause(parent)
 	defer cancel(nil)
-	cmd, err := t.startBootstrap(ctx, bootstrapEnv(ln.Addr().String(), spec.handler, cred))
+	region, _, _ := strings.Cut(string(scope), "/")
+	cmd, err := t.startBootstrap(ctx, bootstrapEnv(ln.Addr().String(), spec.handler, cred, region, sdk, t.caBundlePath))
 	if err != nil {
 		return exitLink, fmt.Errorf("start bootstrap: %w", err)
 	}
@@ -124,18 +145,35 @@ func (t testlink) run(parent context.Context) (int, error) {
 	return exitClosed, nil
 }

-// bootstrapEnv is the bootstrap's whole environment: the runtime variables and
-// the execution role's three credential variables.
-func bootstrapEnv(runtimeAPI, handler string, cred hostlink.Credential) []string {
-	return []string{
-		"AWS_LAMBDA_RUNTIME_API=" + runtimeAPI,
-		"_HANDLER=" + handler,
-		"LAMBDA_TASK_ROOT=" + taskRoot,
-		"PATH=/bin:/sbin:/usr/bin",
-		"AWS_ACCESS_KEY_ID=" + cred.AccessKeyID,
-		"AWS_SECRET_ACCESS_KEY=" + cred.SecretAccessKey,
-		"AWS_SESSION_TOKEN=" + cred.SessionToken,
+// bootstrapEnv supplies the runtime, execution-role and SDK settings.
+func bootstrapEnv(runtimeAPI, handler string, cred hostlink.Credential, region string, sdk *sdkConfig, caPath string) []string {
+	vars := map[string]string{
+		"AWS_LAMBDA_RUNTIME_API": runtimeAPI,
+		"_HANDLER":               handler,
+		"LAMBDA_TASK_ROOT":       taskRoot,
+		"PATH":                   "/bin:/sbin:/usr/bin",
+		"AWS_ACCESS_KEY_ID":      cred.AccessKeyID,
+		"AWS_SECRET_ACCESS_KEY":  cred.SecretAccessKey,
+		"AWS_SESSION_TOKEN":      cred.SessionToken,
+		"AWS_REGION":             region,
+		"AWS_DEFAULT_REGION":     region,
 	}
+	if sdk != nil {
+		vars["AWS_ENDPOINT_URL"], vars["AWS_CA_BUNDLE"] = sdk.endpoint, caPath
+	}
+	for k, v := range cred.Environment {
+		vars[k] = v
+	}
+	keys := make([]string, 0, len(vars))
+	for k := range vars {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	env := make([]string, 0, len(keys))
+	for _, k := range keys {
+		env = append(env, k+"="+vars[k])
+	}
+	return env
 }

 // shutdownGrace is how long past the invocation deadline the agent waits for
@@ -7,8 +7,10 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
+	"encoding/pem"
 	"io"
 	"net/http"
+	"net/http/httptest"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -70,6 +72,7 @@ type bootstrapStarts struct {
 	sealed  bool
 	// handlerOut receives the _HANDLER the bootstrap process itself saw.
 	handlerOut string
+	caBundle   string
 	// env is the environment the agent gave the bootstrap.
 	env []string
 }
@@ -94,6 +97,7 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 	b.started = make(chan struct{}, 1)
 	b.task = t.TempDir()
 	b.handlerOut = filepath.Join(t.TempDir(), "handler")
+	b.caBundle = filepath.Join(t.TempDir(), "sdk", "ca-bundle.pem")
 	return testlink{
 		startup: g.startup(),
 		pkg: taskPackage{
@@ -102,7 +106,9 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 			mountDir:   func(string) error { return nil },
 			sealDir:    func(string) error { b.sealed = true; return nil },
 		},
-		runtimeAddr: "127.0.0.1:0",
+		runtimeAddr:  "127.0.0.1:0",
+		caBundlePath: b.caBundle,
+		systemCAPath: filepath.Join(t.TempDir(), "system-ca.pem"),
 		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
 			b.n++
 			b.env = env
@@ -176,10 +182,87 @@ func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
 		"AWS_ACCESS_KEY_ID=" + testCredential.AccessKeyID,
 		"AWS_SECRET_ACCESS_KEY=" + testCredential.SecretAccessKey,
 		"AWS_SESSION_TOKEN=" + testCredential.SessionToken,
+		"AWS_REGION=ap-southeast-2",
+		"AWS_DEFAULT_REGION=ap-southeast-2",
 	}, b.env, "the bootstrap gets exactly the runtime and credential variables")
 	assert.NotEmpty(t, envValue(b.env, "AWS_LAMBDA_RUNTIME_API"))
 }

+func TestTestlinkProvidesSDKSettingsToBootstrap(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
+	defer server.Close()
+	h := newHost(t)
+	h.accept(ctx)
+	entries := h.entries()
+	entries[sdkEndpointEntry] = []byte(server.URL)
+	entries[sdkCAEntry] = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw})
+	var b bootstrapStarts
+	done := runTestlinkAsync(ctx, b.testlink(t, entries))
+	conn := h.guest(t)
+	defer conn.Close()
+	hs := hostlink.NewTestStream(conn)
+	req := hostlink.Request{Credential: &testCredential, GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: []byte(`{}`)}
+	require.NoError(t, hs.HostSend(req))
+	_, err := hs.HostResult(ctx)
+	require.NoError(t, err)
+	res := wait(t, done, "agent to stop")
+	require.NoError(t, res.err)
+	assert.Equal(t, server.URL, envValue(b.env, "AWS_ENDPOINT_URL"))
+	assert.Equal(t, b.caBundle, envValue(b.env, "AWS_CA_BUNDLE"))
+	assert.Equal(t, "ap-southeast-2", envValue(b.env, "AWS_REGION"))
+	assert.Equal(t, "ap-southeast-2", envValue(b.env, "AWS_DEFAULT_REGION"))
+	ca, err := os.ReadFile(b.caBundle)
+	require.NoError(t, err)
+	assert.Contains(t, string(ca), string(entries[sdkCAEntry]))
+	info, err := os.Stat(b.caBundle)
+	require.NoError(t, err)
+	assert.Equal(t, os.FileMode(0o644), info.Mode().Perm())
+}
+
+func TestTestlinkProvidesFunctionEnvironment(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	var b bootstrapStarts
+	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
+	conn := h.guest(t)
+	defer conn.Close()
+	hs := hostlink.NewTestStream(conn)
+	cred := testCredential
+	cred.Environment = map[string]string{"GREETING": "hello $WORLD", "PATH": "/my/bin"}
+	req := hostlink.Request{Credential: &cred, GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: []byte(`{}`)}
+	require.NoError(t, hs.HostSend(req))
+	_, err := hs.HostResult(ctx)
+	require.NoError(t, err)
+	require.NoError(t, wait(t, done, "agent to stop").err)
+	assert.Equal(t, "hello $WORLD", envValue(b.env, "GREETING"))
+	assert.Equal(t, "/my/bin", envValue(b.env, "PATH"))
+	assert.Equal(t, testCredential.SecretAccessKey, envValue(b.env, "AWS_SECRET_ACCESS_KEY"))
+}
+
+func TestTestlinkRejectsReservedFunctionEnvironment(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	var b bootstrapStarts
+	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
+	conn := h.guest(t)
+	defer conn.Close()
+	cred := testCredential
+	cred.Environment = map[string]string{"AWS_SECRET_ACCESS_KEY": "forged"}
+	req := hostlink.Request{Credential: &cred, GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: []byte(`{}`)}
+	require.NoError(t, hostlink.NewTestStream(conn).HostSend(req))
+	result := wait(t, done, "agent to refuse the environment")
+	assert.Equal(t, exitRefused, result.code)
+	assert.Error(t, result.err)
+	assert.NotContains(t, result.err.Error(), "forged")
+	assert.Zero(t, b.n)
+}
+
 var testCredential = hostlink.Credential{AccessKeyID: "ASIAGUESTKEY", SecretAccessKey: "guest-secret", SessionToken: "guest-token"}

 func envValue(env []string, name string) string {
@@ -21,14 +21,14 @@ Tenant code may be able to reach the listener, but it cannot open the agent's co
 **Authentication:** mutual TLS 1.3, bound to the environment ID and realization generation, using material the user runtime cannot read (ADR-0009:S17).
 For each generation the executor creates ephemeral self-signed host and guest certificates whose single URI SAN is `spinifex-lambda://<Region>/<account>/<function ID>/<environment ID>/<generation>/<role>`, with role `host` or `guest`.
 Each end validates the peer certificate against the one pinned peer certificate and separately requires that URI SAN with the opposite role; hostname verification is not relied on.
-ALPN carries the L-GUEST major version, `spinifex-lambda-guest/1`, and an unsupported version fails the handshake. Builds with the `lambdatestlink` tag offer only `spinifex-lambda-testlink/2` on both host and guest, whose first frame carries the execution-role credential, so a test-link end and a release end fail the handshake; release builds are unchanged.
-In the same builds, the gateway reaches the host daemon on the local abstract socket `spinifex-lambda-testlink-control/3`, and each request carries the runtime-image digest the gateway was built with; the daemon refuses a missing or different digest before starting the attempt, and a gateway and daemon on different socket names never connect.
+ALPN carries the L-GUEST major version, `spinifex-lambda-guest/1`, and an unsupported version fails the handshake. Builds with the `lambdatestlink` tag offer only `spinifex-lambda-testlink/3` on both host and guest, whose first frame carries the execution-role credential and function environment, so a test-link end and a release end fail the handshake; release builds are unchanged.
+In the same builds, the gateway reaches the host daemon on the local abstract socket `spinifex-lambda-testlink-control/4`, and each request carries the runtime-image digest the gateway was built with; the daemon refuses a missing or different digest before starting the attempt, and a gateway and daemon on different socket names never connect.
 On that socket a request's raw payload is at most 6 MiB, checked by the gateway before encoding and by the daemon after decoding.
 Its encoded frame is at most the base64 length of 6 MiB plus 64 KiB of metadata headroom and 64 bytes of field slack, and a receiver checks the frame length before allocating; metadata has no separate bound within the frame.
-The credential travels as its own frame of at most 8 KiB, and the reply has the same payload and frame bounds as the request.
+The credential and environment travel as their own frame of at most 40 KiB, and the reply has the same payload and frame bounds as the request.
 The daemon holds at most two connections at once, from accept to close, and leaves the rest unaccepted in the listen backlog.
 Under TLS 1.3 the guest's handshake can complete before the host refuses its certificate, so the guest treats the connection as accepted only after the host's first message.
-In the same builds, the function owner moves a created function with stored code from `Pending` to `Active` once this node's daemon serves the pinned image and the cluster configuration has exactly one node; a function with environment variables or a code copy over the test-link bound stays `Pending`, and `StateReason` names the unmet condition.
+In the same builds, the function owner moves a created function with stored code from `Pending` to `Active` once this node's daemon serves the pinned image and the cluster configuration has exactly one node; a code copy over the test-link bound stays `Pending`, and `StateReason` names the unmet condition.
 Release builds never leave `Pending`.
 The Mulga deploy role installs a test-link build with `deploy_lambda_testlink=true`; this profile does not meet Q-199 fail-closed revocation, and its guests take management addresses whose record can expire (mulga-0hsub).

@@ -28,11 +28,12 @@ type Request struct {
 	Credential *Credential `json:"credential,omitempty"`
 }

-// Credential is one execution-role session. It is never logged or stored.
+// Credential holds one execution-role session and its function environment.
 type Credential struct {
-	AccessKeyID     string `json:"access_key_id"`
-	SecretAccessKey string `json:"secret_access_key"`
-	SessionToken    string `json:"session_token"`
+	AccessKeyID     string            `json:"access_key_id"`
+	SecretAccessKey string            `json:"secret_access_key"`
+	SessionToken    string            `json:"session_token"`
+	Environment     map[string]string `json:"environment,omitempty"`
 }

 // LogValue keeps every value out of logs.
@@ -4,5 +4,5 @@ package hostlink

 // Protocol is the only ALPN protocol a test-link host and guest offer. It
 // differs from the release protocol because the first frame carries the
-// execution credential, which an older guest would ignore.
-const Protocol = "spinifex-lambda-testlink/2"
+// execution credential and function environment, which an older guest would ignore.
+const Protocol = "spinifex-lambda-testlink/3"
@@ -0,0 +1,67 @@
+// Package inventory is the EC2-owned read capability for an account's complete
+// instance list, for callers outside the EC2 domain.
+package inventory
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	ec2instanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/instance"
+	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// Instances lists every non-terminated instance an account owns, or returns an
+// error. It never returns a shorter list than the truth.
+type Instances interface {
+	ListAccountInstances(ctx context.Context, accountID string) (*ec2.DescribeInstancesOutput, error)
+}
+
+// NATSInstances reads the complete list across nodeIDs over NATS.
+type NATSInstances struct {
+	nc      *nats.Conn
+	nodeIDs []string
+}
+
+var _ Instances = (*NATSInstances)(nil)
+
+// NewNATSInstances returns a reader over the configured node set.
+func NewNATSInstances(nc *nats.Conn, nodeIDs []string) *NATSInstances {
+	return &NATSInstances{nc: nc, nodeIDs: nodeIDs}
+}
+
+// ListAccountInstances passes through the strict listing unchanged: any node
+// that did not answer, or any record that could not be read, is an error.
+func (n *NATSInstances) ListAccountInstances(ctx context.Context, accountID string) (*ec2.DescribeInstancesOutput, error) {
+	return ec2instanceapi.DescribeAccountInstancesStrict(ctx, n.nc, n.nodeIDs, accountID)
+}
+
+// NewENITagReader returns a reader of one ENI's tags straight from its stored
+// record, so an ENI that is absent and one that cannot be read are told apart:
+// a missing key is found=false, a failed read or an undecodable record is an
+// error. accountID is the account the ENI is stored under.
+func NewENITagReader(js jetstream.JetStream, accountID string) func(ctx context.Context, eniID string) (map[string]string, bool, error) {
+	return func(ctx context.Context, eniID string) (map[string]string, bool, error) {
+		kv, err := js.KeyValue(ctx, ec2vpc.KVBucketENIs)
+		if err != nil {
+			return nil, false, fmt.Errorf("open ENI bucket: %w", err)
+		}
+		entry, err := kv.Get(ctx, kvutil.AccountKey(accountID, eniID))
+		if errors.Is(err, jetstream.ErrKeyNotFound) {
+			return nil, false, nil
+		}
+		if err != nil {
+			return nil, false, fmt.Errorf("read ENI record %s: %w", eniID, err)
+		}
+		var record ec2vpc.ENIRecord
+		if err := json.Unmarshal(entry.Value(), &record); err != nil {
+			return nil, false, fmt.Errorf("decode ENI record %s: %w", eniID, err)
+		}
+		return record.Tags, true, nil
+	}
+}
@@ -0,0 +1,49 @@
+package inventory
+
+import (
+	"encoding/json"
+	"testing"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestListAccountInstancesRefusesAnUnknownNodeSet(t *testing.T) {
+	out, err := NewNATSInstances(nil, nil).ListAccountInstances(t.Context(), "000000000000")
+	require.Error(t, err)
+	assert.Nil(t, out, "an unknown node set must not yield a list")
+}
+
+func TestENITagReaderSeparatesGoneFromUnreadable(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	js := testutil.NewJetStream(t, nc)
+	read := NewENITagReader(js, "000000000000")
+
+	_, _, err := read(t.Context(), "eni-1")
+	require.Error(t, err, "a missing ENI bucket is a read failure, not an absent ENI")
+
+	enis, err := kvutil.GetOrCreateBucket(t.Context(), js, ec2vpc.KVBucketENIs, 10)
+	require.NoError(t, err)
+	good, err := json.Marshal(ec2vpc.ENIRecord{NetworkInterfaceId: "eni-good", Tags: map[string]string{"k": "v"}})
+	require.NoError(t, err)
+	_, err = enis.Put(t.Context(), kvutil.AccountKey("000000000000", "eni-good"), good)
+	require.NoError(t, err)
+	_, err = enis.Put(t.Context(), kvutil.AccountKey("000000000000", "eni-corrupt"), []byte("{not json"))
+	require.NoError(t, err)
+
+	tags, found, err := read(t.Context(), "eni-good")
+	require.NoError(t, err)
+	assert.True(t, found)
+	assert.Equal(t, map[string]string{"k": "v"}, tags)
+
+	_, found, err = read(t.Context(), "eni-absent")
+	require.NoError(t, err)
+	assert.False(t, found, "a missing key is the ENI gone")
+
+	_, found, err = read(t.Context(), "eni-corrupt")
+	require.Error(t, err, "an undecodable record is a read failure")
+	assert.False(t, found)
+}
@@ -0,0 +1,158 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"sort"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// ClusterRecordState is what a read of a cluster's record established.
+type ClusterRecordState int
+
+const (
+	ClusterRecordPresent ClusterRecordState = iota
+	ClusterRecordMissing
+	ClusterRecordUnreadable
+)
+
+// ClusterRecordVerdict is the outcome of looking a cluster record up without
+// creating anything. Reason names a missing or unreadable record; Err is set
+// only when the record could not be read.
+type ClusterRecordVerdict struct {
+	State  ClusterRecordState
+	Reason string
+	Err    error
+}
+
+// JudgeClusterRecord reads the cluster record read-only. A missing bucket or a
+// missing key is Missing, any other failure is Unreadable, and neither is ever
+// repaired. The billable reaper and the orphan report both use it, so they
+// judge a record the same way.
+func JudgeClusterRecord(ctx context.Context, js jetstream.JetStream, account, cluster string) ClusterRecordVerdict {
+	kv, err := OpenAccountBucket(ctx, js, account)
+	switch {
+	case errors.Is(err, jetstream.ErrBucketNotFound):
+		return ClusterRecordVerdict{State: ClusterRecordMissing, Reason: "account bucket missing"}
+	case err != nil:
+		return ClusterRecordVerdict{State: ClusterRecordUnreadable, Reason: "account bucket lookup failed", Err: err}
+	}
+	_, err = GetClusterMeta(ctx, kv, cluster)
+	switch {
+	case err == nil:
+		return ClusterRecordVerdict{State: ClusterRecordPresent}
+	case errors.Is(err, ErrClusterNotFound):
+		return ClusterRecordVerdict{State: ClusterRecordMissing, Reason: "cluster record missing"}
+	}
+	return ClusterRecordVerdict{State: ClusterRecordUnreadable, Reason: "cluster record read failed", Err: err}
+}
+
+// ENITagReader returns the tags on a control-plane ENI. found is false when
+// the ENI is gone.
+type ENITagReader func(ctx context.Context, eniID string) (tagSet map[string]string, found bool, err error)
+
+// ControlPlaneRow is one running EKS control-plane VM the report could not tie
+// to a live cluster record. Unreadable rows carry Err and are never omitted.
+type ControlPlaneRow struct {
+	InstanceID       string
+	Cluster          string
+	Account          string
+	Reason           string
+	Age              time.Duration
+	AvailabilityZone string
+	Unreadable       bool
+	Err              error
+}
+
+// FindControlPlaneOrphans classifies the instances in reservations. An instance
+// with no ENI, or whose ENI is gone or not tagged as an EKS control plane, is
+// skipped. A failed ENI read is an unreadable row, never a silent drop. A
+// control plane whose cluster record is present is omitted; one whose bucket or
+// record is missing is an orphan row; any other read failure is unreadable. It
+// only reads.
+func FindControlPlaneOrphans(ctx context.Context, js jetstream.JetStream, reservations []*ec2.Reservation, eni ENITagReader, now time.Time) []ControlPlaneRow {
+	var rows []ControlPlaneRow
+	for _, res := range reservations {
+		if res == nil {
+			continue
+		}
+		for _, inst := range res.Instances {
+			if inst == nil || !instanceRunning(inst) || len(inst.NetworkInterfaces) == 0 || inst.NetworkInterfaces[0] == nil {
+				continue
+			}
+			eniID := aws.StringValue(inst.NetworkInterfaces[0].NetworkInterfaceId)
+			if eniID == "" {
+				continue
+			}
+			row := ControlPlaneRow{InstanceID: aws.StringValue(inst.InstanceId)}
+			if inst.Placement != nil {
+				row.AvailabilityZone = aws.StringValue(inst.Placement.AvailabilityZone)
+			}
+			if inst.LaunchTime != nil {
+				row.Age = now.Sub(*inst.LaunchTime).Truncate(time.Second)
+			}
+
+			tagSet, found, err := eni(ctx, eniID)
+			if err != nil {
+				row.Unreadable, row.Err, row.Reason = true, err, "control-plane ENI read failed"
+				rows = append(rows, row)
+				continue
+			}
+			if !found || tagSet[tags.ManagedByKey] != tags.ManagedByEKS || tagSet[clusterEKSRoleTagKey] != clusterEKSRoleControlPlane {
+				continue
+			}
+			row.Cluster, row.Account = tagSet[clusterEKSClusterTagKey], tagSet[clusterEKSAccountTagKey]
+			if row.Cluster == "" || row.Account == "" {
+				continue
+			}
+
+			verdict := JudgeClusterRecord(ctx, js, row.Account, row.Cluster)
+			switch verdict.State {
+			case ClusterRecordPresent:
+				continue
+			case ClusterRecordUnreadable:
+				row.Unreadable, row.Err = true, verdict.Err
+			}
+			row.Reason = verdict.Reason
+			rows = append(rows, row)
+		}
+	}
+	sort.Slice(rows, func(i, j int) bool { return rows[i].InstanceID < rows[j].InstanceID })
+	return rows
+}
+
+// ReportControlPlaneOrphans writes the orphan report. list must return every
+// non-terminated system instance or an error: a failed or partial listing is
+// returned as an error and nothing is printed, so a short list is never taken
+// for a complete one. Unreadable rows are printed and counted separately.
+func ReportControlPlaneOrphans(ctx context.Context, w io.Writer, js jetstream.JetStream, list func(context.Context) (*ec2.DescribeInstancesOutput, error), eni ENITagReader, now time.Time) (orphans, unreadable int, err error) {
+	out, err := list(ctx)
+	if err != nil {
+		return 0, 0, fmt.Errorf("listing incomplete: %w", err)
+	}
+	if out == nil {
+		return 0, 0, errors.New("listing incomplete: no result")
+	}
+	rows := FindControlPlaneOrphans(ctx, js, out.Reservations, eni, now)
+	for _, r := range rows {
+		if r.Unreadable {
+			unreadable++
+			fmt.Fprintf(w, "UNREADABLE  %s  cluster=%s account=%s az=%s age=%s  %s: %v\n", r.InstanceID, r.Cluster, r.Account, r.AvailabilityZone, r.Age, r.Reason, r.Err)
+			continue
+		}
+		orphans++
+		fmt.Fprintf(w, "ORPHAN      %s  cluster=%s account=%s az=%s age=%s  %s\n", r.InstanceID, r.Cluster, r.Account, r.AvailabilityZone, r.Age, r.Reason)
+	}
+	return orphans, unreadable, nil
+}
+
+func instanceRunning(inst *ec2.Instance) bool {
+	return inst.State != nil && aws.StringValue(inst.State.Name) == ec2.InstanceStateNameRunning
+}
@@ -0,0 +1,198 @@
+package cluster
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	ec2inventory "github.com/mulgadc/spinifex/spinifex/domains/ec2/inventory"
+	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func orphanInstance(id, eni string) *ec2.Instance {
+	return &ec2.Instance{
+		InstanceId:        aws.String(id),
+		State:             &ec2.InstanceState{Name: aws.String(ec2.InstanceStateNameRunning)},
+		LaunchTime:        aws.Time(time.Unix(0, 0)),
+		Placement:         &ec2.Placement{AvailabilityZone: aws.String("ap-southeast-2a")},
+		NetworkInterfaces: []*ec2.InstanceNetworkInterface{{NetworkInterfaceId: aws.String(eni)}},
+	}
+}
+
+func cpTags(cluster, account string) map[string]string {
+	return map[string]string{
+		tags.ManagedByKey:       tags.ManagedByEKS,
+		clusterEKSClusterTagKey: cluster,
+		clusterEKSAccountTagKey: account,
+		clusterEKSRoleTagKey:    clusterEKSRoleControlPlane,
+	}
+}
+
+func workerTags(cluster, account string) map[string]string {
+	m := cpTags(cluster, account)
+	m[clusterEKSRoleTagKey] = "worker"
+	return m
+}
+
+func TestFindControlPlaneOrphansUnreadableRecordIsNotAnOrphan(t *testing.T) {
+	f := newEKSServiceFixture(t)
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+	ctx, cancel := context.WithCancel(t.Context())
+	cancel()
+	eni := func(context.Context, string) (map[string]string, bool, error) {
+		return cpTags("gone", testAccountID), true, nil
+	}
+	res := []*ec2.Reservation{{Instances: []*ec2.Instance{orphanInstance("i-1", "eni-1")}}}
+
+	rows := FindControlPlaneOrphans(ctx, js, res, eni, time.Unix(60, 0))
+
+	require.Len(t, rows, 1)
+	assert.True(t, rows[0].Unreadable, "a read that failed is not a missing record")
+	assert.Error(t, rows[0].Err)
+}
+
+func TestFindControlPlaneOrphans(t *testing.T) {
+	const lost = "999999999999"
+	f := newEKSServiceFixture(t)
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+	require.NoError(t, PutClusterMeta(t.Context(), f.kv, sampleClusterMeta("alive")))
+
+	eniTags := map[string]map[string]string{
+		"eni-orphan":   cpTags("gone", testAccountID),
+		"eni-live":     cpTags("alive", testAccountID),
+		"eni-nobucket": cpTags("lost", lost),
+		"eni-worker":   workerTags("gone", testAccountID),
+	}
+	eni := func(_ context.Context, id string) (map[string]string, bool, error) {
+		if id == "eni-broken" {
+			return nil, false, errors.New("vpc service down")
+		}
+		m, ok := eniTags[id]
+		return m, ok, nil
+	}
+	res := []*ec2.Reservation{{Instances: []*ec2.Instance{
+		orphanInstance("i-orphan", "eni-orphan"),
+		orphanInstance("i-live", "eni-live"),
+		orphanInstance("i-nobucket", "eni-nobucket"),
+		orphanInstance("i-worker", "eni-worker"),
+		orphanInstance("i-ghost", "eni-ghost"),
+		orphanInstance("i-broken", "eni-broken"),
+		{InstanceId: aws.String("i-noeni")},
+	}}}
+
+	rows := FindControlPlaneOrphans(t.Context(), js, res, eni, time.Unix(60, 0))
+
+	got := map[string]ControlPlaneRow{}
+	for _, r := range rows {
+		got[r.InstanceID] = r
+	}
+	assert.Len(t, got, 3, "only orphan, nobucket and the unreadable ENI are reported")
+	assert.Equal(t, "cluster record missing", got["i-orphan"].Reason)
+	assert.False(t, got["i-orphan"].Unreadable)
+	assert.Equal(t, "account bucket missing", got["i-nobucket"].Reason)
+	assert.True(t, got["i-broken"].Unreadable, "a failed ENI read is never dropped")
+	assert.Error(t, got["i-broken"].Err)
+	assert.NotContains(t, got, "i-live", "a present record is omitted")
+	assert.NotContains(t, got, "i-worker")
+	assert.NotContains(t, got, "i-ghost")
+	assert.NotContains(t, got, "i-noeni")
+}
+
+func TestReportControlPlaneOrphansIncompleteListingPrintsNothing(t *testing.T) {
+	f := newEKSServiceFixture(t)
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+	var buf bytes.Buffer
+	list := func(context.Context) (*ec2.DescribeInstancesOutput, error) {
+		return nil, errors.New("ServiceUnavailable")
+	}
+	orphans, unreadable, err := ReportControlPlaneOrphans(t.Context(), &buf, js, list, nil, time.Now())
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "listing incomplete")
+	assert.Zero(t, orphans+unreadable)
+	assert.Empty(t, buf.String(), "no partial list on a failed listing")
+}
+
+func TestReportControlPlaneOrphansPrintsRows(t *testing.T) {
+	f := newEKSServiceFixture(t)
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+	var buf bytes.Buffer
+	list := func(context.Context) (*ec2.DescribeInstancesOutput, error) {
+		return &ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{{Instances: []*ec2.Instance{
+			orphanInstance("i-orphan", "eni-orphan"), orphanInstance("i-broken", "eni-broken"),
+		}}}}, nil
+	}
+	eni := func(_ context.Context, id string) (map[string]string, bool, error) {
+		if id == "eni-broken" {
+			return nil, false, errors.New("boom")
+		}
+		return cpTags("gone", testAccountID), true, nil
+	}
+	orphans, unreadable, err := ReportControlPlaneOrphans(t.Context(), &buf, js, list, eni, time.Unix(60, 0))
+	require.NoError(t, err)
+	assert.Equal(t, 1, orphans)
+	assert.Equal(t, 1, unreadable)
+	assert.Contains(t, buf.String(), "ORPHAN      i-orphan")
+	assert.Contains(t, buf.String(), "UNREADABLE  i-broken")
+	assert.Equal(t, 2, strings.Count(buf.String(), "az=ap-southeast-2a"), "both row kinds print the zone")
+}
+
+func TestFindControlPlaneOrphansCorruptENIRecordIsUnreadable(t *testing.T) {
+	f := newEKSServiceFixture(t)
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+	enis, err := kvutil.GetOrCreateBucket(t.Context(), js, ec2vpc.KVBucketENIs, 10)
+	require.NoError(t, err)
+	good, err := json.Marshal(ec2vpc.ENIRecord{NetworkInterfaceId: "eni-good", Tags: cpTags("gone", testAccountID)})
+	require.NoError(t, err)
+	_, err = enis.Put(t.Context(), kvutil.AccountKey("000000000000", "eni-good"), good)
+	require.NoError(t, err)
+	_, err = enis.Put(t.Context(), kvutil.AccountKey("000000000000", "eni-corrupt"), []byte("{not json"))
+	require.NoError(t, err)
+	res := []*ec2.Reservation{{Instances: []*ec2.Instance{
+		orphanInstance("i-good", "eni-good"), orphanInstance("i-corrupt", "eni-corrupt"), orphanInstance("i-absent", "eni-absent"),
+	}}}
+
+	rows := FindControlPlaneOrphans(t.Context(), js, res, ec2inventory.NewENITagReader(js, "000000000000"), time.Unix(60, 0))
+
+	require.Len(t, rows, 2, "an absent ENI is skipped, the other two are reported")
+	assert.Equal(t, "i-corrupt", rows[0].InstanceID)
+	assert.True(t, rows[0].Unreadable, "a corrupt ENI record is unreadable, never gone")
+	assert.Equal(t, "i-good", rows[1].InstanceID)
+	assert.False(t, rows[1].Unreadable)
+}
+
+func TestFindControlPlaneOrphansOnlyReportsRunningInstances(t *testing.T) {
+	f := newEKSServiceFixture(t)
+	js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+	eni := func(context.Context, string) (map[string]string, bool, error) {
+		return cpTags("gone", testAccountID), true, nil
+	}
+	mk := func(id, state string) *ec2.Instance {
+		i := orphanInstance(id, "eni-"+id)
+		i.State = &ec2.InstanceState{Name: aws.String(state)}
+		return i
+	}
+	noState := orphanInstance("i-nostate", "eni-nostate")
+	noState.State = nil
+	res := []*ec2.Reservation{{Instances: []*ec2.Instance{
+		mk("i-running", ec2.InstanceStateNameRunning),
+		mk("i-stopped", ec2.InstanceStateNameStopped),
+		mk("i-pending", ec2.InstanceStateNamePending),
+		noState,
+	}}}
+
+	rows := FindControlPlaneOrphans(t.Context(), js, res, eni, time.Unix(60, 0))
+
+	require.Len(t, rows, 1)
+	assert.Equal(t, "i-running", rows[0].InstanceID)
+}
@@ -2,7 +2,6 @@ package cluster

 import (
 	"context"
-	"errors"
 	"fmt"
 	"log/slog"

@@ -13,13 +12,10 @@ import (
 	"github.com/nats-io/nats.go/jetstream"
 )

-// EKSBillableReaper is the meta-independent billable cleanup from historic PROP-LIFECYCLE-006 §5 (behaviour per ADR-0003:S2), built on
-// the resource lifecycle GC backstop. A normal DeleteCluster can leave a
-// running control-plane VM behind (terminate completes async; a daemon restart
-// re-launches it from saved running-state) after the cluster meta is already
-// swept — a billable orphan with no owner to drive a retry. This reaper finds a
-// running EKS control-plane VM whose cluster meta is DEFINITIVELY GONE and
-// terminates it. It is node-local: each node reaps the orphans it runs.
+// EKSBillableReaper raises the data-safety alarm for a running EKS control-plane
+// VM whose cluster record cannot be found. It never terminates anything: a
+// missing record cannot tell a completed DeleteCluster from a lost bucket or an
+// out-of-band delete, so removal is an operator decision. It is node-local.
 type EKSBillableReaper struct {
 	svc *EKSServiceImpl
 	// running returns this node's running VMs. The reaper filters to the EKS
@@ -38,10 +34,11 @@ func (s *EKSServiceImpl) NewBillableReaper(running func() ([]*vm.VM, error)) *EK
 func (r *EKSBillableReaper) Class() string         { return "eks-billable" }
 func (r *EKSBillableReaper) Scope() vm.ReaperScope { return vm.ScopeNodeLocal }

-// Sweep terminates every running EKS control-plane VM whose cluster meta is
-// absent. A VM whose cluster still exists (CREATING or DELETING) is left to the
-// cluster's own teardown/retry; an ENI that is gone or untagged is skipped, so
-// the reaper never reaps on uncertainty.
+// Sweep alarms on every running EKS control-plane VM whose cluster record, or
+// whole account bucket, is missing, and removes nothing. A VM whose cluster
+// exists is left to the cluster's own teardown; an ENI that is gone or untagged
+// is skipped, and a read that fails is logged and skipped. It returns 0 because
+// it never removes a VM.
 func (r *EKSBillableReaper) Sweep(ctx context.Context) (int, error) {
 	vms, err := r.running()
 	if err != nil {
@@ -53,11 +50,10 @@ func (r *EKSBillableReaper) Sweep(ctx context.Context) (int, error) {
 		return 0, fmt.Errorf("jetstream: %w", err)
 	}

-	reaped := 0
 	for _, v := range vms {
 		select {
 		case <-ctx.Done():
-			return reaped, ctx.Err()
+			return 0, ctx.Err()
 		default:
 		}
 		if v == nil || v.ManagedBy != tags.ManagedByEKS || v.ENIId == "" {
@@ -66,47 +62,23 @@ func (r *EKSBillableReaper) Sweep(ctx context.Context) (int, error) {

 		clusterName, clusterAccount, ok := r.clusterRefFromENI(ctx, v)
 		if !ok {
-			continue // ENI gone or untagged: cannot confirm orphan-hood, never reap
+			continue // ENI gone or untagged: cannot tie the VM to a cluster
 		}

-		acctKV, err := GetOrCreateAccountBucket(ctx, js, clusterAccount)
-		if err != nil {
-			slog.WarnContext(ctx, "eks-billable: account bucket lookup failed", "account", clusterAccount, "err", err)
-			continue
-		}
-		if _, err := GetClusterMeta(ctx, acctKV, clusterName); !errors.Is(err, ErrClusterNotFound) {
-			// Meta present (live/CREATING/DELETING) or unreadable: not a definitive
-			// orphan — leave it to the cluster's own teardown/retry.
+		verdict := JudgeClusterRecord(ctx, js, clusterAccount, clusterName)
+		switch verdict.State {
+		case ClusterRecordPresent:
+			continue // cluster exists (live, CREATING or DELETING)
+		case ClusterRecordUnreadable:
+			slog.WarnContext(ctx, "eks-billable: "+verdict.Reason, "cluster", clusterName, "account", clusterAccount, "err", verdict.Err)
 			continue
 		}
+		reason := verdict.Reason

-		// ALARM + reap: the cluster is gone but its control-plane VM still runs.
-		slog.WarnContext(ctx, "DATA-SAFETY ALARM: orphaned EKS control-plane VM reaped (cluster meta gone)",
-			"instanceId", v.ID, "cluster", clusterName, "account", clusterAccount, "node", v.LastNode)
-		if err := TerminateK3sServerVM(ctx, r.svc.deps.VPCK3s, r.svc.deps.Instance, v.AccountID, v.ID, v.ENIId); err != nil {
-			slog.WarnContext(ctx, "eks-billable: failed to terminate orphan CP VM", "instanceId", v.ID, "err", err)
-			continue
-		}
-		reaped++
-		r.reclaimOrphanInfra(ctx, v.AccountID, clusterName)
-	}
-	return reaped, nil
-}
-
-// reclaimOrphanInfra tears down the billable infra a reaped orphan's swept meta
-// no longer anchors: the NLB front-end NAT and the managed CP VPC's NAT-GW EIP.
-// Both are keyed by cluster name and idempotent, so a clean prior delete is a
-// no-op. Best-effort — the costliest resource (the VM) is already gone, and
-// DeleteClusterCPVPC releases the NAT-GW EIP before the VPC delete, so the
-// billable address is reclaimed even if the now-empty VPC delete trails. account
-// is the infra account (system account for the managed CP VPC topology).
-func (r *EKSBillableReaper) reclaimOrphanInfra(ctx context.Context, account, clusterName string) {
-	if err := DeleteClusterNLB(ctx, r.svc.deps.NLB, account, clusterName); err != nil {
-		slog.WarnContext(ctx, "eks-billable: reclaim orphan NLB failed", "cluster", clusterName, "err", err)
-	}
-	if err := DeleteClusterCPVPC(ctx, r.svc.cpVPCDeps(), account, clusterName, nil); err != nil {
-		slog.WarnContext(ctx, "eks-billable: reclaim orphan CP VPC (NAT-GW EIP) failed", "cluster", clusterName, "err", err)
+		slog.WarnContext(ctx, "DATA-SAFETY ALARM: EKS control-plane VM has no cluster record; retained for an operator",
+			"reason", reason, "instanceId", v.ID, "cluster", clusterName, "account", clusterAccount, "node", v.LastNode)
 	}
+	return 0, nil
 }

 // clusterRefFromENI reads the cluster name + customer account from the VM's
@@ -1,14 +1,19 @@
 package cluster

 import (
+	"bytes"
 	"context"
+	"log/slog"
 	"testing"

 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/elbv2"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -33,53 +38,119 @@ func cpVM(id, eniID string) *vm.VM {
 	return &vm.VM{ID: id, ManagedBy: tags.ManagedByEKS, ENIId: eniID, AccountID: reaperSysAccount, LastNode: "node-1"}
 }

-// TestRLC5_EKSBillableReaperTerminatesOrphanCPVM exercises the historic PROP-LIFECYCLE-006 §5 (behaviour per ADR-0003:S2)
-// meta-independent billable cleanup: a running EKS control-plane VM whose
-// cluster meta is DEFINITIVELY GONE is a billable orphan and must be terminated
-// by the GC backstop — the real fix for the orphan CP VM surviving a
-// daemon restart after DeleteCluster swept the meta.
-func TestRLC5_EKSBillableReaperTerminatesOrphanCPVM(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	f.vpc.describeByENI = map[string]*ec2.NetworkInterface{
-		"eni-orphan": cpENI("eni-orphan", "gone-cluster", testAccountID),
+// ADR-0003:S7. A running control-plane VM whose cluster record cannot be found
+// is retained and alarmed, never terminated and never has its NLB or VPC torn
+// down: a missing record cannot tell a completed DeleteCluster from a lost
+// bucket or an out-of-band delete.
+func TestS7_EKSBillableReaperNeverTerminatesAControlPlane(t *testing.T) {
+	const lostAccount = "999999999999"
+
+	cases := []struct {
+		name      string
+		account   string
+		prepare   func(t *testing.T, f *eksServiceFixture)
+		wantAlarm string
+		wantLog   string
+	}{
+		{
+			name:      "account bucket missing",
+			account:   lostAccount,
+			prepare:   func(*testing.T, *eksServiceFixture) {},
+			wantAlarm: "account bucket missing",
+		},
+		{
+			name:      "bucket has no meta key",
+			account:   testAccountID,
+			prepare:   func(*testing.T, *eksServiceFixture) {},
+			wantAlarm: "cluster record missing",
+		},
+		{
+			name:    "meta removed by the teardown revision-checked delete",
+			account: testAccountID,
+			prepare: func(t *testing.T, f *eksServiceFixture) {
+				require.NoError(t, PutClusterMeta(t.Context(), f.kv, sampleClusterMeta("orphan")))
+				_, err := kvutil.DeleteIf(t.Context(), f.kv, ClusterMetaKey("orphan"), kvutil.CASConfig{
+					Attempts: maxClusterStateCASRetries,
+					NotFound: ErrClusterNotFound,
+				}, func(*ClusterMeta) error { return nil })
+				require.NoError(t, err)
+			},
+			wantAlarm: "cluster record missing",
+		},
+		{
+			name:    "meta removed out of band",
+			account: testAccountID,
+			prepare: func(t *testing.T, f *eksServiceFixture) {
+				require.NoError(t, PutClusterMeta(t.Context(), f.kv, sampleClusterMeta("orphan")))
+				require.NoError(t, f.kv.Delete(t.Context(), ClusterMetaKey("orphan")))
+			},
+			wantAlarm: "cluster record missing",
+		},
+		{
+			name:    "meta unreadable",
+			account: testAccountID,
+			prepare: func(t *testing.T, f *eksServiceFixture) {
+				_, err := f.kv.Put(t.Context(), ClusterMetaKey("orphan"), []byte("{not json"))
+				require.NoError(t, err)
+			},
+			wantLog: "cluster record read failed",
+		},
 	}
-	orphan := cpVM("i-orphan", "eni-orphan")

-	// Seed the orphan cluster's name/tag-driven billable infra the swept meta no
-	// longer anchors: an NLB front-end and a tagged NAT gateway holding a billable
-	// EIP. The reaper's post-terminate reclaim must sweep both.
-	nlbName := ClusterNLBName("gone-cluster")
-	f.nlb.lbByName[nlbName] = &elbv2.LoadBalancer{
-		LoadBalancerArn:  aws.String("arn:lb-orphan"),
-		LoadBalancerName: aws.String(nlbName),
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			// slog.SetDefault is process-wide, so this test is not parallel.
+			var buf bytes.Buffer
+			restore := slog.Default()
+			slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo})))
+			t.Cleanup(func() { slog.SetDefault(restore) })
+
+			f := newEKSServiceFixture(t)
+			f.vpc.describeByENI = map[string]*ec2.NetworkInterface{
+				"eni-cp": cpENI("eni-cp", "orphan", tc.account),
+			}
+			nlbName := ClusterNLBName("orphan")
+			f.nlb.lbByName[nlbName] = &elbv2.LoadBalancer{
+				LoadBalancerArn:  aws.String("arn:lb-orphan"),
+				LoadBalancerName: aws.String(nlbName),
+			}
+			f.ngw.gws = []*fakeCPNatGateway{{
+				id:    "nat-orphan",
+				tags:  map[string]string{clusterEKSClusterTagKey: "orphan", clusterEKSRoleTagKey: cpVPCRoles.NatGW},
+				state: "available",
+				addrs: []*ec2.NatGatewayAddress{{AllocationId: aws.String("eipalloc-orphan")}},
+			}}
+			tc.prepare(t, f)
+
+			reaper := f.svc.NewBillableReaper(func() ([]*vm.VM, error) { return []*vm.VM{cpVM("i-cp", "eni-cp")}, nil })
+			for range 2 {
+				removed, err := reaper.Sweep(context.Background())
+				require.NoError(t, err)
+				assert.Zero(t, removed, "ADR-0003:S7: the reaper removes nothing")
+			}
+
+			assert.Empty(t, f.inst.terminateCalls, "ADR-0003:S7: a reaper's judgement is never the decision to terminate")
+			assert.Empty(t, f.vpc.deleteCalls, "the control-plane ENI must stay")
+			assert.Contains(t, f.nlb.lbByName, nlbName, "the NLB must stay")
+			assert.Empty(t, f.eip.releaseCalls, "no EIP may be released")
+
+			js := testutil.NewJetStream(t, f.svc.deps.NATSConn)
+			if tc.account == lostAccount {
+				_, err := OpenAccountBucket(t.Context(), js, lostAccount)
+				assert.ErrorIs(t, err, jetstream.ErrBucketNotFound, "a sweep must not create the bucket it found missing")
+			}
+
+			out := buf.String()
+			if tc.wantAlarm != "" {
+				assert.Contains(t, out, "DATA-SAFETY ALARM")
+				assert.Contains(t, out, tc.wantAlarm)
+				assert.Contains(t, out, "i-cp")
+			} else {
+				assert.NotContains(t, out, "DATA-SAFETY ALARM", "an unreadable record is not a missing one")
+				assert.Contains(t, out, tc.wantLog)
+			}
+		})
 	}
-	f.ngw.gws = []*fakeCPNatGateway{{
-		id:    "nat-orphan",
-		tags:  map[string]string{clusterEKSClusterTagKey: "gone-cluster", clusterEKSRoleTagKey: cpVPCRoles.NatGW},
-		state: "available",
-		addrs: []*ec2.NatGatewayAddress{{AllocationId: aws.String("eipalloc-orphan")}},
-	}}
-
-	reaper := f.svc.NewBillableReaper(func() ([]*vm.VM, error) { return []*vm.VM{orphan}, nil })
-
-	reaped, err := reaper.Sweep(context.Background())
-	require.NoError(t, err)
-	assert.Equal(t, 1, reaped, "ADR-0003:S2 behaviour; historic PROP-LIFECYCLE-006 §5 mechanism: a CP VM whose cluster meta is gone must be reaped")
-	assert.Contains(t, f.inst.terminateCalls, "i-orphan", "the orphan CP VM must be terminated")
-	require.Len(t, f.vpc.deleteCalls, 1, "the orphan CP ENI must be deleted")
-	assert.Equal(t, "eni-orphan", aws.StringValue(f.vpc.deleteCalls[0].NetworkInterfaceId))
-
-	// The orphan's billable NLB + NAT-GW EIP must also be reclaimed.
-	assert.NotContains(t, f.nlb.lbByName, nlbName, "the orphan NLB front-end must be reclaimed")
-	require.Len(t, f.eip.releaseCalls, 1, "the orphan NAT-GW EIP must be released")
-	assert.Equal(t, "eipalloc-orphan", aws.StringValue(f.eip.releaseCalls[0].AllocationId))
-
-	// Idempotent: a repeat sweep over the same (still meta-absent) VM is safe —
-	// the terminate + reclaim paths tolerate already-gone infra (no second release).
-	_, err = reaper.Sweep(context.Background())
-	require.NoError(t, err)
-	assert.Len(t, f.eip.releaseCalls, 1, "reclaim must not double-release an already-freed EIP")
 }

 // TestRLC5_EKSBillableReaperSpareLiveAndUncertain enforces the reaper's no-false-
@@ -185,7 +185,8 @@ func AccountWatchBuckets(ctx context.Context, nc *nats.Conn) ([]*kvstore.Bucket,
 	}
 	buckets := make([]*kvstore.Bucket, 0, len(names))
 	for _, name := range names {
-		buckets = append(buckets, kvstore.NewBucket(js, kvstore.Config{Name: name, History: 1}))
+		// Attach only: a watch on a deleted bucket must fail and retry, not recreate it.
+		buckets = append(buckets, kvstore.NewBucket(js, kvstore.Config{Name: name, History: 1, AttachOnly: true}))
 	}
 	return buckets, nil
 }
@@ -206,6 +207,13 @@ func GetOrCreateAccountBucket(ctx context.Context, js jetstream.JetStream, accou
 	return kv, nil
 }

+// OpenAccountBucket returns the per-account KV bucket for accountID without
+// creating it. A bucket that does not exist is jetstream.ErrBucketNotFound, so a
+// read-only caller never turns a lost bucket into an empty one.
+func OpenAccountBucket(ctx context.Context, js jetstream.JetStream, accountID string) (jetstream.KeyValue, error) {
+	return js.KeyValue(ctx, AccountBucketName(accountID))
+}
+
 // InitLeaderBucket creates (or attaches to) the shared spinifex-eks-leader
 // bucket used for per-cluster reconciler leader-lease CAS locks.
 func InitLeaderBucket(ctx context.Context, js jetstream.JetStream) (jetstream.KeyValue, error) {
@@ -7,6 +7,7 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit"
 	ekscluster "github.com/mulgadc/spinifex/spinifex/domains/eks/cluster"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -79,3 +80,26 @@ func TestAccountWatchBuckets_EnumerationErrorPropagates(t *testing.T) {
 	_, err := ekscluster.AccountWatchBuckets(ctx, nc)
 	require.Error(t, err)
 }
+
+// TestAccountWatchBuckets_WatchOnDeletedBucketDoesNotRecreateIt pins attach-only
+// handles: a DNS watch that outlives its bucket must fail and retry, because
+// recreating it hides a deleted account bucket behind an empty one.
+func TestAccountWatchBuckets_WatchOnDeletedBucketDoesNotRecreateIt(t *testing.T) {
+	t.Parallel()
+	_, nc, _ := testutil.StartTestJetStream(t)
+	js := testutil.NewJetStream(t, nc)
+	_, err := ekscluster.GetOrCreateAccountBucket(t.Context(), js, "111111111111")
+	require.NoError(t, err)
+
+	buckets, err := ekscluster.AccountWatchBuckets(t.Context(), nc)
+	require.NoError(t, err)
+	require.Len(t, buckets, 1)
+
+	require.NoError(t, js.DeleteKeyValue(t.Context(), ekscluster.AccountBucketName("111111111111")))
+
+	_, err = buckets[0].Watch(t.Context(), ">")
+	require.Error(t, err, "a watch on a deleted bucket must fail")
+
+	_, err = ekscluster.OpenAccountBucket(t.Context(), js, "111111111111")
+	assert.ErrorIs(t, err, jetstream.ErrBucketNotFound, "the watch must not have recreated the bucket")
+}
@@ -0,0 +1,277 @@
+package awsapi
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/lambda"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/eventsource"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// MappingStore is what the handlers need of the event-source mapping store.
+type MappingStore interface {
+	Create(ctx context.Context, m eventsource.Mapping) error
+	Get(ctx context.Context, uuid string) (eventsource.Mapping, uint64, error)
+	List(ctx context.Context) ([]eventsource.Entry, error)
+	Update(ctx context.Context, m eventsource.Mapping, revision uint64) (uint64, error)
+}
+
+var _ MappingStore = (*eventsource.Store)(nil)
+
+// SourceChecker reads an SQS queue's visibility timeout as the function's
+// execution role, so a queue the role cannot read is refused at create.
+type SourceChecker interface {
+	QueueVisibilityTimeout(ctx context.Context, rec function.Record, queueARN string) (int64, error)
+}
+
+// Batch size bounds for an SQS standard queue without a batching window.
+const (
+	defaultBatchSize = 10
+	maxBatchSize     = 10
+)
+
+// mappingCreateFields are the CreateEventSourceMapping members this build
+// accepts; every other member is refused rather than ignored.
+var mappingCreateFields = map[string]bool{
+	"FunctionName": true, "EventSourceArn": true, "BatchSize": true, "Enabled": true,
+	"MaximumBatchingWindowInSeconds": true,
+}
+
+var mappingUUIDRE = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`)
+
+func mappingNotFound() error {
+	return awserrors.Errorf(awserrors.ErrorResourceNotFoundException, "The resource you requested does not exist.")
+}
+
+// mappingARN is the IAM resource of one mapping, or "*" for a malformed UUID.
+func mappingARN(region, accountID, uuid string) string {
+	if region == "" || accountID == "" || !mappingUUIDRE.MatchString(uuid) {
+		return "*"
+	}
+	return fmt.Sprintf("arn:aws:lambda:%s:%s:event-source-mapping:%s", region, accountID, uuid)
+}
+
+type mappingCreate struct {
+	FunctionName, QueueARN string
+	BatchSize              int64
+}
+
+// decodeMappingCreate reads the body strictly and refuses every option this
+// build does not implement.
+func decodeMappingCreate(body []byte) (*mappingCreate, error) {
+	var raw map[string]json.RawMessage
+	dec := json.NewDecoder(bytes.NewReader(body))
+	if err := dec.Decode(&raw); err != nil {
+		return nil, invalid("The request body could not be parsed as JSON.")
+	}
+	for k := range raw {
+		if !mappingCreateFields[k] {
+			return nil, unsupported(k)
+		}
+	}
+	var in struct {
+		FunctionName                   *string `json:"FunctionName"`
+		EventSourceArn                 *string `json:"EventSourceArn"`
+		BatchSize                      *int64  `json:"BatchSize"`
+		Enabled                        *bool   `json:"Enabled"`
+		MaximumBatchingWindowInSeconds *int64  `json:"MaximumBatchingWindowInSeconds"`
+	}
+	if err := json.Unmarshal(body, &in); err != nil {
+		return nil, invalid("The request body could not be parsed as JSON.")
+	}
+	if in.FunctionName == nil || *in.FunctionName == "" {
+		return nil, invalid("1 validation error detected: Value null at 'functionName' failed to satisfy constraint: Member must not be null")
+	}
+	if in.EventSourceArn == nil || *in.EventSourceArn == "" {
+		return nil, invalid("Unrecognized event source, must be kinesis, dynamodb stream or sqs.")
+	}
+	if in.Enabled != nil && !*in.Enabled {
+		return nil, unsupported("Enabled false")
+	}
+	if in.MaximumBatchingWindowInSeconds != nil && *in.MaximumBatchingWindowInSeconds != 0 {
+		return nil, unsupported("MaximumBatchingWindowInSeconds")
+	}
+	m := &mappingCreate{FunctionName: *in.FunctionName, QueueARN: *in.EventSourceArn, BatchSize: defaultBatchSize}
+	if in.BatchSize != nil {
+		if *in.BatchSize < 1 || *in.BatchSize > maxBatchSize {
+			return nil, invalid("BatchSize must be between 1 and %d for an SQS queue without a batching window.", maxBatchSize)
+		}
+		m.BatchSize = *in.BatchSize
+	}
+	return m, nil
+}
+
+// checkQueueARN refuses anything but a standard SQS queue in the caller's
+// own account and Region.
+func checkQueueARN(region, accountID, queueARN string) error {
+	parts := strings.Split(queueARN, ":")
+	if len(parts) != 6 || parts[0] != "arn" || parts[1] != "aws" {
+		return invalid("Unrecognized event source, must be kinesis, dynamodb stream or sqs.")
+	}
+	if parts[2] != "sqs" {
+		return unsupported("an event source other than SQS")
+	}
+	if parts[3] != region || parts[4] != accountID {
+		return unsupported("a queue in another account or Region")
+	}
+	if parts[5] == "" {
+		return invalid("Unrecognized event source, must be kinesis, dynamodb stream or sqs.")
+	}
+	if strings.HasSuffix(parts[5], ".fifo") {
+		return unsupported("a FIFO queue")
+	}
+	return nil
+}
+
+// mappingUUID derives one UUID per account, Region, function and queue, so the
+// keyed create itself refuses a duplicate with no inventory scan or second record.
+func mappingUUID(accountID, region, function, queueARN string) string {
+	b := sha256.Sum256([]byte(accountID + "\x00" + region + "\x00" + function + "\x00" + queueARN))
+	b[6] = b[6]&0x0f | 0x50
+	b[8] = b[8]&0x3f | 0x80
+	return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:16])
+}
+
+func mappingShape(m eventsource.Mapping, state string) *lambda.EventSourceMappingConfiguration {
+	return &lambda.EventSourceMappingConfiguration{
+		UUID:                           aws.String(m.UUID),
+		BatchSize:                      aws.Int64(m.BatchSize),
+		MaximumBatchingWindowInSeconds: aws.Int64(0),
+		EventSourceArn:                 aws.String(m.EventSourceARN),
+		FunctionArn:                    aws.String(m.FunctionARN),
+		LastModified:                   aws.Time(m.LastModified),
+		State:                          aws.String(state),
+		StateTransitionReason:          aws.String(m.StateTransitionReason),
+	}
+}
+
+func (d Deps) mappings(ctx context.Context) (MappingStore, error) {
+	if d.Mappings == nil {
+		return nil, serviceError("event-source mappings are unavailable")
+	}
+	return d.Mappings(ctx)
+}
+
+// createEventSourceMapping resolves the function, reads the queue as its
+// execution role, and records an Enabled mapping its poller picks up.
+func (d Deps) createEventSourceMapping(ctx context.Context, c *call) (reply, error) {
+	in, err := decodeMappingCreate(c.Body)
+	if err != nil {
+		return reply{}, err
+	}
+	if err := checkQueueARN(c.Region, c.AccountID, in.QueueARN); err != nil {
+		return reply{}, err
+	}
+	name, q, err := resolveFunctionName(c.Region, c.AccountID, in.FunctionName)
+	if err != nil {
+		return reply{}, err
+	}
+	if q != "" {
+		return reply{}, unsupported("a qualified FunctionName")
+	}
+	rec, _, err := d.Meta.Get(ctx, c.AccountID, c.Region, name)
+	if errors.Is(err, function.ErrNotFound) {
+		return reply{}, notFound(c.Region, c.AccountID, name)
+	}
+	if err != nil {
+		return reply{}, err
+	}
+	if d.Sources == nil {
+		return reply{}, serviceError("event-source mappings are unavailable")
+	}
+	visibility, err := d.Sources.QueueVisibilityTimeout(ctx, rec, in.QueueARN)
+	if err != nil {
+		return reply{}, err
+	}
+	if rec.Timeout > visibility {
+		return reply{}, invalid("Queue visibility timeout: %d seconds is less than Function timeout: %d seconds", visibility, rec.Timeout)
+	}
+	store, err := d.mappings(ctx)
+	if err != nil {
+		return reply{}, err
+	}
+	uuid := mappingUUID(c.AccountID, c.Region, name, in.QueueARN)
+	m := eventsource.Mapping{
+		FormatVersion: eventsource.FormatVersion, UUID: uuid, AccountID: c.AccountID, Region: c.Region,
+		FunctionName: name, FunctionARN: arn.FormatLambdaFunction(c.Region, c.AccountID, name),
+		EventSourceARN: in.QueueARN, BatchSize: in.BatchSize, State: eventsource.StateEnabled,
+		StateTransitionReason: eventsource.ReasonUserInitiated, LastModified: time.Now().UTC().Truncate(time.Millisecond),
+	}
+	if err := store.Create(ctx, m); err != nil {
+		if errors.Is(err, eventsource.ErrConflict) {
+			return reply{}, awserrors.Errorf(awserrors.ErrorResourceConflictException,
+				"An event source mapping with SQS arn (\" %s \") and function (\" %s \") already exists. Please update or delete the existing mapping with UUID %s",
+				in.QueueARN, name, uuid)
+		}
+		return reply{}, err
+	}
+	return reply{Status: http.StatusAccepted, Body: mappingShape(m, "Creating")}, nil
+}
+
+// ownMapping reads a mapping of the caller's account and Region.
+func (d Deps) ownMapping(ctx context.Context, c *call) (MappingStore, eventsource.Mapping, uint64, error) {
+	if !mappingUUIDRE.MatchString(c.Params[0]) {
+		return nil, eventsource.Mapping{}, 0, mappingNotFound()
+	}
+	store, err := d.mappings(ctx)
+	if err != nil {
+		return nil, eventsource.Mapping{}, 0, err
+	}
+	m, rev, err := store.Get(ctx, c.Params[0])
+	if errors.Is(err, eventsource.ErrNotFound) || (err == nil && (m.AccountID != c.AccountID || m.Region != c.Region)) {
+		return nil, eventsource.Mapping{}, 0, mappingNotFound()
+	}
+	return store, m, rev, err
+}
+
+func (d Deps) getEventSourceMapping(ctx context.Context, c *call) (reply, error) {
+	_, m, _, err := d.ownMapping(ctx, c)
+	if err != nil {
+		return reply{}, err
+	}
+	return reply{Status: http.StatusOK, Body: mappingShape(m, m.State)}, nil
+}
+
+// deleteEventSourceMapping records Deleting and returns; the poller stops its
+// worker and only then removes the record.
+func (d Deps) deleteEventSourceMapping(ctx context.Context, c *call) (reply, error) {
+	for range 3 {
+		store, m, rev, err := d.ownMapping(ctx, c)
+		if err != nil {
+			return reply{}, err
+		}
+		if m.State == eventsource.StateDeleting {
+			return reply{Status: http.StatusAccepted, Body: mappingShape(m, m.State)}, nil
+		}
+		m.State = eventsource.StateDeleting
+		m.StateTransitionReason = eventsource.ReasonUserInitiated
+		m.LastModified = time.Now().UTC().Truncate(time.Millisecond)
+		_, err = store.Update(ctx, m, rev)
+		if errors.Is(err, eventsource.ErrConflict) {
+			continue
+		}
+		if err != nil {
+			return reply{}, err
+		}
+		return reply{Status: http.StatusAccepted, Body: mappingShape(m, m.State)}, nil
+	}
+	return reply{}, serviceError("the mapping changed concurrently; retry the request")
+}
+
+// listEventSourceMappings is refused: the mapping store can only be listed
+// whole, and a request must not wait on the whole inventory.
+func (d Deps) listEventSourceMappings(_ context.Context, _ *call) (reply, error) {
+	return reply{}, invalid("ListEventSourceMappings is not supported in this build; use GetEventSourceMapping with the mapping UUID")
+}
@@ -0,0 +1,230 @@
+package awsapi_test
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"sync"
+	"testing"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/internal/testkit/gwtest"
+	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/eventsource"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type fakeSources struct {
+	mu         sync.Mutex
+	visibility int64
+	err        error
+	asRole     []string
+}
+
+func (f *fakeSources) QueueVisibilityTimeout(_ context.Context, rec function.Record, _ string) (int64, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.asRole = append(f.asRole, rec.Role)
+	return f.visibility, f.err
+}
+
+const mappingsPath = "/2015-03-31/event-source-mappings/"
+
+var queueARN = "arn:aws:sqs:" + gwtest.Region + ":" + gwtest.AccountID + ":orders"
+
+func newMappingEnv(t *testing.T, statements ...iamdomain.Statement) (*env, *eventsource.Store, *fakeSources) {
+	t.Helper()
+	e := newEnv(t, statements...)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) { b["Timeout"] = 30 }))
+	require.NoError(t, err)
+	_, _, js := testutil.StartTestJetStream(t)
+	store, err := eventsource.Open(t.Context(), js)
+	require.NoError(t, err)
+	src := &fakeSources{visibility: 30}
+	e.deps.Mappings = func(context.Context) (awsapi.MappingStore, error) { return store, nil }
+	e.deps.Sources = src
+	return e, store, src
+}
+
+func mappingBody(mutate func(map[string]any)) map[string]any {
+	b := map[string]any{"FunctionName": "hello", "EventSourceArn": queueARN}
+	if mutate != nil {
+		mutate(b)
+	}
+	return b
+}
+
+func decodeMapping(t *testing.T, raw []byte) map[string]any {
+	t.Helper()
+	var out map[string]any
+	require.NoError(t, json.Unmarshal(raw, &out))
+	return out
+}
+
+func TestEventSourceMapping_CreateGetDelete(t *testing.T) {
+	e, store, src := newMappingEnv(t)
+	rec, err := e.do(http.MethodPost, mappingsPath, mappingBody(func(b map[string]any) { b["BatchSize"] = 4 }))
+	require.NoError(t, err)
+	require.Equal(t, http.StatusAccepted, rec.Code)
+	created := decodeMapping(t, rec.Body.Bytes())
+	assert.Equal(t, "Creating", created["State"])
+	assert.EqualValues(t, 4, created["BatchSize"])
+	assert.Equal(t, queueARN, created["EventSourceArn"])
+	assert.Equal(t, []string{roleARN}, src.asRole, "the queue is read as the function's execution role")
+	uuid, _ := created["UUID"].(string)
+
+	stored, _, err := store.Get(t.Context(), uuid)
+	require.NoError(t, err)
+	assert.Equal(t, eventsource.StateEnabled, stored.State)
+
+	rec, err = e.do(http.MethodGet, mappingsPath+uuid, nil)
+	require.NoError(t, err)
+	assert.Equal(t, "Enabled", decodeMapping(t, rec.Body.Bytes())["State"])
+
+	_, err = e.do(http.MethodPost, mappingsPath, mappingBody(nil))
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
+
+	for range 2 {
+		rec, err = e.do(http.MethodDelete, mappingsPath+uuid, nil)
+		require.NoError(t, err)
+		require.Equal(t, http.StatusAccepted, rec.Code)
+		assert.Equal(t, "Deleting", decodeMapping(t, rec.Body.Bytes())["State"])
+	}
+	stored, _, err = store.Get(t.Context(), uuid)
+	require.NoError(t, err, "delete leaves the record for the poller to remove")
+	assert.Equal(t, eventsource.StateDeleting, stored.State)
+}
+
+func TestEventSourceMapping_ConcurrentDuplicateCreatesYieldOneMapping(t *testing.T) {
+	first, store, src := newMappingEnv(t)
+	const n = 8
+	gateways := []*env{first}
+	for len(gateways) < n {
+		e := newEnv(t)
+		_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) { b["Timeout"] = 30 }))
+		require.NoError(t, err)
+		e.deps.Mappings = first.deps.Mappings
+		e.deps.Sources = src
+		gateways = append(gateways, e)
+	}
+	codes := make(chan error, n)
+	var wg sync.WaitGroup
+	for _, e := range gateways {
+		wg.Go(func() {
+			_, err := e.do(http.MethodPost, mappingsPath, mappingBody(nil))
+			codes <- err
+		})
+	}
+	wg.Wait()
+	close(codes)
+	created := 0
+	for err := range codes {
+		if err == nil {
+			created++
+			continue
+		}
+		testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
+	}
+	assert.Equal(t, 1, created)
+	entries, err := store.List(t.Context())
+	require.NoError(t, err)
+	assert.Len(t, entries, 1)
+}
+
+func TestEventSourceMapping_ListIsRefusedWithoutReadingTheStore(t *testing.T) {
+	e, store, _ := newMappingEnv(t)
+	_, err := e.do(http.MethodPost, mappingsPath, mappingBody(nil))
+	require.NoError(t, err)
+	for _, path := range []string{mappingsPath, mappingsPath + "?FunctionName=hello", mappingsPath + "?MaxItems=2"} {
+		_, err = e.do(http.MethodGet, path, nil)
+		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+	}
+	entries, err := store.List(t.Context())
+	require.NoError(t, err)
+	assert.Len(t, entries, 1, "the refusal leaves the mapping in place")
+}
+
+func TestEventSourceMapping_RefusesWhatThisBuildDoesNotImplement(t *testing.T) {
+	e, store, _ := newMappingEnv(t)
+	cases := map[string]func(map[string]any){
+		"FIFO queue":      func(b map[string]any) { b["EventSourceArn"] = queueARN + ".fifo" },
+		"another account": func(b map[string]any) { b["EventSourceArn"] = "arn:aws:sqs:" + gwtest.Region + ":210987654321:orders" },
+		"another Region":  func(b map[string]any) { b["EventSourceArn"] = "arn:aws:sqs:us-east-1:" + gwtest.AccountID + ":orders" },
+		"not SQS": func(b map[string]any) {
+			b["EventSourceArn"] = "arn:aws:kinesis:" + gwtest.Region + ":" + gwtest.AccountID + ":stream/s"
+		},
+		"filter criteria":   func(b map[string]any) { b["FilterCriteria"] = map[string]any{"Filters": []any{}} },
+		"disabled":          func(b map[string]any) { b["Enabled"] = false },
+		"batching window":   func(b map[string]any) { b["MaximumBatchingWindowInSeconds"] = 5 },
+		"batch over 10":     func(b map[string]any) { b["BatchSize"] = 11 },
+		"batch zero":        func(b map[string]any) { b["BatchSize"] = 0 },
+		"qualified name":    func(b map[string]any) { b["FunctionName"] = "hello:1" },
+		"missing source":    func(b map[string]any) { delete(b, "EventSourceArn") },
+		"missing function":  func(b map[string]any) { delete(b, "FunctionName") },
+		"starting position": func(b map[string]any) { b["StartingPosition"] = "LATEST" },
+	}
+	for name, mutate := range cases {
+		t.Run(name, func(t *testing.T) {
+			_, err := e.do(http.MethodPost, mappingsPath, mappingBody(mutate))
+			testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+		})
+	}
+	entries, err := store.List(t.Context())
+	require.NoError(t, err)
+	assert.Empty(t, entries)
+}
+
+func TestEventSourceMapping_VisibilityTimeoutMustCoverTheFunctionTimeout(t *testing.T) {
+	e, store, src := newMappingEnv(t)
+	src.visibility = 29
+	_, err := e.do(http.MethodPost, mappingsPath, mappingBody(nil))
+	testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+
+	src.visibility, src.err = 0, awserrors.Errorf(awserrors.ErrorAccessDeniedException, "denied")
+	_, err = e.do(http.MethodPost, mappingsPath, mappingBody(nil))
+	testutil.RequireAWSCode(t, err, awserrors.ErrorAccessDeniedException)
+
+	entries, err := store.List(t.Context())
+	require.NoError(t, err)
+	assert.Empty(t, entries)
+
+	_, err = e.do(http.MethodPost, mappingsPath, mappingBody(func(b map[string]any) { b["FunctionName"] = "absent" }))
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+}
+
+func TestEventSourceMapping_AnotherAccountsMappingIsNotFound(t *testing.T) {
+	e, store, _ := newMappingEnv(t)
+	m := eventsource.Mapping{
+		FormatVersion: eventsource.FormatVersion, UUID: "0a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d", AccountID: "210987654321",
+		Region: gwtest.Region, FunctionName: "hello", EventSourceARN: "arn:aws:sqs:" + gwtest.Region + ":210987654321:orders",
+		BatchSize: 1, State: eventsource.StateEnabled,
+	}
+	require.NoError(t, store.Create(t.Context(), m))
+	_, err := e.do(http.MethodGet, mappingsPath+m.UUID, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+	_, err = e.do(http.MethodDelete, mappingsPath+m.UUID, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+	_, err = e.do(http.MethodGet, mappingsPath+"not-a-uuid", nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+}
+
+func TestEventSourceMapping_GetAndDeleteAuthorizeAgainstTheMappingARN(t *testing.T) {
+	const uuid = "0a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d"
+	mapping := "arn:aws:lambda:" + gwtest.Region + ":" + gwtest.AccountID + ":event-source-mapping:" + uuid
+	e, store, _ := newMappingEnv(t, gwtest.Statement("Allow", "*", "*"), gwtest.Statement("Deny", "lambda:*", mapping))
+	require.NoError(t, store.Create(t.Context(), eventsource.Mapping{
+		FormatVersion: eventsource.FormatVersion, UUID: uuid, AccountID: gwtest.AccountID, Region: gwtest.Region,
+		FunctionName: "hello", EventSourceARN: queueARN, BatchSize: 1, State: eventsource.StateEnabled,
+	}))
+	_, err := e.do(http.MethodGet, mappingsPath+uuid, nil)
+	gwtest.AssertDenied(t, err)
+	_, err = e.do(http.MethodDelete, mappingsPath+uuid, nil)
+	gwtest.AssertDenied(t, err)
+	m, _, err := store.Get(t.Context(), uuid)
+	require.NoError(t, err)
+	assert.Equal(t, eventsource.StateEnabled, m.State)
+}
@@ -553,7 +553,6 @@ func TestCreateFunction_ParameterValidation(t *testing.T) {
 		"missing role":    func(b map[string]any) { b["Role"] = "arn:aws:iam::" + gwtest.AccountID + ":role/ghost" },
 		"not a role arn":  func(b map[string]any) { b["Role"] = "arn:aws:iam::" + gwtest.AccountID + ":user/alice" },
 		"malformed body":  func(b map[string]any) { b["Timeout"] = "soon" },
-		"empty env name":  func(b map[string]any) { b["Environment"] = map[string]any{"Variables": map[string]string{"": "x"}} },
 		"description len": func(b map[string]any) { b["Description"] = strings.Repeat("d", 257) },
 	}
 	for name, mutate := range cases {
@@ -566,6 +565,31 @@ func TestCreateFunction_ParameterValidation(t *testing.T) {
 	}
 }

+func TestCreateFunction_EnvironmentValidation(t *testing.T) {
+	for name, tc := range map[string]struct {
+		vars map[string]string
+		code string
+	}{
+		"empty name":           {map[string]string{"": "secret"}, awserrors.ErrorValidationException},
+		"one character":        {map[string]string{"A": "secret"}, awserrors.ErrorValidationException},
+		"invalid character":    {map[string]string{"BAD-NAME": "secret"}, awserrors.ErrorValidationException},
+		"invalid and reserved": {map[string]string{"A": "secret", "AWS_REGION": "secret"}, awserrors.ErrorValidationException},
+		"reserved":             {map[string]string{"AWS_SECRET_ACCESS_KEY": "secret"}, awserrors.ErrorInvalidParameterValueException},
+		"nul value":            {map[string]string{"VALUE": "secret\x00"}, awserrors.ErrorInvalidParameterValueException},
+		"too large":            {map[string]string{"VALUE": strings.Repeat("x", 4092)}, awserrors.ErrorInvalidParameterValueException},
+	} {
+		t.Run(name, func(t *testing.T) {
+			e := newEnv(t)
+			_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) {
+				b["Environment"] = map[string]any{"Variables": tc.vars}
+			}))
+			testutil.RequireAWSCode(t, err, tc.code)
+			assert.NotContains(t, err.Error(), "secret")
+			assert.Zero(t, e.store.count())
+		})
+	}
+}
+
 func TestCreateFunction_RoleErrorsDoNotLeakAcrossAccounts(t *testing.T) {
 	e := newEnv(t)
 	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) { b["Role"] = foreignRoleARN }))
@@ -765,8 +789,9 @@ func TestAuthorization(t *testing.T) {

 func TestRegistration_ResolvesActionsAndInventory(t *testing.T) {
 	reg := awsapi.NewRegistration(awsapi.Deps{})
-	assert.Equal(t, []string{"CreateFunction", "GetFunction", "GetFunctionCodeSigningConfig", "GetFunctionConfiguration", "Invoke", "ListVersionsByFunction"}, reg.Inventory.Registered)
+	assert.Equal(t, []string{"CreateEventSourceMapping", "CreateFunction", "DeleteEventSourceMapping", "GetEventSourceMapping", "GetFunction", "GetFunctionCodeSigningConfig", "GetFunctionConfiguration", "Invoke", "ListEventSourceMappings", "ListVersionsByFunction"}, reg.Inventory.Registered)
 	assert.Equal(t, "ListVersionsByFunction", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x/versions", nil)))
+	assert.Equal(t, "DeleteEventSourceMapping", reg.ResolveAction(httptest.NewRequest(http.MethodDelete, "/2015-03-31/event-source-mappings/x", nil)))
 	assert.Equal(t, "GetFunctionCodeSigningConfig", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2020-06-30/functions/x/code-signing-config", nil)))
 	assert.Equal(t, "CreateFunction", reg.ResolveAction(httptest.NewRequest(http.MethodPost, "/2015-03-31/functions", nil)))
 	assert.Equal(t, "GetFunction", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x", nil)))
@@ -114,16 +114,32 @@ func (d Deps) invoke(ctx context.Context, c *call) (reply, error) {
 	if err != nil {
 		return reply{}, err
 	}
+	out, err := d.runSync(ctx, rec, revision, c.Invoke.Qualifier, c.Body)
+	if err != nil {
+		return reply{}, err
+	}
+	hdr := http.Header{}
+	hdr.Set("X-Amz-Executed-Version", out.ExecutedVersion)
+	if out.FunctionError != "" {
+		hdr.Set("X-Amz-Function-Error", out.FunctionError)
+	}
+	return reply{Status: http.StatusOK, Body: rawJSON(out.Payload), Header: hdr}, nil
+}
+
+// runSync runs one synchronous attempt on a resolved record and classifies
+// its result; every error is the AWS error Invoke returns. A nil error means
+// the result is valid, with or without a function error.
+func (d Deps) runSync(ctx context.Context, rec function.Record, revision uint64, qualifier string, payload []byte) (SyncResult, error) {
 	if rec.State != function.StateActive {
-		return reply{}, awserrors.Errorf(awserrors.ErrorResourceConflictException,
+		return SyncResult{}, awserrors.Errorf(awserrors.ErrorResourceConflictException,
 			"The operation cannot be performed at this time. The function is currently in the following state: %s", rec.State)
 	}
 	if d.Exec == nil || rec.Code.State != function.CodeStored {
-		return reply{}, serviceError("function execution is unavailable")
+		return SyncResult{}, serviceError("function execution is unavailable")
 	}

 	if err := ctx.Err(); err != nil {
-		return reply{}, err
+		return SyncResult{}, err
 	}
 	// Once accepted, a disconnecting caller does not cancel the attempt; only
 	// the wait is bounded, and a late result is discarded.
@@ -133,31 +149,37 @@ func (d Deps) invoke(ctx context.Context, c *call) (reply, error) {
 	out, err := d.Exec.InvokeSync(actx, SyncInvocation{
 		Function:  rec,
 		Revision:  revision,
-		Qualifier: c.Invoke.Qualifier,
-		Payload:   c.Body,
+		Qualifier: qualifier,
+		Payload:   payload,
 		WaitUntil: waitUntil,
 	})
 	if refusal, ok := errors.AsType[*RefusalError](err); ok {
 		if _, coded := awserrors.ResolveErrorCode(refusal.Err); coded {
-			return reply{}, refusal.Err
+			return SyncResult{}, refusal.Err
 		}
 		err = refusal
 	}
 	if err != nil {
 		slog.ErrorContext(ctx, "Lambda: synchronous attempt outcome unknown", "function_id", rec.FunctionID, "err", err)
-		return reply{}, serviceError("the invocation outcome is unknown")
+		return SyncResult{}, serviceError("the invocation outcome is unknown")
 	}
 	if out.ExecutedVersion == "" || len(out.Payload) > maxSyncPayload {
 		slog.ErrorContext(ctx, "Lambda: executor returned an invalid result", "function_id", rec.FunctionID,
 			"payload_bytes", len(out.Payload))
-		return reply{}, serviceError("the invocation outcome is unknown")
+		return SyncResult{}, serviceError("the invocation outcome is unknown")
 	}
-	hdr := http.Header{}
-	hdr.Set("X-Amz-Executed-Version", out.ExecutedVersion)
-	if out.FunctionError != "" {
-		hdr.Set("X-Amz-Function-Error", out.FunctionError)
+	return out, nil
+}
+
+// InvokeEventSource runs one event-source batch on $LATEST through the same
+// state, code and result checks as Invoke, without Invoke's IAM check: the
+// mapping was authorized when it was created.
+func (d Deps) InvokeEventSource(ctx context.Context, accountID, region, name string, payload []byte) (SyncResult, error) {
+	rec, revision, err := d.Meta.Get(ctx, accountID, region, name)
+	if err != nil {
+		return SyncResult{}, err
 	}
-	return reply{Status: http.StatusOK, Body: rawJSON(out.Payload), Header: hdr}, nil
+	return d.runSync(ctx, rec, revision, "", payload)
 }

 // checkInvokeOptions refuses every request option this build does not
@@ -54,6 +54,12 @@ type Deps struct {
 	AEAD cipher.AEAD
 	// Exec runs synchronous attempts; nil refuses every Invoke.
 	Exec Executor
+	// Mappings opens the event-source mapping store; nil refuses every
+	// mapping action.
+	Mappings func(ctx context.Context) (MappingStore, error)
+	// Sources reads a queue as a function's execution role; nil refuses
+	// CreateEventSourceMapping.
+	Sources SourceChecker
 }

 // NewRegistration returns the Lambda REST-JSON registration.
@@ -136,6 +142,8 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 		if name, _, err := resolveFunctionName(inv.Region, inv.AccountID, params[0]); err == nil {
 			resource = functionARN(inv.Region, inv.AccountID, name)
 		}
+	case "GetEventSourceMapping", "DeleteEventSourceMapping":
+		resource = mappingARN(inv.Region, inv.AccountID, params[0])
 	case "Invoke":
 		// Invoke validates deterministically before authorizing, and its
 		// resource comes from that same validated target. URL.Query drops
@@ -160,8 +168,7 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 		}
 	}
 	// Only CreateFunction and GetFunction touch the code store.
-	needsCodes := action != "GetFunctionConfiguration" && action != "GetFunctionCodeSigningConfig" &&
-		action != "ListVersionsByFunction" && action != "Invoke"
+	needsCodes := action == "CreateFunction" || action == "GetFunction"
 	if d.Meta == nil || (d.Codes == nil && needsCodes) {
 		return errors.New(awserrors.ErrorServiceException)
 	}
@@ -66,6 +66,30 @@ var routes = []route{
 		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
 			return d.invoke(ctx, c)
 		}},
+	{Method: "POST", Pattern: "/2015-03-31/event-source-mappings/", Action: "CreateEventSourceMapping",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.createEventSourceMapping(ctx, c)
+		}},
+	{Method: "POST", Pattern: "/2015-03-31/event-source-mappings", Action: "CreateEventSourceMapping",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.createEventSourceMapping(ctx, c)
+		}},
+	{Method: "GET", Pattern: "/2015-03-31/event-source-mappings/", Action: "ListEventSourceMappings",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.listEventSourceMappings(ctx, c)
+		}},
+	{Method: "GET", Pattern: "/2015-03-31/event-source-mappings", Action: "ListEventSourceMappings",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.listEventSourceMappings(ctx, c)
+		}},
+	{Method: "GET", Pattern: "/2015-03-31/event-source-mappings/{UUID}", Action: "GetEventSourceMapping",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.getEventSourceMapping(ctx, c)
+		}},
+	{Method: "DELETE", Pattern: "/2015-03-31/event-source-mappings/{UUID}", Action: "DeleteEventSourceMapping",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.deleteEventSourceMapping(ctx, c)
+		}},
 }

 var router = rest.NewRouter("lambda", routes)
@@ -4,6 +4,7 @@ import (
 	"archive/zip"
 	"bytes"
 	"encoding/json"
+	"errors"
 	"reflect"
 	"regexp"
 	"strings"
@@ -13,16 +14,16 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/domains/lambda/codestore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/lambdaenv"
 )

 const (
-	supportedRuntime  = "provided.al2023"
-	defaultTimeout    = 3
-	maxTimeout        = 900
-	defaultMemory     = 128
-	maxMemory         = 10240
-	maxDescription    = 256
-	maxEnvironmentLen = 4096
+	supportedRuntime = "provided.al2023"
+	defaultTimeout   = 3
+	maxTimeout       = 900
+	defaultMemory    = 128
+	maxMemory        = 10240
+	maxDescription   = 256
 	// maxBodyBytes bounds a CreateFunction body: the base64 of the largest
 	// code copy plus room for the rest of the document.
 	maxBodyBytes = (codestore.MaxCodeBytes/3+1)*4 + 1<<20
@@ -258,16 +259,14 @@ func envVars(e *lambda.Environment) (map[string]string, error) {
 		return nil, nil
 	}
 	vars := make(map[string]string, len(e.Variables))
-	size := 0
 	for k, v := range e.Variables {
-		if k == "" {
-			return nil, invalid("Environment variable names must not be empty")
-		}
 		vars[k] = aws.StringValue(v)
-		size += len(k) + len(vars[k])
 	}
-	if size > maxEnvironmentLen {
-		return nil, invalid("Environment variables exceed %d bytes", maxEnvironmentLen)
+	if err := lambdaenv.Validate(vars); err != nil {
+		if errors.Is(err, lambdaenv.ErrName) {
+			return nil, awserrors.Errorf(awserrors.ErrorValidationException, "Environment variable names must match [a-zA-Z][a-zA-Z0-9_]+")
+		}
+		return nil, invalid("%s", err)
 	}
 	return vars, nil
 }
@@ -0,0 +1,170 @@
+// Package eventsource holds the Lambda event-source mapping record and its
+// JetStream KV store.
+package eventsource
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// FormatVersion is the only record layout this build reads or writes.
+const FormatVersion = 1
+
+// KVBucket holds every mapping, keyed by UUID; the UUID is global in AWS.
+const KVBucket = "lambda-event-source-mappings"
+
+// Mapping states this build records. A mapping is Enabled from creation and
+// Deleting from the delete request until its poller has stopped.
+const (
+	StateEnabled  = "Enabled"
+	StateDeleting = "Deleting"
+)
+
+// ReasonUserInitiated is the transition reason of a state the caller asked for.
+const ReasonUserInitiated = "USER_INITIATED"
+
+// ErrNotFound is returned for a UUID with no mapping.
+var ErrNotFound = errors.New("eventsource: mapping not found")
+
+// ErrConflict is returned when the record changed since it was read.
+var ErrConflict = errors.New("eventsource: mapping changed concurrently")
+
+// Mapping is one SQS event-source mapping.
+type Mapping struct {
+	FormatVersion  int    `json:"formatVersion"`
+	UUID           string `json:"uuid"`
+	AccountID      string `json:"accountId"`
+	Region         string `json:"region"`
+	FunctionName   string `json:"functionName"`
+	FunctionARN    string `json:"functionArn"`
+	EventSourceARN string `json:"eventSourceArn"`
+	BatchSize      int64  `json:"batchSize"`
+	State          string `json:"state"`
+	// StateTransitionReason is USER_INITIATED, or the last receive failure
+	// prefixed "PROBLEM: " while polling cannot proceed.
+	StateTransitionReason string    `json:"stateTransitionReason"`
+	LastModified          time.Time `json:"lastModified"`
+}
+
+// Store reads and writes mappings in KVBucket.
+type Store struct{ kv jetstream.KeyValue }
+
+// Open returns the store, creating its bucket on first use.
+func Open(ctx context.Context, js jetstream.KeyValueManager) (*Store, error) {
+	kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucket, 1)
+	if err != nil {
+		return nil, fmt.Errorf("eventsource: open bucket: %w", err)
+	}
+	return &Store{kv: kv}, nil
+}
+
+func key(uuid string) string { return "esm." + uuid }
+
+// Create writes a new mapping; a UUID already present is a conflict.
+func (s *Store) Create(ctx context.Context, m Mapping) error {
+	b, err := json.Marshal(m)
+	if err != nil {
+		return err
+	}
+	if _, err := s.kv.Create(ctx, key(m.UUID), b); err != nil {
+		if errors.Is(err, jetstream.ErrKeyExists) {
+			return ErrConflict
+		}
+		return fmt.Errorf("eventsource: create: %w", err)
+	}
+	return nil
+}
+
+// Get returns the mapping and its KV revision.
+func (s *Store) Get(ctx context.Context, uuid string) (Mapping, uint64, error) {
+	e, err := s.kv.Get(ctx, key(uuid))
+	if errors.Is(err, jetstream.ErrKeyNotFound) {
+		return Mapping{}, 0, ErrNotFound
+	}
+	if err != nil {
+		return Mapping{}, 0, fmt.Errorf("eventsource: get: %w", err)
+	}
+	m, err := decode(e.Value())
+	return m, e.Revision(), err
+}
+
+// Entry is one listed mapping with the revision it was read at.
+type Entry struct {
+	Mapping  Mapping
+	Revision uint64
+}
+
+// List returns every mapping in the bucket once.
+func (s *Store) List(ctx context.Context) ([]Entry, error) {
+	lister, err := s.kv.ListKeys(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("eventsource: list: %w", err)
+	}
+	defer func() { _ = lister.Stop() }()
+	var out []Entry
+	// A KV listing may repeat a key that changes while it is being listed.
+	seen := map[string]bool{}
+	for k := range lister.Keys() {
+		if seen[k] {
+			continue
+		}
+		seen[k] = true
+		e, err := s.kv.Get(ctx, k)
+		if errors.Is(err, jetstream.ErrKeyNotFound) {
+			continue
+		}
+		if err != nil {
+			return nil, fmt.Errorf("eventsource: list get: %w", err)
+		}
+		m, err := decode(e.Value())
+		if err != nil {
+			return nil, err
+		}
+		out = append(out, Entry{Mapping: m, Revision: e.Revision()})
+	}
+	return out, nil
+}
+
+// Update replaces the mapping only if it is still at revision.
+func (s *Store) Update(ctx context.Context, m Mapping, revision uint64) (uint64, error) {
+	b, err := json.Marshal(m)
+	if err != nil {
+		return 0, err
+	}
+	rev, err := s.kv.Update(ctx, key(m.UUID), b, revision)
+	if err != nil {
+		if errors.Is(err, jetstream.ErrKeyExists) {
+			return 0, ErrConflict
+		}
+		return 0, fmt.Errorf("eventsource: update: %w", err)
+	}
+	return rev, nil
+}
+
+// Remove deletes the mapping only if it is still at revision.
+func (s *Store) Remove(ctx context.Context, uuid string, revision uint64) error {
+	if err := s.kv.Delete(ctx, key(uuid), jetstream.LastRevision(revision)); err != nil {
+		if errors.Is(err, jetstream.ErrKeyExists) {
+			return ErrConflict
+		}
+		return fmt.Errorf("eventsource: remove: %w", err)
+	}
+	return nil
+}
+
+func decode(b []byte) (Mapping, error) {
+	var m Mapping
+	if err := json.Unmarshal(b, &m); err != nil {
+		return Mapping{}, fmt.Errorf("eventsource: decode: %w", err)
+	}
+	if m.FormatVersion != FormatVersion {
+		return Mapping{}, fmt.Errorf("eventsource: record format %d is not %d", m.FormatVersion, FormatVersion)
+	}
+	return m, nil
+}
@@ -0,0 +1,42 @@
+package eventsource_test
+
+import (
+	"testing"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/eventsource"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestStore_RevisionsGuardEveryChange(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	s, err := eventsource.Open(t.Context(), js)
+	require.NoError(t, err)
+
+	entries, err := s.List(t.Context())
+	require.NoError(t, err)
+	assert.Empty(t, entries)
+
+	m := eventsource.Mapping{FormatVersion: eventsource.FormatVersion, UUID: "u1", State: eventsource.StateEnabled}
+	require.NoError(t, s.Create(t.Context(), m))
+	require.ErrorIs(t, s.Create(t.Context(), m), eventsource.ErrConflict)
+
+	got, rev, err := s.Get(t.Context(), "u1")
+	require.NoError(t, err)
+	assert.Equal(t, m, got)
+
+	m.State = eventsource.StateDeleting
+	newRev, err := s.Update(t.Context(), m, rev)
+	require.NoError(t, err)
+	_, err = s.Update(t.Context(), m, rev)
+	require.ErrorIs(t, err, eventsource.ErrConflict, "a stale revision never overwrites")
+	require.ErrorIs(t, s.Remove(t.Context(), "u1", rev), eventsource.ErrConflict, "a stale revision never removes")
+
+	require.NoError(t, s.Remove(t.Context(), "u1", newRev))
+	_, _, err = s.Get(t.Context(), "u1")
+	require.ErrorIs(t, err, eventsource.ErrNotFound)
+	entries, err = s.List(t.Context())
+	require.NoError(t, err)
+	assert.Empty(t, entries)
+}
@@ -0,0 +1,50 @@
+// Package lambdaenv validates function environment variables.
+package lambdaenv
+
+import (
+	"errors"
+	"regexp"
+	"strings"
+)
+
+var (
+	ErrName     = errors.New("invalid environment variable name")
+	ErrReserved = errors.New("reserved environment variable name")
+	ErrValue    = errors.New("environment variable contains a NUL byte")
+	ErrSize     = errors.New("environment variables exceed 4096 bytes")
+)
+
+var namePattern = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9_]+$`)
+
+var reserved = map[string]struct{}{
+	"AWS_ACCESS_KEY": {}, "AWS_ACCESS_KEY_ID": {}, "AWS_SECRET_ACCESS_KEY": {}, "AWS_SESSION_TOKEN": {},
+	"AWS_DEFAULT_REGION": {}, "AWS_REGION": {}, "AWS_EXECUTION_ENV": {},
+	"AWS_LAMBDA_RUNTIME_API": {}, "AWS_LAMBDA_FUNCTION_NAME": {}, "AWS_LAMBDA_FUNCTION_MEMORY_SIZE": {},
+	"AWS_LAMBDA_FUNCTION_VERSION": {}, "AWS_LAMBDA_INITIALIZATION_TYPE": {},
+	"AWS_LAMBDA_LOG_GROUP_NAME": {}, "AWS_LAMBDA_LOG_STREAM_NAME": {},
+	"AWS_LAMBDA_MAX_CONCURRENCY": {}, "AWS_LAMBDA_METADATA_API": {}, "AWS_LAMBDA_METADATA_TOKEN": {},
+	"LAMBDA_TASK_ROOT": {}, "LAMBDA_RUNTIME_DIR": {},
+}
+
+// Validate checks values without including customer names or values in errors.
+func Validate(vars map[string]string) error {
+	for k := range vars {
+		if !namePattern.MatchString(k) {
+			return ErrName
+		}
+	}
+	size := 0
+	for k, v := range vars {
+		if _, ok := reserved[k]; ok {
+			return ErrReserved
+		}
+		if strings.ContainsRune(v, '\x00') {
+			return ErrValue
+		}
+		size += len(k) + len(v)
+	}
+	if size > 4096 {
+		return ErrSize
+	}
+	return nil
+}
@@ -0,0 +1,33 @@
+package lambdaenv_test
+
+import (
+	"errors"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/lambdaenv"
+)
+
+func TestValidate(t *testing.T) {
+	for name, tc := range map[string]struct {
+		vars map[string]string
+		want error
+	}{
+		"absent": {},
+		"literal values and configurable settings": {vars: map[string]string{"GREETING": "hello $WORLD", "PATH": "/my/bin", "AWS_ENDPOINT_URL": "https://example.test"}},
+		"maximum size":                   {vars: map[string]string{"AA": strings.Repeat("x", 4094)}},
+		"empty name":                     {vars: map[string]string{"": "x"}, want: lambdaenv.ErrName},
+		"one-character name":             {vars: map[string]string{"A": "x"}, want: lambdaenv.ErrName},
+		"invalid character":              {vars: map[string]string{"bad-name": "x"}, want: lambdaenv.ErrName},
+		"reserved credential":            {vars: map[string]string{"AWS_SECRET_ACCESS_KEY": "x"}, want: lambdaenv.ErrReserved},
+		"invalid name precedes reserved": {vars: map[string]string{"A": "x", "AWS_REGION": "x"}, want: lambdaenv.ErrName},
+		"nul value":                      {vars: map[string]string{"VALUE": "x\x00"}, want: lambdaenv.ErrValue},
+		"over maximum":                   {vars: map[string]string{"AA": strings.Repeat("x", 4095)}, want: lambdaenv.ErrSize},
+	} {
+		t.Run(name, func(t *testing.T) {
+			if got := lambdaenv.Validate(tc.vars); !errors.Is(got, tc.want) {
+				t.Fatalf("Validate() = %v, want %v", got, tc.want)
+			}
+		})
+	}
+}
@@ -3,11 +3,16 @@ package cli
 import (
 	"context"
 	"fmt"
+	"maps"
 	"os"
+	"slices"
 	"time"

+	"github.com/aws/aws-sdk-go/service/ec2"
+	ec2inventory "github.com/mulgadc/spinifex/spinifex/domains/ec2/inventory"
 	ekscluster "github.com/mulgadc/spinifex/spinifex/domains/eks/cluster"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/spf13/cobra"
 )

@@ -75,3 +80,57 @@ func runAdminEksRestoreSnapshot(cmd *cobra.Command, _ []string) {
 	fmt.Printf("New control plane %s launched (snapshot %s)\n", out.NewInstanceID, out.Snapshot)
 	fmt.Printf("Status: %s\n", out.Status)
 }
+
+var adminEksOrphansCmd = &cobra.Command{
+	Use:   "orphans",
+	Short: "List running EKS control-plane VMs that have no cluster record",
+	Long: `orphans reports each running EKS control-plane VM whose cluster record, or whole
+account bucket, is missing (ORPHAN), and each it could not judge because a read
+failed (UNREADABLE). It only reads and never creates or removes anything.
+
+It lists the system account's instances through the strict inventory, so if any
+configured node does not answer it prints "listing incomplete" and exits non-zero
+without a partial list. It cannot say which node was silent. An instance with no
+ENI, or whose ENI is gone or not tagged as an EKS control plane, is skipped.
+
+Exit status is non-zero when the listing is incomplete or any row is ORPHAN or
+UNREADABLE.`,
+	Run: runAdminEksOrphans,
+}
+
+func init() {
+	adminEksCmd.AddCommand(adminEksOrphansCmd)
+}
+
+func runAdminEksOrphans(_ *cobra.Command, _ []string) {
+	cfg, nc, err := loadConfigAndConnect()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	defer nc.Close()
+
+	js, err := jetstream.New(nc)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	nodeIDs := slices.Sorted(maps.Keys(cfg.Nodes))
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	list := func(ctx context.Context) (*ec2.DescribeInstancesOutput, error) {
+		return ec2inventory.NewNATSInstances(nc, nodeIDs).ListAccountInstances(ctx, awsidentifiers.GlobalAccountID)
+	}
+	eni := ekscluster.ENITagReader(ec2inventory.NewENITagReader(js, awsidentifiers.GlobalAccountID))
+
+	orphans, unreadable, err := ekscluster.ReportControlPlaneOrphans(ctx, os.Stdout, js, list, eni, time.Now())
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+	fmt.Printf("%d orphan(s), %d unreadable\n", orphans, unreadable)
+	if orphans+unreadable > 0 {
+		os.Exit(1)
+	}
+}
@@ -18,8 +18,12 @@ import (

 // releaseOnly is cfg as a release build would offer it: ALPNv1 alone.
 func releaseOnly(cfg *tls.Config) *tls.Config {
+	return protocolOnly(cfg, lambdav1.ALPNv1)
+}
+
+func protocolOnly(cfg *tls.Config, protocol string) *tls.Config {
 	cfg = cfg.Clone()
-	cfg.NextProtos = []string{lambdav1.ALPNv1}
+	cfg.NextProtos = []string{protocol}
 	verify := cfg.VerifyConnection
 	cfg.VerifyConnection = func(cs tls.ConnectionState) error {
 		cs.NegotiatedProtocol = hostlink.Protocol // let only the ALPN offer differ
@@ -28,6 +32,42 @@ func releaseOnly(cfg *tls.Config) *tls.Config {
 	return cfg
 }

+func TestTestLinkRejectsPreviousProtocolBothWays(t *testing.T) {
+	const previous = "spinifex-lambda-testlink/2"
+	t.Run("old guest", func(t *testing.T) {
+		ln := listen(t)
+		g := newGeneration(t, gen1)
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		hostSide := acceptAsync(g, ln, ctx, 2*time.Second)
+		id := guestFor(t, g, ln.Addr().String())
+		raw, err := net.Dial("tcp", ln.Addr().String())
+		require.NoError(t, err)
+		defer raw.Close()
+		require.Error(t, tls.Client(raw, protocolOnly(id.ClientConfig(), previous)).HandshakeContext(ctx))
+		require.Error(t, (<-hostSide).err)
+	})
+	t.Run("old host", func(t *testing.T) {
+		ln := listen(t)
+		g := newGeneration(t, gen1)
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		hostErr := make(chan error, 1)
+		go func() {
+			c, err := ln.Accept()
+			if err != nil {
+				hostErr <- err
+				return
+			}
+			defer c.Close()
+			hostErr <- tls.Server(c, protocolOnly(g.ServerConfig(), previous)).HandshakeContext(ctx)
+		}()
+		_, err := hostlink.Dial(ctx, guestFor(t, g, ln.Addr().String()))
+		require.Error(t, err)
+		require.Error(t, <-hostErr)
+	})
+}
+
 func TestTestLinkProtocolIsNotTheReleaseProtocol(t *testing.T) {
 	assert.NotEqual(t, lambdav1.ALPNv1, hostlink.Protocol)
 }
@@ -23,7 +23,7 @@ import (
 // ControlName is the abstract Unix socket name the node's daemon serves. An
 // abstract name has no directory entry; the kernel frees it when the last
 // reference to the listening socket closes.
-const ControlName = "spinifex-lambda-testlink-control/3"
+const ControlName = "spinifex-lambda-testlink-control/4"

 // Service accounts at each end of the control socket.
 const (
@@ -43,9 +43,8 @@ func joinAfterFunc(ctx context.Context, f func()) func() {
 	}
 }

-// MaxCredentialBytes bounds the encoded credential; a session's three values
-// fit well inside it.
-const MaxCredentialBytes = 8 << 10
+// MaxCredentialBytes includes the credential and JSON-escaped function environment.
+const MaxCredentialBytes = 40 << 10

 // Declared control-socket bounds, independent of any message bus. Request and
 // Reply payloads are capped raw; the frame cap leaves MetadataHeadroomBytes
@@ -61,8 +60,8 @@ const (
 	MaxAdmitted = 2
 )

-// Credential is one execution-role session for one attempt; it goes to the
-// guest in the first frame and never on NATS, to disk or into a log.
+// Credential carries one execution-role session and its function environment
+// to the guest, never on NATS, to disk or into a log.
 type Credential = hostlink.Credential

 // ErrNotSent means the exchange failed before any request byte was written, so
@@ -83,6 +83,17 @@ func TestControl_OneRequestOneReply(t *testing.T) {
 	assert.Equal(t, testCred, s.cred.Load())
 }

+func TestControl_FullEnvironmentFitsWithCredential(t *testing.T) {
+	name := controlName(t)
+	s, _ := serveControl(t, name, os.Getuid(), nil)
+	cred := testCred
+	cred.SessionToken = string(bytes.Repeat([]byte("t"), 8<<10-128))
+	cred.Environment = map[string]string{"AA": string(bytes.Repeat([]byte{1}, 4094))}
+	_, err := client(name, os.Getuid()).Exchange(t.Context(), []byte(`{}`), cred, soon())
+	require.NoError(t, err)
+	assert.Equal(t, cred, s.cred.Load())
+}
+
 func TestControl_RequestSizeBoundary(t *testing.T) {
 	name := controlName(t)
 	s, _ := serveControl(t, name, os.Getuid(), nil)
@@ -219,7 +230,7 @@ func TestControl_BadFramesAreRefused(t *testing.T) {
 		"incomplete credential": append(rawFrame([]byte("{}")),
 			rawFrame([]byte(`{"access_key_id":"ASIA"}`))...),
 		"oversized credential": append(rawFrame([]byte("{}")),
-			rawFrame(bytes.Repeat([]byte("c"), MaxCredentialBytes+1))...),
+			rawFrame(bytes.Repeat([]byte("c"), MaxCredentialBytes+1))[:4]...),
 	}
 	for name, frame := range cases {
 		t.Run(name, func(t *testing.T) {
@@ -441,6 +452,7 @@ func TestControl_WriteErrorAfterDeliveryIsUnknown(t *testing.T) {
 const (
 	legacyControlName = "spinifex-lambda-testlink-control"
 	v2ControlName     = "spinifex-lambda-testlink-control/2"
+	v3ControlName     = "spinifex-lambda-testlink-control/3"
 )

 // A gateway and daemon on different control protocol names never connect, in
@@ -453,6 +465,8 @@ func TestControl_MixedProtocolNamesNeverConnect(t *testing.T) {
 		"new daemon, unversioned gateway": {prefix + ControlName, prefix + legacyControlName},
 		"/2 daemon, new gateway":          {prefix + v2ControlName, prefix + ControlName},
 		"new daemon, /2 gateway":          {prefix + ControlName, prefix + v2ControlName},
+		"/3 daemon, new gateway":          {prefix + v3ControlName, prefix + ControlName},
+		"new daemon, /3 gateway":          {prefix + ControlName, prefix + v3ControlName},
 	} {
 		t.Run(name, func(t *testing.T) {
 			s, _ := serveControl(t, tc.daemon, os.Getuid(), nil)
@@ -478,7 +492,9 @@ func TestControl_AcceptStoppedRunsBeforeTheDrain(t *testing.T) {
 		AcceptStopped: func() { close(stopped) }}
 	done := make(chan error, 1)
 	go func() { done <- srv.Serve(context.Background(), ln) }()
-	go func() { _, _ = client(name, os.Getuid()).Exchange(context.Background(), []byte("{}"), testCred, soon()) }()
+	go func() {
+		_, _ = client(name, os.Getuid()).Exchange(context.Background(), []byte("{}"), testCred, soon())
+	}()
 	select {
 	case <-inHandle:
 	case <-time.After(5 * time.Second):
@@ -83,6 +83,9 @@ type Daemon struct {
 	MTU     int
 	WorkDir string
 	HTTP    *http.Client
+	// SDKEndpoint and SDKCA are public gateway connection settings for the guest.
+	SDKEndpoint string
+	SDKCA       []byte
 	// Image is the prepared runtime image every launch boots; the daemon
 	// prepares it once before serving and keeps it open for its whole life.
 	Image *Image
@@ -582,6 +585,10 @@ func (d *Daemon) prepare(req Request, code []byte, dir, tap, mac, ip string) (*v
 	entries["test-package-sha256"] = []byte(req.CodeSHA256)
 	entries["test-grant"] = grantEntry(req.Grant)
 	entries["test-handler"] = []byte(req.Handler)
+	if d.SDKEndpoint != "" && len(d.SDKCA) != 0 {
+		entries["test-sdk-endpoint"] = []byte(d.SDKEndpoint)
+		entries["test-sdk-ca"] = d.SDKCA
+	}
 	netcfgBytes := fmt.Appendf(nil, "NIC0_MAC=%s\nNIC0_CIDR=%s/24\nNIC0_DEFAULT=0\n", mac, ip)
 	if d.Firecracker != "" {
 		archive := make(map[string][]byte, len(entries)+1)
@@ -64,7 +64,9 @@ func TestDaemon_StartThenFailedLaunchedWriteIsUnknown(t *testing.T) {
 	owner := &failLaunched{Grants: d.Grants}
 	d.Grants = owner

-	rep := d.Handle(context.Background(), g.request(srv.URL, code), testCred)
+	cred := testCred
+	cred.Environment = map[string]string{"GREETING": "private-function-environment"}
+	rep := d.Handle(context.Background(), g.request(srv.URL, code), cred)
 	// An identity is read only from a live process's pidfd, so one was started.
 	require.Len(t, owner.got, 1, "no process was started, so this run proves nothing about after-start replies")
 	require.NotZero(t, owner.got[0].PID)
@@ -72,7 +74,7 @@ func TestDaemon_StartThenFailedLaunchedWriteIsUnknown(t *testing.T) {
 	require.Equal(t, OutcomeUnknown, rep.Outcome, rep.Error)
 	require.Equal(t, ErrLaunchNotRecorded.Error(), rep.Error)

-	secrets := []string{testCred.AccessKeyID, testCred.SecretAccessKey, testCred.SessionToken, srv.URL}
+	secrets := []string{cred.AccessKeyID, cred.SecretAccessKey, cred.SessionToken, cred.Environment["GREETING"], srv.URL}
 	files := 0
 	require.NoError(t, filepath.WalkDir(d.WorkDir, func(p string, e fs.DirEntry, err error) error {
 		if err != nil || e.IsDir() {
@@ -13,9 +13,6 @@ import (
 // handed an Active function it cannot run.
 func (d *Daemon) Readiness(imagePin string, nodes int) function.Readiness {
 	return func(_ context.Context, rec function.Record) error {
-		if rec.EnvCipher != "" {
-			return notReady("The function cannot run here: test-link Lambda does not support environment variables.")
-		}
 		if !validHandler(rec.Handler) {
 			return notReady("The function cannot run here: test-link Lambda runs only a handler of 1 to 128 characters with no whitespace.")
 		}
@@ -133,8 +133,7 @@ func TestReadiness_RefusesFunctionsTheTestLinkCannotRun(t *testing.T) {
 	storedFunctionHandler(t, svc, "edgehandler", strings.Repeat("h", 128), "", 9)

 	require.NoError(t, svc.ActivatePending(t.Context()))
-	assert.Equal(t, "The function cannot run here: test-link Lambda does not support environment variables.", functionState(t, svc, "env").StateReason)
-	assert.Equal(t, function.StatePending, functionState(t, svc, "env").State)
+	assert.Equal(t, function.StateActive, functionState(t, svc, "env").State)
 	assert.Equal(t, "The function cannot run here: its code is larger than test-link Lambda accepts.", functionState(t, svc, "huge").StateReason)
 	assert.Equal(t, function.StatePending, functionState(t, svc, "huge").State)
 	assert.Equal(t, function.StateActive, functionState(t, svc, "edge").State, "the largest accepted code copy activates")
@@ -159,7 +158,9 @@ func TestReadiness_ServingEndsWhenAcceptingStops(t *testing.T) {
 	}
 	d.serve(t.Context(), ln, srv)
 	require.True(t, d.Serving())
-	go func() { _, _ = (&ControlClient{Name: name, DaemonUID: os.Getuid(), RequestLimit: MaxFrameBytes, ReplyLimit: MaxFrameBytes, IOTimeout: 5 * time.Second}).Exchange(context.Background(), []byte("{}"), testCred, time.Now().Add(5*time.Second)) }()
+	go func() {
+		_, _ = (&ControlClient{Name: name, DaemonUID: os.Getuid(), RequestLimit: MaxFrameBytes, ReplyLimit: MaxFrameBytes, IOTimeout: 5 * time.Second}).Exchange(context.Background(), []byte("{}"), testCred, time.Now().Add(5*time.Second))
+	}()
 	select {
 	case <-inHandle:
 	case <-time.After(5 * time.Second):
@@ -368,6 +368,36 @@ func TestPrepare_ScopeAndGrantEntryComeFromTheGrant(t *testing.T) {
 	require.Equal(t, "ap-southeast-2/123456789012/fn1/env-x/3", strings.TrimSpace(string(expect)))
 }

+func TestPrepare_DeliversSDKSettingsForEachVMM(t *testing.T) {
+	for _, firecracker := range []bool{false, true} {
+		t.Run(fmt.Sprintf("firecracker=%t", firecracker), func(t *testing.T) {
+			req := baseRequest("https://example.invalid/code", []byte("zip"))
+			req.Grant = execution.Grant{GrantID: "g-7", AttemptID: req.AttemptID, EnvironmentID: "env-x", Generation: 3, Node: testNode}
+			d := &Daemon{HostIP: "127.0.0.1", SDKEndpoint: "https://10.15.8.1:9999", SDKCA: []byte("public-ca")}
+			if firecracker {
+				d.Firecracker = "/test/firecracker"
+			}
+			dir := filepath.Join(t.TempDir(), "attempt")
+			_, _, ln, err := d.prepare(req, []byte("zip"), dir, "tap0", "02:00:00:00:00:01", "10.0.0.2")
+			require.NoError(t, err)
+			defer ln.Close()
+			endpointPath, caPath := filepath.Join(dir, "fwcfg-test-sdk-endpoint"), filepath.Join(dir, "fwcfg-test-sdk-ca")
+			if firecracker {
+				root := filepath.Join(t.TempDir(), "config")
+				extract(t, filepath.Join(dir, configArchive), root)
+				entryDir := filepath.Join(root, "by_name", lambdav1.FwCfgPrefix)
+				endpointPath, caPath = filepath.Join(entryDir, "test-sdk-endpoint", "raw"), filepath.Join(entryDir, "test-sdk-ca", "raw")
+			}
+			endpoint, err := os.ReadFile(endpointPath)
+			require.NoError(t, err)
+			ca, err := os.ReadFile(caPath)
+			require.NoError(t, err)
+			require.Equal(t, d.SDKEndpoint, string(endpoint))
+			require.Equal(t, d.SDKCA, ca)
+		})
+	}
+}
+
 // elfHeader is a bare 64-bit little-endian shared-object header for machine.
 func elfHeader(machine elf.Machine) []byte {
 	h := make([]byte, 64)
@@ -7,6 +7,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/hex"
+	"encoding/pem"
 	"errors"
 	"fmt"
 	"io"
@@ -82,15 +83,21 @@ func (d *Daemon) startLambdaTestLink() {
 		return
 	}
 	httpClient, trustErr := d.lambdaTestLinkHTTP()
+	sdkEndpoint, sdkCA, sdkErr := d.lambdaTestLinkSDKConfig()
+	if sdkErr != nil {
+		slog.Warn("Lambda test link: SDK gateway settings unavailable", "err", sdkErr)
+	}
 	imageDir := filepath.Join(microvmDir, "lambda-testlink")
 	tl := &testlink.Daemon{
-		Node:    d.node,
-		Grants:  grants,
-		Taps:    taps,
-		IPs:     d.mgmtIPAllocator,
-		Bridge:  d.mgmtBridgeName(),
-		HostIP:  d.mgmtBridgeIP,
-		WorkDir: workDir,
+		Node:        d.node,
+		Grants:      grants,
+		Taps:        taps,
+		IPs:         d.mgmtIPAllocator,
+		Bridge:      d.mgmtBridgeName(),
+		HostIP:      d.mgmtBridgeIP,
+		WorkDir:     workDir,
+		SDKEndpoint: sdkEndpoint,
+		SDKCA:       sdkCA,

 		Initializer:       filepath.Join(imageDir, "hardkill.so"),
 		InitializerSHA256: lambdaHardKillSHA256,
@@ -135,7 +142,6 @@ func (d *Daemon) startLambdaTestLink() {
 // missed, by a crash or an earlier refusal, waits for the next pass.
 const lambdaActivationInterval = 30 * time.Second

-
 // lambdaTestLinkReady runs the serving checks and, only once they pass,
 // prepares the pinned image into tl; a refusal leaves tl.Image nil.
 func lambdaTestLinkReady(tl *testlink.Daemon, imageDir, imagePin string, trustErr error) error {
@@ -201,4 +207,32 @@ func (d *Daemon) lambdaTestLinkHTTP() (*http.Client, error) {
 	}}, nil
 }

+// lambdaTestLinkSDKConfig returns public settings for the guest's AWS SDK.
+func (d *Daemon) lambdaTestLinkSDKConfig() (string, []byte, error) {
+	if d.mgmtBridgeIP == "" || d.config.NATS.CACert == "" {
+		return "", nil, errors.New("management address or cluster CA not configured")
+	}
+	ca, err := os.ReadFile(d.config.NATS.CACert)
+	if err != nil {
+		return "", nil, errors.New("cluster CA unreadable")
+	}
+	if len(ca) > 64<<10 || !x509.NewCertPool().AppendCertsFromPEM(ca) {
+		return "", nil, errors.New("cluster CA invalid or too large")
+	}
+	var public []byte
+	for rest := ca; len(rest) != 0; {
+		block, next := pem.Decode(rest)
+		if block == nil {
+			break
+		}
+		rest = next
+		if block.Type == "CERTIFICATE" {
+			if _, err := x509.ParseCertificate(block.Bytes); err == nil {
+				public = append(public, pem.EncodeToMemory(block)...)
+			}
+		}
+	}
+	return d.resolveSystemGatewayBaseURL(), public, nil
+}
+
 var _ testlink.Taps = (*host.OVSPlumber)(nil)
@@ -154,8 +154,17 @@ func (a *MgmtIPAllocator) BindKV(jsManager *JetStreamManager, node string) {
 // an address is free cluster-wide on its own — and is refused otherwise.
 func (a *MgmtIPAllocator) Allocate(instanceID string) (string, error) {
 	a.mu.Lock()
+	// Addresses this node's own instances hold stay off the table even when the
+	// cluster record has lost their entries.
+	held := make([]MgmtIPEntry, 0, len(a.allocated))
+	for id, ip := range a.allocated {
+		held = append(held, MgmtIPEntry{IP: ip, InstanceID: id})
+	}
 	if ip, ok := a.allocated[instanceID]; ok {
 		a.mu.Unlock()
+		if other := heldByOther(held, ip, instanceID); other != "" {
+			return "", fmt.Errorf("mgmt IP %s for %s is also held locally by %s", ip, instanceID, other)
+		}
 		return ip, nil
 	}
 	jsManager, node, subnet := a.jsManager, a.node, a.subnet
@@ -171,29 +180,7 @@ func (a *MgmtIPAllocator) Allocate(instanceID string) (string, error) {
 	// CAS write that follows is a harmless no-op against the same revision.
 	var allocatedIP string
 	var mutateErr error
-	_, err := updateMgmtIPAMWithRetry(jsManager, subnet, func(r *MgmtIPRecord) {
-		if r.Subnet == "" {
-			r.Subnet = subnet
-		}
-
-		// Idempotent: an instance that already holds an address (allocated
-		// by this node earlier, or by another node before a restart lost
-		// the local cache) keeps it rather than taking a second one.
-		for _, e := range r.Allocated {
-			if e.InstanceID == instanceID {
-				allocatedIP = e.IP
-				return
-			}
-		}
-
-		ip, findErr := nextFreeIP(a.baseIP, a.rangeMin, a.rangeMax, r.Allocated)
-		if findErr != nil {
-			mutateErr = findErr
-			return
-		}
-		r.Allocated = append(r.Allocated, MgmtIPEntry{IP: ip, InstanceID: instanceID, Node: node})
-		allocatedIP = ip
-	}, true)
+	_, err := updateMgmtIPAMWithRetry(jsManager, subnet, a.allocMutate(instanceID, node, subnet, held, &allocatedIP, &mutateErr), true)
 	if err != nil {
 		return "", fmt.Errorf("mgmt IP CAS update for subnet %s: %w", subnet, err)
 	}
@@ -209,6 +196,16 @@ func (a *MgmtIPAllocator) Allocate(instanceID string) (string, error) {
 	return allocatedIP, nil
 }

+// heldByOther names a cached instance other than id holding ip, or "".
+func heldByOther(held []MgmtIPEntry, ip, id string) string {
+	for _, e := range held {
+		if e.IP == ip && e.InstanceID != id {
+			return e.InstanceID
+		}
+	}
+	return ""
+}
+
 // nextFreeIP scans rangeMin-rangeMax on the /24 rooted at baseIP, skipping
 // every IP already present in allocated regardless of which entry's Node it
 // belongs to — the range is shared
@@ -511,3 +508,37 @@ func (d *Daemon) primeMgmtNeighEntry(instanceID, mgmtIP, mgmtMAC string) {
 			"instanceId", instanceID, "mgmtIP", mgmtIP, "mgmtMAC", mgmtMAC, "bridge", bridge, "err", err)
 	}
 }
+
+// allocMutate returns the CAS callback for Allocate. Results go out through
+// ipOut and errOut because casUpdate's mutate has no return value.
+func (a *MgmtIPAllocator) allocMutate(instanceID, node, subnet string, held []MgmtIPEntry, ipOut *string, errOut *error) func(*MgmtIPRecord) {
+	return func(r *MgmtIPRecord) {
+		// Each CAS attempt starts clean, so a retry never returns an earlier attempt's result.
+		*ipOut, *errOut = "", nil
+		if r.Subnet == "" {
+			r.Subnet = subnet
+		}
+
+		// Idempotent: an instance that already holds an address (allocated
+		// by this node earlier, or by another node before a restart lost
+		// the local cache) keeps it rather than taking a second one.
+		for _, e := range r.Allocated {
+			if e.InstanceID == instanceID {
+				if other := heldByOther(held, e.IP, instanceID); other != "" {
+					*errOut = fmt.Errorf("mgmt IP %s recorded for %s is held locally by %s", e.IP, instanceID, other)
+					return
+				}
+				*ipOut = e.IP
+				return
+			}
+		}
+
+		ip, findErr := nextFreeIP(a.baseIP, a.rangeMin, a.rangeMax, append(held, r.Allocated...))
+		if findErr != nil {
+			*errOut = findErr
+			return
+		}
+		r.Allocated = append(r.Allocated, MgmtIPEntry{IP: ip, InstanceID: instanceID, Node: node})
+		*ipOut = ip
+	}
+}
@@ -472,6 +472,76 @@ func TestMgmtIPAllocator_Allocate_IdempotentAcrossNodes(t *testing.T) {
 	}
 }

+// An instance whose cluster-record entry has gone (the record lost it while the
+// instance kept running) still holds its address on this node, so a new
+// allocation must not be handed that address.
+func TestMgmtIPAllocator_Allocate_SkipsLocallyHeldWhenRecordLost(t *testing.T) {
+	t.Parallel()
+	jsm := newTestMgmtJSM(t)
+
+	a, err := NewMgmtIPAllocator("10.41.8.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	a.BindKV(jsm, "node-1")
+	cleanupMgmtIPAM(t, jsm, a)
+
+	held, err := a.Allocate("i-system")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := jsm.clusterKV.Delete(context.Background(), mgmtIPAMKeyPrefix+a.subnet); err != nil {
+		t.Fatalf("drop record: %v", err)
+	}
+
+	got, err := a.Allocate("lambda-attempt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got == held {
+		t.Fatalf("new allocation reused %s, still held by i-system", got)
+	}
+	if again, err := a.Allocate("i-system"); err != nil || again != held {
+		t.Errorf("holder re-allocate = %q, %v; want %q", again, err, held)
+	}
+}
+
+// Allocate refuses a recorded entry whose address another local instance holds.
+func TestMgmtIPAllocator_Allocate_RefusesRecordedConflictWithLocalHolder(t *testing.T) {
+	t.Parallel()
+	jsm := newTestMgmtJSM(t)
+
+	a, err := NewMgmtIPAllocator("10.42.8.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	a.BindKV(jsm, "node-1")
+	cleanupMgmtIPAM(t, jsm, a)
+
+	held, err := a.Allocate("i-system")
+	if err != nil {
+		t.Fatal(err)
+	}
+	other, err := NewMgmtIPAllocator("10.42.8.2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	other.BindKV(jsm, "node-2")
+	if err := jsm.clusterKV.Delete(context.Background(), mgmtIPAMKeyPrefix+a.subnet); err != nil {
+		t.Fatalf("drop record: %v", err)
+	}
+	// The record now names the held address for a different instance.
+	other.Claim("lambda-attempt", held)
+
+	if got, err := a.Allocate("lambda-attempt"); err == nil {
+		t.Fatalf("Allocate returned %s, which i-system holds locally", got)
+	}
+	a.Claim("i-dup", held)
+	if got, err := a.Allocate("i-dup"); err == nil {
+		t.Fatalf("cached duplicate returned %s, which i-system holds locally", got)
+	}
+}
+
 // TestMgmtIPAllocator_Release_FreesAddressClusterWide proves Release's
 // effect is visible cluster-wide: a second allocator, not the one that
 // released the address, can immediately take it.
@@ -868,8 +938,9 @@ func TestMgmtIPAllocator_ClaimLetsTheNewHolderReleaseItLater(t *testing.T) {
 		t.Errorf("the holder could not release its own address: entry still %+v", entry)
 	}

-	// And the address is genuinely back, not merely unrecorded.
-	reused, err := node1.Allocate("i-next")
+	// And the address is genuinely back, not merely unrecorded. node-1's cache
+	// still names its dropped copy, so the holder that released it allocates.
+	reused, err := node2.Allocate("i-next")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1110,3 +1181,34 @@ func TestMgmtIPAllocator_ReleaseDropsLocalOnForeignHolderAndKVFailure(t *testing
 		t.Errorf("local entries after release with KV down = %d, want 0", node3.AllocatedCount())
 	}
 }
+
+// A CAS retry that sees the conflicting entry gone must report its own
+// result, not the refusal left by the earlier attempt.
+func TestMgmtIPAllocator_AllocMutate_RetryClearsEarlierConflict(t *testing.T) {
+	a, err := NewMgmtIPAllocator("10.30.8.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	held := []MgmtIPEntry{{IP: "10.30.8.10", InstanceID: "i-sqs"}}
+	var ip string
+	var mutateErr error
+	mutate := a.allocMutate("i-fn", "node1", "10.30.8.0/24", held, &ip, &mutateErr)
+
+	mutate(&MgmtIPRecord{Allocated: []MgmtIPEntry{{IP: "10.30.8.10", InstanceID: "i-fn", Node: "node1"}}})
+	if mutateErr == nil || ip != "" {
+		t.Fatalf("conflicted attempt: ip=%q err=%v, want refusal", ip, mutateErr)
+	}
+
+	clean := &MgmtIPRecord{}
+	mutate(clean)
+	if mutateErr != nil {
+		t.Fatalf("retry kept the earlier refusal: %v", mutateErr)
+	}
+	if ip == "" || ip == "10.30.8.10" {
+		t.Fatalf("retry allocated %q, want a free address other than the held one", ip)
+	}
+	want := MgmtIPEntry{IP: ip, InstanceID: "i-fn", Node: "node1"}
+	if len(clean.Allocated) != 1 || clean.Allocated[0] != want {
+		t.Fatalf("record = %+v, want [%+v]", clean.Allocated, want)
+	}
+}
@@ -0,0 +1,44 @@
+//go:build lambdatestlink
+
+//test:in-package exercises daemon-owned gateway configuration
+package daemon
+
+import (
+	"encoding/pem"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLambdaSDKConfigUsesManagementGatewayAndClusterCA(t *testing.T) {
+	server := httptest.NewTLSServer(nil)
+	defer server.Close()
+	ca := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw})
+	path := filepath.Join(t.TempDir(), "ca.pem")
+	require.NoError(t, os.WriteFile(path, ca, 0o600))
+	d := &Daemon{mgmtBridgeIP: "10.15.8.1", config: &config.Config{
+		NATS: config.NATSConfig{CACert: path}, AWSGW: config.AWSGWConfig{Host: "0.0.0.0:9999"},
+	}}
+	endpoint, gotCA, err := d.lambdaTestLinkSDKConfig()
+	require.NoError(t, err)
+	assert.Equal(t, "https://10.15.8.1:9999", endpoint)
+	assert.Equal(t, ca, gotCA)
+
+	withKey := append(append([]byte{}, ca...), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: []byte("private")})...)
+	withKey = append(withKey, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: []byte("not a certificate")})...)
+	require.NoError(t, os.WriteFile(path, withKey, 0o600))
+	_, gotCA, err = d.lambdaTestLinkSDKConfig()
+	require.NoError(t, err)
+	assert.Equal(t, ca, gotCA, "only public certificates reach the guest")
+
+	require.NoError(t, os.WriteFile(path, []byte("bad CA"), 0o600))
+	_, _, err = d.lambdaTestLinkSDKConfig()
+	require.Error(t, err)
+	_, err = d.lambdaTestLinkHTTP()
+	require.Error(t, err, "configured invalid download trust must still refuse serving")
+}
@@ -393,7 +393,9 @@ func launchService(config *config.ClusterConfig) error {
 	}
 	// Built but routed only in test-link builds: Lambda stays out of the public
 	// router until its runtime image exists.
-	lambdaControlPlane := lambdaawsapi.NewRegistration(lambdaawsapi.Deps{
+	lambdaMappings := lambdaMappingStore(natsConn)
+	lambdaEventSources := newESMRuntime(lambdaMappings, stsService, nodeConfig.Region)
+	lambdaDeps := lambdaawsapi.Deps{
 		Meta: lambdafunction.NewNATSClient(natsConn),
 		Codes: func(ctx context.Context) (lambdaawsapi.CodeStore, error) {
 			store, err := lambdaCodeStore.Store(ctx)
@@ -405,8 +407,13 @@ func launchService(config *config.ClusterConfig) error {
 		Roles:  iamdomain.NewRoleResolver(iamService),
 		Trusts: lambdaRoleTrusts(iamService),
 		AEAD:   lambdaKey.AEAD,
-		Exec:   lambdaTestExecutor(natsConn, config.Node, lambdaCodeStore, stsService),
-	})
+		Exec:   lambdaTestExecutor(natsConn, config.Node, lambdaCodeStore, stsService, lambdaKey.AEAD),
+		Mappings: func(ctx context.Context) (lambdaawsapi.MappingStore, error) {
+			return lambdaMappings(ctx)
+		},
+		Sources: lambdaEventSources.sources(),
+	}
+	lambdaControlPlane := lambdaawsapi.NewRegistration(lambdaDeps)

 	// ECR auth bridge: load (or first-run create) the ES256 signing key from the
 	// cluster-replicated awsgw-keys KV bucket, then build the token issuer
@@ -779,6 +786,7 @@ func launchService(config *config.ClusterConfig) error {
 	go quotaSvc.RunBedrockRPMSync(janitorCtx, js)

 	handler := gw.SetupRoutes()
+	lambdaEventSources.start(janitorCtx, handler, lambdaDeps, sqsEndpoint(registryHost, registryPort))

 	// Load TLS certificate
 	cert, err := tls.LoadX509KeyPair(nodeConfig.AWSGW.TLSCert, nodeConfig.AWSGW.TLSKey)
@@ -4,7 +4,10 @@ import (
 	"context"

 	"github.com/aws/aws-sdk-go/service/sts"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/eventsource"
 	stsdomain "github.com/mulgadc/spinifex/spinifex/domains/sts"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
 )

 // lambdaEnvironmentRoleIssuer is the in-process STS entry point the test-link
@@ -15,3 +18,14 @@ type lambdaEnvironmentRoleIssuer interface {
 }

 var _ lambdaEnvironmentRoleIssuer = (*stsdomain.STSServiceImpl)(nil)
+
+// lambdaMappingStore opens the event-source mapping bucket on first use.
+func lambdaMappingStore(nc *nats.Conn) func(ctx context.Context) (*eventsource.Store, error) {
+	return func(ctx context.Context) (*eventsource.Store, error) {
+		js, err := jetstream.New(nc)
+		if err != nil {
+			return nil, err
+		}
+		return eventsource.Open(ctx, js)
+	}
+}
@@ -4,6 +4,7 @@ package awsgw

 import (
 	"context"
+	"crypto/cipher"
 	"log/slog"
 	"time"

@@ -18,7 +19,7 @@ import (

 // lambdaTestExecutor sends each Invoke to this node's test-link daemon over
 // the control socket. Without the daemon's service account it does not serve.
-func lambdaTestExecutor(nc *nats.Conn, node string, codes *codestore.Manager, sessions lambdaEnvironmentRoleIssuer) awsapi.Executor {
+func lambdaTestExecutor(nc *nats.Conn, node string, codes *codestore.Manager, sessions lambdaEnvironmentRoleIssuer, aead cipher.AEAD) awsapi.Executor {
 	daemonUID, err := testlink.LookupUID(testlink.DaemonUser)
 	if err != nil {
 		slog.Error("Lambda test link: control peer account unavailable; Invoke is not served", "err", err)
@@ -31,6 +32,7 @@ func lambdaTestExecutor(nc *nats.Conn, node string, codes *codestore.Manager, se
 		Control:      testLinkControl(daemonUID),
 		RequestLimit: testlink.MaxFrameBytes,
 		Sessions:     sessions,
+		AEAD:         aead,
 		Node:         node,
 		ImageDigest:  lambdaImageDigest,
 		Codes: func(ctx context.Context) (codePresigner, error) {
@@ -0,0 +1,593 @@
+//go:build lambdatestlink
+
+package awsgw
+
+import (
+	"context"
+	"crypto/rand"
+	"crypto/tls"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/sqs"
+	"github.com/aws/aws-sdk-go/service/sqs/sqsiface"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/eventsource"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	stsdomain "github.com/mulgadc/spinifex/spinifex/domains/sts"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// Poller timing. The receive is a long poll, so an idle queue costs one
+// request per esmWaitSeconds; a session is reissued well before it expires.
+const (
+	esmWaitSeconds    = 20
+	esmReconcileEvery = 5 * time.Second
+	esmMaxBackoff     = 60 * time.Second
+	esmSessionSeconds = 900
+	esmRenewBefore    = 10 * time.Minute
+	esmDeleteTimeout  = 15 * time.Second
+	// esmMaxEvent is the synchronous request bound one event must fit.
+	esmMaxEvent = 6 * 1024 * 1024
+	// esmMaxReason bounds a recorded failure reason.
+	esmMaxReason = 512
+)
+
+// esmInvoker runs one event through Invoke's checks, without Invoke's IAM check.
+type esmInvoker interface {
+	InvokeEventSource(ctx context.Context, accountID, region, name string, payload []byte) (awsapi.SyncResult, error)
+}
+
+var _ esmInvoker = awsapi.Deps{}
+
+// esmStore is what the poller needs of the mapping store.
+type esmStore interface {
+	Get(ctx context.Context, uuid string) (eventsource.Mapping, uint64, error)
+	List(ctx context.Context) ([]eventsource.Entry, error)
+	Update(ctx context.Context, m eventsource.Mapping, revision uint64) (uint64, error)
+	Remove(ctx context.Context, uuid string, revision uint64) error
+}
+
+var _ esmStore = (*eventsource.Store)(nil)
+
+// esmRuntime polls every enabled SQS mapping on this gateway. It is one loop
+// per gateway with no cross-node lease, so it is correct on a single node only.
+type esmRuntime struct {
+	store    func(ctx context.Context) (esmStore, error)
+	sessions lambdaEnvironmentRoleIssuer
+	region   string
+	meta     function.Client
+	invoke   esmInvoker
+	// queue builds an SQS client signed as one session; nil means the
+	// gateway handler, set by start.
+	queue func(cred *credentials.Credentials) sqsiface.SQSAPI
+	now   func() time.Time
+	// interval and wait pace reconcile and back-off; zero uses the defaults.
+	interval time.Duration
+	backoff  time.Duration
+
+	mu      sync.Mutex
+	handler http.Handler
+	// endpoint is the public SQS base URL queue URLs are built on.
+	endpoint string
+}
+
+func newESMRuntime(open func(ctx context.Context) (*eventsource.Store, error), sessions lambdaEnvironmentRoleIssuer, region string) *esmRuntime {
+	store := func(ctx context.Context) (esmStore, error) { return open(ctx) }
+	return &esmRuntime{store: store, sessions: sessions, region: region, now: time.Now}
+}
+
+func (r *esmRuntime) sources() awsapi.SourceChecker { return r }
+
+// start runs the poller until ctx ends; deps is the composed Lambda control plane.
+func (r *esmRuntime) start(ctx context.Context, h http.Handler, deps awsapi.Deps, sqsEndpoint string) {
+	r.mu.Lock()
+	r.handler = h
+	r.endpoint = sqsEndpoint
+	if r.queue == nil {
+		r.queue = r.handlerQueue
+	}
+	r.meta = deps.Meta
+	r.invoke = deps
+	r.mu.Unlock()
+	go r.run(ctx)
+}
+
+// handlerTransport carries a signed SDK request straight into the gateway
+// handler. It never leaves the TLS-terminating gateway, so it is marked as
+// received over TLS and is authorized exactly as one from a client.
+type handlerTransport struct{ h http.Handler }
+
+func (t handlerTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	in := req.Clone(req.Context())
+	in.TLS = &tls.ConnectionState{Version: tls.VersionTLS13, HandshakeComplete: true}
+	in.RemoteAddr = "127.0.0.1:0"
+	in.RequestURI = req.URL.RequestURI()
+	if in.Host == "" {
+		in.Host = req.URL.Host
+	}
+	rec := httptest.NewRecorder()
+	t.h.ServeHTTP(rec, in)
+	return rec.Result(), nil
+}
+
+func (r *esmRuntime) handlerQueue(cred *credentials.Credentials) sqsiface.SQSAPI {
+	r.mu.Lock()
+	h, endpoint := r.handler, r.endpoint
+	r.mu.Unlock()
+	sess := session.Must(session.NewSession(&aws.Config{
+		Region:      aws.String(r.region),
+		Endpoint:    aws.String(endpoint),
+		Credentials: cred,
+		HTTPClient:  &http.Client{Transport: handlerTransport{h: h}},
+		MaxRetries:  aws.Int(0),
+	}))
+	return sqs.New(sess)
+}
+
+// esmSession is one execution-role session the poller signs with.
+type esmSession struct {
+	cred    *credentials.Credentials
+	keyID   string
+	role    string
+	expires time.Time
+	scope   stsdomain.LambdaEnvironmentSessionScope
+}
+
+func (r *esmRuntime) issue(ctx context.Context, rec function.Record) (*esmSession, error) {
+	b := make([]byte, 6)
+	if _, err := rand.Read(b); err != nil {
+		return nil, err
+	}
+	scope := stsdomain.LambdaEnvironmentSessionScope{
+		AccountID:     rec.AccountID,
+		FunctionID:    rec.FunctionID,
+		FunctionARN:   arn.FormatLambdaFunction(rec.Region, rec.AccountID, rec.FunctionName),
+		EnvironmentID: "esm-" + hex.EncodeToString(b),
+	}
+	out, err := r.sessions.AssumeRoleForLambdaEnvironment(ctx, stsdomain.LambdaEnvironmentSessionRequest{
+		AccountID: scope.AccountID, RoleARN: rec.Role, FunctionARN: scope.FunctionARN,
+		FunctionID: scope.FunctionID, EnvironmentID: scope.EnvironmentID, MaxLifetimeSeconds: esmSessionSeconds,
+	})
+	if err != nil {
+		return nil, err
+	}
+	c := out.Credentials
+	return &esmSession{
+		cred: credentials.NewStaticCredentials(aws.StringValue(c.AccessKeyId), aws.StringValue(c.SecretAccessKey),
+			aws.StringValue(c.SessionToken)),
+		keyID: aws.StringValue(c.AccessKeyId), role: rec.Role, expires: aws.TimeValue(c.Expiration), scope: scope,
+	}, nil
+}
+
+func (r *esmRuntime) revoke(s *esmSession) {
+	if s == nil {
+		return
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), revokeTimeout)
+	defer cancel()
+	if err := r.sessions.RevokeLambdaEnvironmentSession(ctx, s.keyID, s.scope); err != nil {
+		slog.Warn("Lambda event source: poller session not revoked; it stays valid until expiry",
+			"environment_id", s.scope.EnvironmentID, "err", err)
+	}
+}
+
+// fresh returns s while it is the function's role and outlives need;
+// otherwise it issues a replacement and revokes s, which no request holds.
+func (r *esmRuntime) fresh(ctx context.Context, rec function.Record, s *esmSession, need time.Duration) (*esmSession, error) {
+	if s != nil && s.role == rec.Role && s.expires.Sub(r.now()) > need {
+		return s, nil
+	}
+	next, err := r.issue(ctx, rec)
+	if err != nil {
+		return s, err
+	}
+	r.revoke(s)
+	return next, nil
+}
+
+// eventClient renews *s before each event unless it outlives that event's
+// invocation and delete, so no acknowledgement runs on expired credentials.
+func (r *esmRuntime) eventClient(ctx context.Context, rec function.Record, s **esmSession) func() (sqsiface.SQSAPI, error) {
+	need := time.Duration(rec.Timeout)*time.Second + esmDeleteTimeout + time.Minute
+	return func() (sqsiface.SQSAPI, error) {
+		next, err := r.fresh(ctx, rec, *s, need)
+		*s = next
+		if err != nil {
+			return nil, err
+		}
+		return r.queue(next.cred), nil
+	}
+}
+
+func queueName(queueARN string) (account, name string) {
+	parts := strings.Split(queueARN, ":")
+	if len(parts) != 6 {
+		return "", ""
+	}
+	return parts[4], parts[5]
+}
+
+// queueURL builds the queue's URL rather than asking GetQueueUrl, which the
+// execution role is not required to allow.
+func (r *esmRuntime) queueURL(queueARN string) (string, error) {
+	r.mu.Lock()
+	endpoint := r.endpoint
+	r.mu.Unlock()
+	account, name := queueName(queueARN)
+	if endpoint == "" || name == "" {
+		return "", errors.New("SQS is not served on this gateway")
+	}
+	return endpoint + "/" + account + "/" + name, nil
+}
+
+// QueueVisibilityTimeout reads the queue as the function's execution role
+// with a short-lived session it revokes before returning.
+func (r *esmRuntime) QueueVisibilityTimeout(ctx context.Context, rec function.Record, queueARN string) (int64, error) {
+	s, err := r.issue(ctx, rec)
+	if err != nil {
+		if code, ok := awserrors.ResolveErrorCode(err); ok && code == awserrors.ErrorAccessDenied {
+			return 0, awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, "The role defined for the function cannot be assumed by Lambda.")
+		}
+		return 0, err
+	}
+	defer r.revoke(s)
+	q := r.queue(s.cred)
+	url, err := r.queueURL(queueARN)
+	if err != nil {
+		return 0, awserrors.Errorf(awserrors.ErrorServiceException, "%s", err.Error())
+	}
+	out, err := q.GetQueueAttributesWithContext(ctx, &sqs.GetQueueAttributesInput{
+		QueueUrl: aws.String(url), AttributeNames: aws.StringSlice([]string{sqs.QueueAttributeNameVisibilityTimeout}),
+	})
+	if err != nil {
+		return 0, sourceError(err)
+	}
+	var v int64
+	if _, err := fmt.Sscan(aws.StringValue(out.Attributes[sqs.QueueAttributeNameVisibilityTimeout]), &v); err != nil {
+		return 0, fmt.Errorf("queue visibility timeout: %w", err)
+	}
+	return v, nil
+}
+
+// sourceError reports a queue the execution role cannot read as the
+// caller's fault, as Lambda does.
+func sourceError(err error) error {
+	var ae awserr.Error
+	if !errors.As(err, &ae) {
+		return err
+	}
+	switch ae.Code() {
+	case "AccessDenied", "AccessDeniedException":
+		return awserrors.Errorf(awserrors.ErrorInvalidParameterValueException,
+			"The function execution role does not have permissions to call GetQueueAttributes on SQS")
+	case sqs.ErrCodeQueueDoesNotExist, "QueueDoesNotExist":
+		return awserrors.Errorf(awserrors.ErrorInvalidParameterValueException,
+			"Error occurred while ReceiveMessage. SQS Error Code: AWS.SimpleQueueService.NonExistentQueue. SQS Error Message: The specified queue does not exist.")
+	}
+	return err
+}
+
+type esmWorker struct {
+	cancel context.CancelFunc
+	done   chan struct{}
+}
+
+func (w *esmWorker) stop() {
+	w.cancel()
+	<-w.done
+}
+
+func (r *esmRuntime) run(ctx context.Context) {
+	workers := map[string]*esmWorker{}
+	defer func() {
+		for _, w := range workers {
+			w.stop()
+		}
+	}()
+	every := r.interval
+	if every == 0 {
+		every = esmReconcileEvery
+	}
+	t := time.NewTicker(every)
+	defer t.Stop()
+	for {
+		r.reconcile(ctx, workers)
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+		}
+	}
+}
+
+// reconcile starts a worker for each enabled mapping, including those
+// written before a restart, and finishes each delete only after its worker
+// has stopped, so no invoke or message delete follows it.
+func (r *esmRuntime) reconcile(ctx context.Context, workers map[string]*esmWorker) {
+	store, err := r.store(ctx)
+	if err != nil {
+		slog.Warn("Lambda event source: mapping store unavailable", "err", err)
+		return
+	}
+	entries, err := store.List(ctx)
+	if err != nil {
+		slog.Warn("Lambda event source: list mappings failed", "err", err)
+		return
+	}
+	seen := map[string]bool{}
+	for _, e := range entries {
+		m := e.Mapping
+		seen[m.UUID] = true
+		switch m.State {
+		case eventsource.StateEnabled:
+			if workers[m.UUID] == nil {
+				wctx, cancel := context.WithCancel(ctx)
+				w := &esmWorker{cancel: cancel, done: make(chan struct{})}
+				workers[m.UUID] = w
+				go func() {
+					defer close(w.done)
+					r.work(wctx, m)
+				}()
+			}
+		case eventsource.StateDeleting:
+			if w := workers[m.UUID]; w != nil {
+				w.stop()
+				delete(workers, m.UUID)
+			}
+			if err := r.finishDelete(ctx, store, m.UUID); err != nil {
+				slog.Warn("Lambda event source: mapping not removed; retrying", "uuid", m.UUID, "err", err)
+			}
+		}
+	}
+	for id, w := range workers {
+		if !seen[id] {
+			w.stop()
+			delete(workers, id)
+		}
+	}
+}
+
+func (r *esmRuntime) finishDelete(ctx context.Context, store esmStore, id string) error {
+	m, rev, err := store.Get(ctx, id)
+	if errors.Is(err, eventsource.ErrNotFound) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	if m.State != eventsource.StateDeleting {
+		return nil
+	}
+	return store.Remove(ctx, id, rev)
+}
+
+// setReason records a receive failure on an enabled mapping, or clears it
+// back to USER_INITIATED with reason "".
+func (r *esmRuntime) setReason(ctx context.Context, id, reason string) {
+	if reason == "" {
+		reason = eventsource.ReasonUserInitiated
+	} else {
+		reason = "PROBLEM: " + reason
+		if len(reason) > esmMaxReason {
+			reason = reason[:esmMaxReason]
+		}
+	}
+	store, err := r.store(ctx)
+	if err != nil {
+		return
+	}
+	m, rev, err := store.Get(ctx, id)
+	if err != nil || m.State != eventsource.StateEnabled || m.StateTransitionReason == reason {
+		return
+	}
+	m.StateTransitionReason = reason
+	if _, err := store.Update(ctx, m, rev); err != nil && !errors.Is(err, eventsource.ErrConflict) {
+		slog.Warn("Lambda event source: reason not recorded", "uuid", id, "err", err)
+	}
+}
+
+func (r *esmRuntime) wait(ctx context.Context, d time.Duration) {
+	t := time.NewTimer(d)
+	defer t.Stop()
+	select {
+	case <-ctx.Done():
+	case <-t.C:
+	}
+}
+
+// work polls one mapping until ctx ends. A failure keeps the mapping, records
+// why, and retries after a back-off.
+func (r *esmRuntime) work(ctx context.Context, m eventsource.Mapping) {
+	var s *esmSession
+	defer func() { r.revoke(s) }()
+	base := r.backoff
+	if base == 0 {
+		base = time.Second
+	}
+	backoff := base
+	fail := func(reason string) {
+		r.setReason(context.WithoutCancel(ctx), m.UUID, reason)
+		r.wait(ctx, backoff)
+		backoff = min(2*backoff, esmMaxBackoff)
+	}
+	for ctx.Err() == nil {
+		rec, _, err := r.meta.Get(ctx, m.AccountID, m.Region, m.FunctionName)
+		if err != nil {
+			fail("Function lookup failed: " + err.Error())
+			continue
+		}
+		next, err := r.fresh(ctx, rec, s, esmRenewBefore)
+		s = next
+		if err != nil {
+			fail("The execution role could not be assumed: " + err.Error())
+			continue
+		}
+		q := r.queue(s.cred)
+		url, err := r.queueURL(m.EventSourceARN)
+		if err != nil {
+			fail("Error occurred while ReceiveMessage: " + err.Error())
+			continue
+		}
+		out, err := q.ReceiveMessageWithContext(ctx, &sqs.ReceiveMessageInput{
+			QueueUrl:              aws.String(url),
+			MaxNumberOfMessages:   aws.Int64(m.BatchSize),
+			WaitTimeSeconds:       aws.Int64(esmWaitSeconds),
+			AttributeNames:        aws.StringSlice([]string{"All"}),
+			MessageAttributeNames: aws.StringSlice([]string{"All"}),
+		})
+		if err != nil {
+			if ctx.Err() == nil {
+				fail("Error occurred while ReceiveMessage: " + err.Error())
+			}
+			continue
+		}
+		r.setReason(ctx, m.UUID, "")
+		if err := r.process(ctx, m, r.eventClient(ctx, rec, &s), url, out.Messages); err != nil {
+			slog.Warn("Lambda event source: batch not processed; messages stay on the queue", "uuid", m.UUID, "err", err)
+			r.wait(ctx, backoff)
+			backoff = min(2*backoff, esmMaxBackoff)
+			continue
+		}
+		backoff = base
+	}
+}
+
+// sqsRecord and sqsAttribute are the AWS SQS event record shape.
+type sqsRecord struct {
+	MessageID         string                  `json:"messageId"`
+	ReceiptHandle     string                  `json:"receiptHandle"`
+	Body              string                  `json:"body"`
+	Attributes        map[string]string       `json:"attributes"`
+	MessageAttributes map[string]sqsAttribute `json:"messageAttributes"`
+	MD5OfBody         string                  `json:"md5OfBody"`
+	EventSource       string                  `json:"eventSource"`
+	EventSourceARN    string                  `json:"eventSourceARN"`
+	AWSRegion         string                  `json:"awsRegion"`
+}
+
+type sqsAttribute struct {
+	StringValue      *string  `json:"stringValue,omitempty"`
+	BinaryValue      []byte   `json:"binaryValue,omitempty"`
+	StringListValues []string `json:"stringListValues"`
+	BinaryListValues [][]byte `json:"binaryListValues"`
+	DataType         string   `json:"dataType"`
+}
+
+func eventRecord(m eventsource.Mapping, msg *sqs.Message) sqsRecord {
+	rec := sqsRecord{
+		MessageID: aws.StringValue(msg.MessageId), ReceiptHandle: aws.StringValue(msg.ReceiptHandle),
+		Body: aws.StringValue(msg.Body), Attributes: map[string]string{}, MessageAttributes: map[string]sqsAttribute{},
+		MD5OfBody: aws.StringValue(msg.MD5OfBody), EventSource: "aws:sqs", EventSourceARN: m.EventSourceARN, AWSRegion: m.Region,
+	}
+	for k, v := range msg.Attributes {
+		rec.Attributes[k] = aws.StringValue(v)
+	}
+	for k, v := range msg.MessageAttributes {
+		rec.MessageAttributes[k] = sqsAttribute{
+			StringValue: v.StringValue, BinaryValue: v.BinaryValue, StringListValues: []string{},
+			BinaryListValues: [][]byte{}, DataType: aws.StringValue(v.DataType),
+		}
+	}
+	return rec
+}
+
+// esmEvent is one invocation's payload and the messages it carries.
+type esmEvent struct {
+	Payload  []byte
+	Messages []*sqs.Message
+}
+
+// splitEvents groups messages into events whose encoding fits esmMaxEvent.
+// A message too large for an event of its own is returned in oversized and
+// is neither invoked nor deleted.
+func splitEvents(m eventsource.Mapping, msgs []*sqs.Message) (events []esmEvent, oversized []*sqs.Message) {
+	const frame = len(`{"Records":[]}`)
+	var recs []json.RawMessage
+	var batch []*sqs.Message
+	size := frame
+	flush := func() {
+		if len(recs) == 0 {
+			return
+		}
+		payload, _ := json.Marshal(struct {
+			Records []json.RawMessage `json:"Records"`
+		}{recs})
+		events = append(events, esmEvent{Payload: payload, Messages: batch})
+		recs, batch, size = nil, nil, frame
+	}
+	for _, msg := range msgs {
+		enc, err := json.Marshal(eventRecord(m, msg))
+		if err != nil || frame+len(enc) > esmMaxEvent {
+			oversized = append(oversized, msg)
+			continue
+		}
+		extra := len(enc)
+		if len(recs) > 0 {
+			extra++
+		}
+		if size+extra > esmMaxEvent {
+			flush()
+			extra = len(enc)
+		}
+		recs = append(recs, enc)
+		batch = append(batch, msg)
+		size += extra
+	}
+	flush()
+	return events, oversized
+}
+
+// process invokes each event and deletes its messages only on a classified
+// success with no function error; anything else leaves them to reappear.
+// process invokes and acknowledges each event in turn. It returns the first
+// failure, so the worker backs off before receiving again.
+func (r *esmRuntime) process(ctx context.Context, m eventsource.Mapping, client func() (sqsiface.SQSAPI, error), url string, msgs []*sqs.Message) error {
+	events, oversized := splitEvents(m, msgs)
+	if len(oversized) > 0 {
+		r.setReason(ctx, m.UUID, fmt.Sprintf("%d message(s) exceed the %d-byte event limit and are not delivered", len(oversized), esmMaxEvent))
+	}
+	for _, ev := range events {
+		// A stopped worker invokes nothing further; delete waits for it.
+		if ctx.Err() != nil {
+			return nil
+		}
+		q, err := client()
+		if err != nil {
+			return fmt.Errorf("execution role session: %w", err)
+		}
+		res, err := r.invoke.InvokeEventSource(ctx, m.AccountID, m.Region, m.FunctionName, ev.Payload)
+		if err != nil {
+			return err
+		}
+		if res.FunctionError != "" {
+			return fmt.Errorf("function error %s", res.FunctionError)
+		}
+		entries := make([]*sqs.DeleteMessageBatchRequestEntry, len(ev.Messages))
+		for i, msg := range ev.Messages {
+			entries[i] = &sqs.DeleteMessageBatchRequestEntry{Id: aws.String(fmt.Sprint(i)), ReceiptHandle: msg.ReceiptHandle}
+		}
+		dctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), esmDeleteTimeout)
+		out, err := q.DeleteMessageBatchWithContext(dctx, &sqs.DeleteMessageBatchInput{QueueUrl: aws.String(url), Entries: entries})
+		cancel()
+		if err != nil {
+			slog.Warn("Lambda event source: processed messages not deleted; they will be delivered again", "uuid", m.UUID, "err", err)
+		} else if len(out.Failed) > 0 {
+			slog.Warn("Lambda event source: some processed messages not deleted", "uuid", m.UUID, "failed", len(out.Failed))
+		}
+	}
+	return nil
+}
@@ -0,0 +1,23 @@
+//go:build !lambdatestlink
+
+package awsgw
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/eventsource"
+)
+
+// Without the test link no mapping is polled and CreateEventSourceMapping has
+// no queue reader, so it refuses; Lambda is unrouted there anyway.
+type esmRuntime struct{}
+
+func newESMRuntime(func(context.Context) (*eventsource.Store, error), lambdaEnvironmentRoleIssuer, string) *esmRuntime {
+	return &esmRuntime{}
+}
+
+func (*esmRuntime) sources() awsapi.SourceChecker { return nil }
+
+func (*esmRuntime) start(context.Context, http.Handler, awsapi.Deps, string) {}
@@ -0,0 +1,517 @@
+//go:build lambdatestlink
+
+//test:in-package — the poller, its session handling and its in-process
+// transport are unexported test-link internals.
+package awsgw
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/service/sqs"
+	"github.com/aws/aws-sdk-go/service/sqs/sqsiface"
+	"github.com/aws/aws-sdk-go/service/sts"
+	"github.com/mulgadc/bluebottle/pkg/iampolicy"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/eventsource"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	stsdomain "github.com/mulgadc/spinifex/spinifex/domains/sts"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/credential"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/testlink"
+	"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/policyauth"
+	"github.com/stretchr/testify/require"
+)
+
+const esmQueueARN = "arn:aws:sqs:ap-southeast-2:123456789012:in"
+
+// esmQueue is an SQS fake: Receive hands out queued messages once, Delete
+// records what was deleted.
+type esmQueue struct {
+	sqsiface.SQSAPI
+	mu         sync.Mutex
+	pending    []*sqs.Message
+	deleted    []string
+	receives   int
+	receiveErr error
+	urls       []string
+	// redeliver keeps received messages pending, as an expired visibility would.
+	redeliver bool
+}
+
+func (q *esmQueue) ReceiveMessageWithContext(ctx aws.Context, in *sqs.ReceiveMessageInput, _ ...request.Option) (*sqs.ReceiveMessageOutput, error) {
+	q.mu.Lock()
+	q.receives++
+	q.urls = append(q.urls, aws.StringValue(in.QueueUrl))
+	err := q.receiveErr
+	n := min(int(aws.Int64Value(in.MaxNumberOfMessages)), len(q.pending))
+	msgs := q.pending[:n]
+	if !q.redeliver {
+		q.pending = q.pending[n:]
+	}
+	q.mu.Unlock()
+	if err != nil {
+		return nil, err
+	}
+	if len(msgs) == 0 {
+		t := time.NewTimer(20 * time.Millisecond)
+		defer t.Stop()
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case <-t.C:
+		}
+	}
+	return &sqs.ReceiveMessageOutput{Messages: msgs}, nil
+}
+
+func (q *esmQueue) DeleteMessageBatchWithContext(_ aws.Context, in *sqs.DeleteMessageBatchInput, _ ...request.Option) (*sqs.DeleteMessageBatchOutput, error) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	for _, e := range in.Entries {
+		q.deleted = append(q.deleted, aws.StringValue(e.ReceiptHandle))
+	}
+	return &sqs.DeleteMessageBatchOutput{}, nil
+}
+
+func (q *esmQueue) state() (deleted []string, receives int) {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+	return append([]string(nil), q.deleted...), q.receives
+}
+
+// esmInvokes records each event and returns the configured outcome.
+type esmInvokes struct {
+	mu       sync.Mutex
+	payloads []string
+	result   awsapi.SyncResult
+	err      error
+	// started, when set, is closed as the first call begins, which then
+	// blocks until release is closed.
+	started, release chan struct{}
+	// during, when set, runs inside every invocation.
+	during func()
+}
+
+func (f *esmInvokes) InvokeEventSource(_ context.Context, _, _, _ string, payload []byte) (awsapi.SyncResult, error) {
+	f.mu.Lock()
+	f.payloads = append(f.payloads, string(payload))
+	started, release, during := f.started, f.release, f.during
+	f.started = nil
+	f.mu.Unlock()
+	if during != nil {
+		during()
+	}
+	if started != nil {
+		close(started)
+		<-release
+	}
+	return f.result, f.err
+}
+
+func (f *esmInvokes) count() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return len(f.payloads)
+}
+
+// esmSessions issues numbered sessions with a fixed lifetime.
+type esmSessions struct {
+	mu       sync.Mutex
+	life     time.Duration
+	issued   int
+	revoked  []string
+	issueErr error
+}
+
+func (f *esmSessions) AssumeRoleForLambdaEnvironment(_ context.Context, _ stsdomain.LambdaEnvironmentSessionRequest) (*sts.AssumeRoleOutput, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	if f.issueErr != nil {
+		return nil, f.issueErr
+	}
+	f.issued++
+	id := "ASIA" + strings.Repeat("K", f.issued)
+	return &sts.AssumeRoleOutput{Credentials: &sts.Credentials{
+		AccessKeyId: aws.String(id), SecretAccessKey: aws.String("s"), SessionToken: aws.String("t"),
+		Expiration: aws.Time(time.Now().Add(f.life)),
+	}}, nil
+}
+
+func (f *esmSessions) RevokeLambdaEnvironmentSession(_ context.Context, keyID string, _ stsdomain.LambdaEnvironmentSessionScope) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.revoked = append(f.revoked, keyID)
+	return nil
+}
+
+type esmMeta struct{ function.Client }
+
+func (esmMeta) Get(_ context.Context, accountID, region, name string) (function.Record, uint64, error) {
+	return function.Record{FunctionID: "fn1", AccountID: accountID, Region: region, FunctionName: name,
+		Role: "arn:aws:iam::123456789012:role/fn-exec", Timeout: 3}, 1, nil
+}
+
+func esmMapping(id string) eventsource.Mapping {
+	return eventsource.Mapping{
+		FormatVersion: eventsource.FormatVersion, UUID: id, AccountID: "123456789012", Region: "ap-southeast-2",
+		FunctionName: "f", EventSourceARN: esmQueueARN, BatchSize: 10, State: eventsource.StateEnabled,
+		StateTransitionReason: eventsource.ReasonUserInitiated,
+	}
+}
+
+func msg(id, body string) *sqs.Message {
+	return &sqs.Message{MessageId: aws.String(id), ReceiptHandle: aws.String("rh-" + id), Body: aws.String(body)}
+}
+
+func newTestESM(t *testing.T, q *esmQueue, inv *esmInvokes) (*esmRuntime, *eventsource.Store, *esmSessions) {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	store, err := eventsource.Open(context.Background(), js)
+	require.NoError(t, err)
+	sessions := &esmSessions{life: time.Hour}
+	r := newESMRuntime(func(context.Context) (*eventsource.Store, error) { return store, nil }, sessions, "ap-southeast-2")
+	r.meta = esmMeta{}
+	r.invoke = inv
+	r.queue = func(*credentials.Credentials) sqsiface.SQSAPI { return q }
+	r.endpoint = "https://gw.example:9999"
+	r.interval = 20 * time.Millisecond
+	r.backoff = 10 * time.Millisecond
+	return r, store, sessions
+}
+
+// Verifies: ADR-0009:S16
+func TestESM_DeletesOnlyOnClassifiedSuccess(t *testing.T) {
+	cases := []struct {
+		name   string
+		result awsapi.SyncResult
+		err    error
+		delete bool
+	}{
+		{name: "success", result: awsapi.SyncResult{ExecutedVersion: "$LATEST", Payload: []byte(`{}`)}, delete: true},
+		{name: "function error", result: awsapi.SyncResult{ExecutedVersion: "$LATEST", FunctionError: "Unhandled"}},
+		{name: "refusal", err: &awsapi.RefusalError{Err: errors.New("node busy")}},
+		{name: "unknown outcome", err: errors.New("the invocation outcome is unknown")},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			q := &esmQueue{}
+			inv := &esmInvokes{result: tc.result, err: tc.err}
+			r, _, _ := newTestESM(t, q, inv)
+			err := r.process(context.Background(), esmMapping("u1"), func() (sqsiface.SQSAPI, error) { return q, nil }, "url",
+				[]*sqs.Message{msg("1", "a"), msg("2", "b")})
+			require.Equal(t, tc.delete, err == nil, "only a processed batch reports success")
+			require.Equal(t, 1, inv.count())
+			deleted, _ := q.state()
+			if tc.delete {
+				require.Equal(t, []string{"rh-1", "rh-2"}, deleted)
+			} else {
+				require.Empty(t, deleted)
+			}
+		})
+	}
+}
+
+func TestESM_EventShapeAndSizeSplit(t *testing.T) {
+	m := esmMapping("u1")
+	quoted := strings.Repeat(`"`, 700*1024) // each byte encodes as two
+	msgs := []*sqs.Message{}
+	for i := range 6 {
+		msgs = append(msgs, msg(string(rune('a'+i)), quoted))
+	}
+	events, oversized := splitEvents(m, msgs)
+	require.Empty(t, oversized)
+	require.Len(t, events, 2, "8.4 MiB of encoded records must split under the 6 MiB bound")
+	total := 0
+	for _, ev := range events {
+		require.LessOrEqual(t, len(ev.Payload), esmMaxEvent)
+		total += len(ev.Messages)
+	}
+	require.Equal(t, 6, total)
+	require.Contains(t, string(events[0].Payload), `"eventSource":"aws:sqs"`)
+	require.Contains(t, string(events[0].Payload), `"eventSourceARN":"`+esmQueueARN+`"`)
+	require.Contains(t, string(events[0].Payload), `"messageAttributes":{}`)
+
+	huge := strings.Repeat("\x00", esmMaxEvent/6+1) // each byte encodes as \u0000
+	events, oversized = splitEvents(m, []*sqs.Message{msg("big", huge), msg("ok", "small")})
+	require.Len(t, oversized, 1)
+	require.Equal(t, "big", aws.StringValue(oversized[0].MessageId))
+	require.Len(t, events, 1)
+
+	q := &esmQueue{}
+	inv := &esmInvokes{result: awsapi.SyncResult{ExecutedVersion: "$LATEST"}}
+	r, _, _ := newTestESM(t, q, inv)
+	require.NoError(t, r.process(context.Background(), m, func() (sqsiface.SQSAPI, error) { return q, nil }, "url", []*sqs.Message{msg("big", huge)}))
+	require.Zero(t, inv.count(), "an oversized message is never invoked")
+	deleted, _ := q.state()
+	require.Empty(t, deleted, "an oversized message is never deleted")
+}
+
+// A mapping written before the gateway starts is polled, and a Deleting
+// record is removed only after its worker stopped; nothing follows it.
+func TestESM_RestartReloadAndDeleteStopsWorker(t *testing.T) {
+	q := &esmQueue{pending: []*sqs.Message{msg("1", "hello")}}
+	inv := &esmInvokes{result: awsapi.SyncResult{ExecutedVersion: "$LATEST"}}
+	r, store, _ := newTestESM(t, q, inv)
+	ctx := t.Context()
+	require.NoError(t, store.Create(ctx, esmMapping("u1")))
+
+	go r.run(ctx)
+	require.Eventually(t, func() bool { d, _ := q.state(); return len(d) == 1 }, 5*time.Second, 10*time.Millisecond)
+	_, receives := q.state()
+	require.Positive(t, receives)
+	q.mu.Lock()
+	firstURL := q.urls[0]
+	q.mu.Unlock()
+	require.Equal(t, "https://gw.example:9999/123456789012/in", firstURL)
+
+	m, rev, err := store.Get(ctx, "u1")
+	require.NoError(t, err)
+	m.State = eventsource.StateDeleting
+	_, err = store.Update(ctx, m, rev)
+	require.NoError(t, err)
+	require.Eventually(t, func() bool {
+		_, _, err := store.Get(ctx, "u1")
+		return errors.Is(err, eventsource.ErrNotFound)
+	}, 5*time.Second, 10*time.Millisecond)
+
+	_, after := q.state()
+	q.mu.Lock()
+	q.pending = []*sqs.Message{msg("2", "late")}
+	q.mu.Unlock()
+	time.Sleep(200 * time.Millisecond)
+	deleted, receivesNow := q.state()
+	require.Equal(t, after, receivesNow, "no receive after the delete completed")
+	require.Equal(t, []string{"rh-1"}, deleted)
+	require.Equal(t, 1, inv.count())
+}
+
+// A delete requested while a batch is running completes only after that batch
+// finished, so its message delete precedes the record's removal.
+func TestESM_DeleteWaitsForInFlightBatch(t *testing.T) {
+	q := &esmQueue{pending: []*sqs.Message{msg("1", "hello")}}
+	inv := &esmInvokes{result: awsapi.SyncResult{ExecutedVersion: "$LATEST"},
+		started: make(chan struct{}), release: make(chan struct{})}
+	started := inv.started
+	r, store, _ := newTestESM(t, q, inv)
+	ctx := t.Context()
+	require.NoError(t, store.Create(ctx, esmMapping("u4")))
+	go r.run(ctx)
+	<-started
+
+	m, rev, err := store.Get(ctx, "u4")
+	require.NoError(t, err)
+	m.State = eventsource.StateDeleting
+	_, err = store.Update(ctx, m, rev)
+	require.NoError(t, err)
+	require.Never(t, func() bool {
+		_, _, err := store.Get(ctx, "u4")
+		return errors.Is(err, eventsource.ErrNotFound)
+	}, 300*time.Millisecond, 10*time.Millisecond, "the delete must wait for the running batch")
+
+	close(inv.release)
+	require.Eventually(t, func() bool {
+		_, _, err := store.Get(ctx, "u4")
+		return errors.Is(err, eventsource.ErrNotFound)
+	}, 5*time.Second, 10*time.Millisecond)
+	deleted, _ := q.state()
+	require.Equal(t, []string{"rh-1"}, deleted, "the finished batch's message was deleted before removal")
+}
+
+// A Deleting record left by a crash before the gateway restarted is removed.
+func TestESM_InterruptedDeleteResumes(t *testing.T) {
+	q := &esmQueue{}
+	r, store, _ := newTestESM(t, q, &esmInvokes{})
+	ctx := t.Context()
+	m := esmMapping("u2")
+	m.State = eventsource.StateDeleting
+	require.NoError(t, store.Create(ctx, m))
+	go r.run(ctx)
+	require.Eventually(t, func() bool {
+		_, _, err := store.Get(ctx, "u2")
+		return errors.Is(err, eventsource.ErrNotFound)
+	}, 5*time.Second, 10*time.Millisecond)
+	_, receives := q.state()
+	require.Zero(t, receives)
+}
+
+func TestESM_SessionReissuedBeforeExpiry(t *testing.T) {
+	r, _, sessions := newTestESM(t, &esmQueue{}, &esmInvokes{})
+	rec, _, _ := esmMeta{}.Get(context.Background(), "123456789012", "ap-southeast-2", "f")
+	sessions.life = esmRenewBefore + time.Minute
+	s, err := r.fresh(context.Background(), rec, nil, esmRenewBefore)
+	require.NoError(t, err)
+	same, err := r.fresh(context.Background(), rec, s, esmRenewBefore)
+	require.NoError(t, err)
+	require.Same(t, s, same, "a session far from expiry is kept")
+
+	r.now = func() time.Time { return time.Now().Add(2 * time.Minute) }
+	next, err := r.fresh(context.Background(), rec, s, esmRenewBefore)
+	require.NoError(t, err)
+	require.NotSame(t, s, next)
+	require.Equal(t, []string{s.keyID}, sessions.revoked, "the replaced session is revoked")
+
+	rec.Role = "arn:aws:iam::123456789012:role/other"
+	r.now = time.Now
+	third, err := r.fresh(context.Background(), rec, next, esmRenewBefore)
+	require.NoError(t, err)
+	require.NotSame(t, next, third, "a changed execution role gets a new session")
+}
+
+// Time passes during each invocation; the second event of one receive must
+// run and be acknowledged on a renewed session, not the one about to expire.
+func TestESM_EachSplitEventRunsOnACoveringSession(t *testing.T) {
+	q := &esmQueue{}
+	var mu sync.Mutex
+	now := time.Now()
+	inv := &esmInvokes{result: awsapi.SyncResult{ExecutedVersion: "$LATEST"}, during: func() {
+		mu.Lock()
+		now = now.Add(29 * time.Minute)
+		mu.Unlock()
+	}}
+	r, _, sessions := newTestESM(t, q, inv)
+	r.now = func() time.Time { mu.Lock(); defer mu.Unlock(); return now }
+	sessions.life = 30 * time.Minute
+	rec, _, _ := esmMeta{}.Get(context.Background(), "123456789012", "ap-southeast-2", "f")
+	big := strings.Repeat("a", esmMaxEvent/2)
+	var s *esmSession
+	t.Cleanup(func() { r.revoke(s) })
+	err := r.process(context.Background(), esmMapping("u1"), r.eventClient(context.Background(), rec, &s), "url",
+		[]*sqs.Message{msg("1", big), msg("2", big)})
+	require.NoError(t, err)
+	require.Equal(t, 2, inv.count(), "the two messages cannot share one event")
+	require.Equal(t, 2, sessions.issued, "the second event renews the session")
+	require.Equal(t, []string{"ASIAK"}, sessions.revoked)
+	deleted, _ := q.state()
+	require.Len(t, deleted, 2)
+}
+
+// A failed invocation makes the worker back off before it receives again.
+func TestESM_InvocationFailureBacksOff(t *testing.T) {
+	q := &esmQueue{pending: []*sqs.Message{msg("1", "hello")}, redeliver: true}
+	inv := &esmInvokes{err: errors.New("TooManyRequestsException: busy")}
+	r, store, _ := newTestESM(t, q, inv)
+	r.backoff = 300 * time.Millisecond
+	require.NoError(t, store.Create(context.Background(), esmMapping("u1")))
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan struct{})
+	go func() { defer close(done); r.run(ctx) }()
+	time.Sleep(time.Second)
+	cancel()
+	<-done
+	_, receives := q.state()
+	require.LessOrEqual(t, receives, 4, "a failing batch is retried after a back-off, not in a tight loop")
+	deleted, _ := q.state()
+	require.Empty(t, deleted)
+}
+
+func TestESM_ReceiveFailureRecordedAndCleared(t *testing.T) {
+	q := &esmQueue{receiveErr: errors.New("AWS.SimpleQueueService.NonExistentQueue: The specified queue does not exist.")}
+	r, store, _ := newTestESM(t, q, &esmInvokes{})
+	ctx, cancel := context.WithCancel(t.Context())
+	defer cancel()
+	require.NoError(t, store.Create(ctx, esmMapping("u3")))
+	done := make(chan struct{})
+	go func() { defer close(done); r.work(ctx, esmMapping("u3")) }()
+	require.Eventually(t, func() bool {
+		m, _, err := store.Get(ctx, "u3")
+		return err == nil && strings.HasPrefix(m.StateTransitionReason, "PROBLEM: ") &&
+			strings.Contains(m.StateTransitionReason, "NonExistentQueue")
+	}, 5*time.Second, 10*time.Millisecond)
+	m, _, err := store.Get(ctx, "u3")
+	require.NoError(t, err)
+	require.Equal(t, eventsource.StateEnabled, m.State, "a receive failure keeps the mapping")
+
+	q.mu.Lock()
+	q.receiveErr = nil
+	q.mu.Unlock()
+	require.Eventually(t, func() bool {
+		m, _, err := store.Get(ctx, "u3")
+		return err == nil && m.StateTransitionReason == eventsource.ReasonUserInitiated
+	}, 5*time.Second, 10*time.Millisecond)
+	cancel()
+	<-done
+}
+
+func TestESM_HandlerTransportCarriesSignedRequest(t *testing.T) {
+	var got *http.Request
+	var body string
+	h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		got = r
+		b, _ := io.ReadAll(r.Body)
+		body = string(b)
+		w.Header().Set("Content-Type", "application/x-amz-json-1.0")
+		_, _ = w.Write([]byte(`{"Attributes":{"VisibilityTimeout":"45"}}`))
+	})
+	r, _, _ := newTestESM(t, &esmQueue{}, &esmInvokes{})
+	r.handler = h
+	q := r.handlerQueue(credentials.NewStaticCredentials("ASIAX", "secret", "token"))
+	out, err := q.GetQueueAttributes(&sqs.GetQueueAttributesInput{QueueUrl: aws.String("https://gw.example:9999/123456789012/in"),
+		AttributeNames: aws.StringSlice([]string{"VisibilityTimeout"})})
+	require.NoError(t, err)
+	require.Equal(t, "45", aws.StringValue(out.Attributes["VisibilityTimeout"]))
+	require.Contains(t, got.Header.Get("Authorization"), "Credential=ASIAX/")
+	require.Contains(t, got.Header.Get("Authorization"), "/ap-southeast-2/sqs/aws4_request")
+	require.Equal(t, "token", got.Header.Get("X-Amz-Security-Token"))
+	require.Equal(t, "gw.example:9999", got.Host)
+	require.Contains(t, body, `"QueueUrl":"https://gw.example:9999/123456789012/in"`)
+	keys := policyauth.RequestConditionKeys(got, credential.Principal{})
+	require.Equal(t, "true", keys[iampolicy.KeySecureTransport], "a role requiring TLS must still authorize the poller")
+}
+
+// activeMeta serves the invocation record as an Active function with code.
+type activeMeta struct{ function.Client }
+
+func (activeMeta) Get(_ context.Context, _, _, _ string) (function.Record, uint64, error) {
+	rec := invocation(nil).Function
+	rec.State = function.StateActive
+	rec.Code.State = function.CodeStored
+	return rec, 1, nil
+}
+
+// noVersionExec returns a result with no ExecutedVersion, which Invoke rejects.
+type noVersionExec struct{}
+
+func (noVersionExec) InvokeSync(context.Context, awsapi.SyncInvocation) (awsapi.SyncResult, error) {
+	return awsapi.SyncResult{Payload: []byte(`{}`)}, nil
+}
+
+// Both batches run through Deps.InvokeEventSource: an invalid executor result
+// and a real executor refused by a node that already holds an attempt.
+func TestESM_InvokeEventSourceFailuresDeleteNothing(t *testing.T) {
+	store := grantStore(t)
+	held := &fakeControl{max: 1 << 20, err: errors.New("no reply")}
+	_, err := executor(held, nil, store).InvokeSync(context.Background(), invocation(nil))
+	require.Error(t, err)
+	busy := &fakeControl{max: 1 << 20, reply: &testlink.Reply{Outcome: testlink.OutcomeDone}}
+
+	cases := []struct {
+		name string
+		exec awsapi.Executor
+	}{
+		{name: "missing ExecutedVersion", exec: noVersionExec{}},
+		{name: "node busy", exec: executor(busy, nil, store)},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			q := &esmQueue{}
+			r, _, _ := newTestESM(t, q, nil)
+			r.invoke = awsapi.Deps{Meta: activeMeta{}, Exec: tc.exec}
+			err := r.process(context.Background(), esmMapping("u1"), func() (sqsiface.SQSAPI, error) { return q, nil }, "url",
+				[]*sqs.Message{msg("1", "a")})
+			require.Error(t, err)
+			deleted, _ := q.state()
+			require.Empty(t, deleted)
+		})
+	}
+	require.Empty(t, busy.sent, "the busy node is sent nothing")
+}
@@ -4,6 +4,7 @@ package awsgw

 import (
 	"context"
+	"crypto/cipher"
 	"crypto/rand"
 	"encoding/hex"
 	"encoding/json"
@@ -21,6 +22,7 @@ import (
 	stsdomain "github.com/mulgadc/spinifex/spinifex/domains/sts"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/lambdaenv"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/testlink"
 )

@@ -58,12 +60,13 @@ type testLinkExecutor struct {
 	// RequestLimit bounds the encoded request frame, without its credential.
 	RequestLimit int64
 	Sessions     lambdaEnvironmentRoleIssuer
+	AEAD         cipher.AEAD
 	Node         string
 	// ImageDigest is the runtime image every request requires of the daemon.
 	ImageDigest string
-	Codes  func(context.Context) (codePresigner, error)
-	Grants func(context.Context) (grantIssuer, error)
-	now    func() time.Time
+	Codes       func(context.Context) (codePresigner, error)
+	Grants      func(context.Context) (grantIssuer, error)
+	now         func() time.Time
 }

 var _ awsapi.Executor = (*testLinkExecutor)(nil)
@@ -78,11 +81,16 @@ func (e *testLinkExecutor) InvokeSync(ctx context.Context, in awsapi.SyncInvocat
 		return awsapi.SyncResult{}, &awsapi.RefusalError{Err: awserrors.Errorf(awserrors.ErrorServiceException,
 			"the test execution path has no valid runtime image pin")}
 	}
-	// The test link cannot deliver environment variables, so it refuses a
-	// function that has any before presigning, granting or sending.
+	var environment map[string]string
 	if rec.EnvCipher != "" {
-		return awsapi.SyncResult{}, &awsapi.RefusalError{Err: awserrors.Errorf(awserrors.ErrorServiceException,
-			"the test execution path does not support function environment variables")}
+		if e.AEAD == nil {
+			return awsapi.SyncResult{}, &awsapi.RefusalError{Err: awserrors.Errorf(awserrors.ErrorServiceException, "the function environment is unavailable")}
+		}
+		var err error
+		environment, err = function.OpenEnv(e.AEAD, rec.AccountID, rec.FunctionID, rec.EnvCipher)
+		if err != nil || lambdaenv.Validate(environment) != nil {
+			return awsapi.SyncResult{}, &awsapi.RefusalError{Err: awserrors.Errorf(awserrors.ErrorServiceException, "the function environment is unavailable")}
+		}
 	}
 	if rec.Code.SHA256 == "" || rec.Code.Size <= 0 || rec.Code.Size > testlink.MaxCodeBytes {
 		return awsapi.SyncResult{}, &awsapi.RefusalError{Err: fmt.Errorf("code copy of %d bytes is outside the test bound", rec.Code.Size)}
@@ -152,6 +160,7 @@ func (e *testLinkExecutor) InvokeSync(ctx context.Context, in awsapi.SyncInvocat
 		AccessKeyID:     aws.StringValue(session.Credentials.AccessKeyId),
 		SecretAccessKey: aws.StringValue(session.Credentials.SecretAccessKey),
 		SessionToken:    aws.StringValue(session.Credentials.SessionToken),
+		Environment:     environment,
 	}
 	revoke := func() error {
 		rctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), revokeTimeout)
@@ -4,6 +4,8 @@ package awsgw

 import (
 	"context"
+	"crypto/aes"
+	"crypto/cipher"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -165,6 +167,61 @@ func TestLambdaTestLinkExecutor_RefusesWithoutSending(t *testing.T) {
 	}
 }

+func TestLambdaTestLinkExecutor_OpensEnvironmentBeforeDispatch(t *testing.T) {
+	block, err := aes.NewCipher(make([]byte, 32))
+	require.NoError(t, err)
+	aead, err := cipher.NewGCM(block)
+	require.NoError(t, err)
+	vars := map[string]string{"GREETING": "private environment value"}
+	sealed, err := function.SealEnv(aead, "123456789012", "fn1", vars)
+	require.NoError(t, err)
+	input := invocation([]byte(`{}`))
+	input.Function.EnvCipher = sealed
+	n := &fakeControl{max: 1 << 20, reply: &testlink.Reply{Outcome: testlink.OutcomeDone}}
+	e := executor(n, nil, grantStore(t))
+	e.AEAD = aead
+	_, err = e.InvokeSync(t.Context(), input)
+	require.NoError(t, err)
+	require.Len(t, n.creds, 1)
+	require.Equal(t, vars, n.creds[0].Environment)
+	require.NotContains(t, string(n.sent[0]), vars["GREETING"])
+}
+
+func TestLambdaTestLinkExecutor_RefusesUnreadableEnvironment(t *testing.T) {
+	block, err := aes.NewCipher(make([]byte, 32))
+	require.NoError(t, err)
+	aead, err := cipher.NewGCM(block)
+	require.NoError(t, err)
+	wrongAccount, err := function.SealEnv(aead, "000000000001", "fn1", map[string]string{"GREETING": "private value"})
+	require.NoError(t, err)
+	reserved, err := function.SealEnv(aead, "123456789012", "fn1", map[string]string{"AWS_REGION": "private value"})
+	require.NoError(t, err)
+	for name, tc := range map[string]struct {
+		sealed string
+		aead   cipher.AEAD
+	}{
+		"key missing":       {wrongAccount, nil},
+		"wrong account":     {wrongAccount, aead},
+		"reserved variable": {reserved, aead},
+		"unreadable":        {"not ciphertext", aead},
+	} {
+		t.Run(name, func(t *testing.T) {
+			input := invocation(nil)
+			input.Function.EnvCipher = tc.sealed
+			n, store, sessions := &fakeControl{max: 1 << 20}, grantStore(t), &fakeSessions{}
+			e := executorWith(n, sessions, nil, store)
+			e.AEAD = tc.aead
+			_, err := e.InvokeSync(t.Context(), input)
+			var refusal *awsapi.RefusalError
+			require.ErrorAs(t, err, &refusal)
+			require.NotContains(t, err.Error(), "private value")
+			require.Empty(t, n.sent)
+			require.Empty(t, sessions.issued)
+			require.Empty(t, grantState(t, store))
+		})
+	}
+}
+
 // A gateway built without a valid image pin refuses before presigning,
 // granting, issuing or connecting.
 func TestLambdaTestLinkExecutor_InvalidImagePinRefusesBeforeConnecting(t *testing.T) {
@@ -3,6 +3,7 @@
 package awsgw

 import (
+	"crypto/cipher"
 	"github.com/mulgadc/spinifex/spinifex/domains/lambda/awsapi"
 	"github.com/mulgadc/spinifex/spinifex/domains/lambda/codestore"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
@@ -11,7 +12,7 @@ import (

 // Without the test link there is no executor, so Invoke refuses before it
 // presigns, sends or allocates anything, and Lambda stays unrouted.
-func lambdaTestExecutor(*nats.Conn, string, *codestore.Manager, lambdaEnvironmentRoleIssuer) awsapi.Executor {
+func lambdaTestExecutor(*nats.Conn, string, *codestore.Manager, lambdaEnvironmentRoleIssuer, cipher.AEAD) awsapi.Executor {
 	return nil
 }

@@ -51,7 +51,7 @@ func (e *spyExec) InvokeSync(context.Context, lambdaawsapi.SyncInvocation) (lamb
 // A default build composes the gateway with Lambda unrouted and no executor: a
 // signed Invoke is refused before any Lambda handler runs and starts no attempt.
 func TestDefaultBuild_InvokeNeverReachesLambda(t *testing.T) {
-	require.Nil(t, lambdaTestExecutor(nil, "node1", nil, nil), "a default build has no Lambda executor")
+	require.Nil(t, lambdaTestExecutor(nil, "node1", nil, nil, nil), "a default build has no Lambda executor")

 	meta, exec := &spyFunctions{}, &spyExec{}
 	lambda := lambdaawsapi.NewRegistration(lambdaawsapi.Deps{Meta: meta, Exec: exec})
@@ -5,6 +5,7 @@ package awsgw

 import (
 	"context"
+	"crypto/cipher"
 	"crypto/rand"
 	"encoding/hex"
 	"encoding/json"
@@ -17,6 +18,7 @@ import (

 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/mulgadc/bluebottle/pkg/masterkey"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
 	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
@@ -51,6 +53,7 @@ type seamEnv struct {
 	roleARN  string
 	grants   *execution.Store
 	sessions jetstream.KeyValue
+	aead     cipher.AEAD
 }

 func newSeamEnv(t *testing.T) *seamEnv {
@@ -58,6 +61,8 @@ func newSeamEnv(t *testing.T) *seamEnv {
 	_, nc, _ := testutil.StartTestJetStream(t)
 	masterKey, err := iamdomain.GenerateMasterKey()
 	require.NoError(t, err)
+	lambdaKey, err := masterkey.New(masterKey)
+	require.NoError(t, err)
 	iamSvc, err := iamdomain.NewIAMServiceImpl(t.Context(), nc, masterKey)
 	require.NoError(t, err)
 	stsSvc, err := stsdomain.NewSTSServiceImpl(t.Context(), nc, iamSvc, masterKey, seamNoJWKS{})
@@ -78,7 +83,7 @@ func newSeamEnv(t *testing.T) *seamEnv {
 	require.NoError(t, err)
 	sessions, err := js.KeyValue(t.Context(), stsdomain.KVBucketSessionCredentials)
 	require.NoError(t, err)
-	return &seamEnv{nc: nc, iam: iamSvc, sts: stsSvc, roleARN: aws.StringValue(role.Role.Arn), grants: grants, sessions: sessions}
+	return &seamEnv{nc: nc, iam: iamSvc, sts: stsSvc, roleARN: aws.StringValue(role.Role.Arn), grants: grants, sessions: sessions, aead: lambdaKey.AEAD}
 }

 // resolve authenticates cred through the real verifier and resolves its
@@ -29,6 +29,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/testlink"
+	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/require"
 )

@@ -212,7 +213,7 @@ func composedExecutor(t *testing.T, env *seamEnv, codeURL string, client *http.C
 	grants := &countedGrants{grantIssuer: env.grants}
 	presigns := &atomic.Int32{}
 	daemon := startSeamDaemon(t, &testlink.Daemon{Node: "node1", Grants: env.grants, HTTP: client}, guest)
-	exec := &testLinkExecutor{Control: daemon.client, RequestLimit: testlink.MaxFrameBytes, Sessions: env.sts, Node: "node1", ImageDigest: testImageDigest,
+	exec := &testLinkExecutor{Control: daemon.client, RequestLimit: testlink.MaxFrameBytes, Sessions: env.sts, AEAD: env.aead, Node: "node1", ImageDigest: testImageDigest,
 		Codes:  func(context.Context) (codePresigner, error) { return urlPresigner{codeURL, presigns}, nil },
 		Grants: func(context.Context) (grantIssuer, error) { return grants, nil },
 	}
@@ -375,16 +376,42 @@ func TestLambdaTestLinkSeam_RuntimeSeesInvokedARNAndHandler(t *testing.T) {
 	}
 }

-// A function with environment variables is refused before any code link is
-// presigned, any grant is issued or anything is sent to a node.
-func TestLambdaTestLinkSeam_EnvironmentVariablesRefusedBeforeDispatch(t *testing.T) {
+func TestLambdaTestLinkSeam_EnvironmentReachesGuestWithoutLeaking(t *testing.T) {
+	env := newSeamEnv(t)
+	var subscriptions []*nats.Subscription
+	for _, subject := range []string{">", "$JS.>", "$KV.>"} {
+		sub, err := env.nc.SubscribeSync(subject)
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = sub.Unsubscribe() })
+		subscriptions = append(subscriptions, sub)
+	}
+	require.NoError(t, env.nc.Flush())
+	vars := map[string]string{"GREETING": "private-environment-marker"}
+	sealed, err := function.SealEnv(env.aead, seamAccount, "fn1", vars)
+	require.NoError(t, err)
 	srv := codeServer(t)
-	run := invokeComposedAt(t, "/2015-03-31/functions/f/invocations", seamFunction{timeout: 10, envCipher: "sealed"},
-		srv.URL+"/code", srv.Client(), func(ctx context.Context, _ net.Conn) { <-ctx.Done() })
-	require.Equal(t, http.StatusInternalServerError, run.resp.StatusCode)
-	require.Equal(t, "ServiceException", run.resp.Header.Get("X-Amzn-Errortype"))
-	require.Contains(t, string(run.body), "does not support function environment variables")
-	require.Zero(t, run.presigns, "a code link was presigned")
-	require.Zero(t, run.grants, "a grant was issued")
-	require.Empty(t, run.requests, "the request reached a node")
+	creds := make(chan hostlink.Credential, 1)
+	run := invokeComposedIn(t, env, "/2015-03-31/functions/f/invocations", seamFunction{timeout: 10, envCipher: sealed},
+		srv.URL+"/code", srv.Client(), agentGuestWith(guestOpts{creds: creds}, func(base string) { respond(base, `{}`) }))
+	require.Equal(t, http.StatusOK, run.resp.StatusCode, string(run.body))
+	require.Equal(t, vars, recv(t, creds, "guest credential").Environment)
+	for _, request := range run.requests {
+		require.NotContains(t, string(request), vars["GREETING"])
+	}
+	for _, reply := range run.replies {
+		require.NotContains(t, string(reply), vars["GREETING"])
+	}
+	require.NoError(t, env.nc.Flush())
+	for _, sub := range subscriptions {
+		for {
+			msg, err := sub.NextMsg(50 * time.Millisecond)
+			if errors.Is(err, nats.ErrTimeout) {
+				break
+			}
+			require.NoError(t, err)
+			require.NotContains(t, msg.Subject, vars["GREETING"])
+			require.NotContains(t, string(msg.Data), vars["GREETING"])
+		}
+	}
+	require.NotContains(t, run.log, vars["GREETING"])
 }
Author	SHA1	Message	Date
joshsiv-mulga	663d8525e7	fix(lambda): validate environment names before reserved keys	2026-11-06 22:43:08 +10:00
joshsiv-mulga	b569d233ba	feat(lambda): deliver function environment to test-link guests	2026-11-06 22:36:31 +10:00
Josh Siviour	ac117a3a1e	feat(lambda): provide test-link SDK gateway settings (#1655 )	2026-11-06 21:43:50 +10:00
Josh Siviour	45dc82a85d	fix(eks): never terminate a control plane whose cluster record is missing (#1654 ) * fix(eks): alarm on a control-plane VM with no cluster record instead of terminating it * chore(eks): order test imports * feat(eks): add spx admin eks orphans and share the cluster-record judgement with the billable reaper * fix(eks): list orphan control planes through an EC2-owned inventory capability * test(eks): cover the orphan report ENI reader and the inventory fail-closed path * fix(eks): report only running control planes and treat a missing ENI as gone * fix(eks): open DNS watch buckets attach-only so a deleted account bucket is not recreated * fix(eks): read control-plane ENIs from their record so a corrupt one is unreadable, and report the zone	2026-11-06 21:37:27 +10:00
Josh Siviour	2a2097b4cf	feat(lambda): trigger test-link functions from SQS queues (#1652 ) * feat(lambda): record SQS event-source mappings behind Create, Get, List and Delete Create refuses every option this build does not implement, reads the queue as the function's execution role and requires the queue's visibility window to cover the function's run limit. Delete records Deleting for the poller to finish. Get and Delete authorize against the mapping ARN. * feat(lambda): poll mapped SQS queues in test-link gateways The poller receives and deletes as the function's execution role and invokes through the internal executor. A batch is deleted only after the function succeeds; an error, refusal or unknown outcome leaves the messages to reappear. * test(lambda): give the poller tests their in-package reason * fix: derive the mapping UUID so duplicate creates collide atomically Create no longer scans every mapping. The UUID is derived from the account, Region, function and queue, so the keyed create refuses a duplicate pair in one atomic write with no second record to orphan. * fix: mark the poller in-process SQS request as received over TLS The request never leaves the TLS-terminating gateway, so an execution role that requires aws:SecureTransport must still authorize it. * fix: renew the poller session per event and back off on failed batches Each event now runs on a session that outlives its invocation and its delete, so a later split event is never acknowledged on expired credentials. A failed, refused or unknown invocation makes the worker back off before the next receive. * feat: page ListEventSourceMappings with Marker and NextMarker * test: drive the poller through InvokeEventSource on invalid and busy outcomes * fix: skip locally held management addresses when allocating The shared record can lose an entry an instance on this node still holds. Allocate now treats every address in the local cache as taken, refuses cached or recorded hits that another local instance holds, and resets its results on each CAS attempt. * fix: refuse ListEventSourceMappings until listing can be bounded The mapping store can only be listed whole, so a list request would wait on the whole inventory. Refuse it, and skip repeated keys when the gateway lists mappings at startup.	2026-11-06 14:54:18 +10:00