test: stub QEMU for the test-link package so it runs on hosts without it

The test link now passes the stored handler to the guest as _HANDLER and the
validated qualifier into the invoked-function ARN. Functions with environment
variables are refused before any presign, grant or dispatch, since delivering
them needs a separate secret-transport design.

.gitignore

@@ -27,6 +27,7 @@ build/microvm/sqs-worker/.stage.*/
 build/microvm/lambda-runtime/vmlinuz
 build/microvm/lambda-runtime/initramfs.cpio.gz
 build/microvm/lambda-runtime/.stage.*/
+build/microvm/lambda-testlink-runtime/
 build/qboot/
 !spinifex/runtime/roles/spinifexui/frontend/dist/
 tmp/

Makefile

@@ -82,6 +82,35 @@ go_build:
 	@echo -e "\n....Building $(GO_PROJECT_NAME)"
 	go build $(GO_BUILD_MOD) -ldflags "$(LDFLAGS)" -o ./bin/$(GO_PROJECT_NAME) cmd/spinifex/main.go
 
+# Controlled-test only: builds the tagged daemon and its hard-kill initializer
+# together, pinning the initializer's digest into that daemon build. The daemon
+# keeps its symbols so the pin can be read back from the binary.
+# LAMBDA_IMAGE_DIGEST is the test-link runtime image digest that
+# build-lambda-testlink-image prints; it is read raw and validated before use.
+LAMBDA_TESTLINK_DIR := ./bin/lambda-testlink
+LAMBDA_TESTLINK_PKG := github.com/mulgadc/spinifex/spinifex/runtime/daemon
+LAMBDA_TESTLINK_PIN := $(LAMBDA_TESTLINK_PKG).lambdaHardKillSHA256
+LAMBDA_DIGEST_RE := ^[0-9a-f]{64}$$
+build-lambda-testlink: override export LAMBDA_IMAGE_DIGEST := $(value LAMBDA_IMAGE_DIGEST)
+build-lambda-testlink:
+	@[[ "$$LAMBDA_IMAGE_DIGEST" =~ $(LAMBDA_DIGEST_RE) ]] || { echo "LAMBDA_IMAGE_DIGEST must be 64 lowercase hex digits"; exit 1; }
+	@host=$$(uname -m); goos=$$(go env GOOS); goarch=$$(go env GOARCH); \
+	test "$$host" = x86_64 && test "$$goos/$$goarch" = linux/amd64 || { echo "the test link supports only an x86_64 host building linux/amd64, not $$host building $$goos/$$goarch"; exit 1; }
+	@rm -f $(LAMBDA_TESTLINK_DIR)/hardkill.so $(LAMBDA_TESTLINK_DIR)/$(GO_PROJECT_NAME)
+	@mkdir -p $(LAMBDA_TESTLINK_DIR).tmp
+	@rm -f $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME)
+	cc -std=c17 -O2 -Wall -Wextra -Werror -shared -fPIC -o $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so spinifex/runtime/compute/lambda/testlink/hardkill/hardkill.c -lrt
+	scripts/check-glibc-ceiling.sh 2.34 $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so
+	@digest=$$(sha256sum $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so | cut -d' ' -f1); \
+	test $${#digest} -eq 64 || { echo "initializer digest missing"; exit 1; }; \
+	echo "initializer sha256 $$digest"; \
+	pin="-X $(LAMBDA_TESTLINK_PIN)=$$digest -X $(LAMBDA_TESTLINK_PKG).lambdaImageDigest=$$LAMBDA_IMAGE_DIGEST"; \
+	LAMBDA_TESTLINK_INITIALIZER=$(abspath $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so) go test -count=1 -tags lambdatestlink -ldflags "$$pin" -run '^TestLambda(HardKillPin|ImageDigestPin)$$' ./spinifex/runtime/daemon/ && \
+	go build $(GO_BUILD_MOD) -tags lambdatestlink -ldflags "$(subst -s -w ,,$(LDFLAGS)) $$pin" -o $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME) cmd/spinifex/main.go
+	@mkdir -p $(LAMBDA_TESTLINK_DIR)
+	mv $(LAMBDA_TESTLINK_DIR).tmp/hardkill.so $(LAMBDA_TESTLINK_DIR).tmp/$(GO_PROJECT_NAME) $(LAMBDA_TESTLINK_DIR)/
+	@rmdir $(LAMBDA_TESTLINK_DIR).tmp
+
 build-installer:
 	@echo -e "\n....Building spinifex-installer"
 	go build -ldflags "-s -w" -o ./bin/spinifex-installer cmd/installer/main.go
@@ -197,6 +226,24 @@ bin/lambda-agent: go.mod go.sum $$(call go_local_srcs,./cmd/lambda-agent)
 	GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags '-s -w' -o $@ ./cmd/lambda-agent
 
 build-lambda-runtime-microvm-image: $(LAMBDA_RUNTIME_MICROVM_ARTIFACTS) ## Build the Lambda runtime microVM kernel and initramfs (not shipped)
+	GOWORK=off go run ./cmd/lambda-image-agent-tag $(LAMBDA_RUNTIME_MICROVM_OUT_DIR)/initramfs.cpio.gz absent
+
+# Controlled-test only: a separately built guest whose agent has the
+# lambdatestlink tag, in its own outputs, never touching bin/lambda-agent or the
+# default image. It prints the digest to pass as LAMBDA_IMAGE_DIGEST.
+LAMBDA_TESTLINK_IMAGE_DIR := $(MICROVM_OUT_DIR)/lambda-testlink-runtime
+build-lambda-testlink-image: ## Build the test-link Lambda guest image and print its digest (not shipped)
+	@host=$$(uname -m); goos=$$(go env GOOS); goarch=$$(go env GOARCH); \
+	test "$$host" = x86_64 && test "$$goos/$$goarch" = linux/amd64 || { echo "the test link supports only an x86_64 host building linux/amd64, not $$host building $$goos/$$goarch"; exit 1; }
+	$(MAKE) --no-print-directory $(MICROVM_OUT_DIR)/qboot.rom
+	@mkdir -p bin $(LAMBDA_TESTLINK_IMAGE_DIR)
+	GOWORK=off GOFLAGS= CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -tags lambdatestlink -ldflags '-s -w' -o bin/lambda-agent-testlink ./cmd/lambda-agent
+	./scripts/build-lambda-runtime-microvm-image.sh bin/lambda-agent-testlink $(LAMBDA_TESTLINK_IMAGE_DIR)
+	GOWORK=off go run ./cmd/lambda-image-agent-tag $(LAMBDA_TESTLINK_IMAGE_DIR)/initramfs.cpio.gz present
+	install -m 0644 $(MICROVM_OUT_DIR)/qboot.rom $(LAMBDA_TESTLINK_IMAGE_DIR)/qboot.rom
+	@digest=$$(GOWORK=off go run -tags lambdatestlink ./cmd/lambda-image-digest $(LAMBDA_TESTLINK_IMAGE_DIR)) && \
+	[[ "$$digest" =~ $(LAMBDA_DIGEST_RE) ]] || { echo "image digest not computed"; exit 1; }; \
+	echo "LAMBDA_IMAGE_DIGEST=$$digest"
 
 # qboot is the microvm firmware. The source tarball ships with each release
 # because qboot is GPL-2.0.
@@ -329,7 +376,10 @@ test-cover:
 test-race:
 	@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
 	$(_Q)LOG_IGNORE=1 SPINIFEX_SQS_R3=1 go test $(GOTESTFLAGS) -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... ./contracts/... ./architecture/... $(_RACEQ)
-	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s ./cmd/lambda-agent/ ./spinifex/agents/lambda/hostlink/ $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s ./cmd/lambda-agent/ ./spinifex/agents/lambda/hostlink/ ./spinifex/runtime/compute/lambda/testlink/ ./spinifex/domains/network/host/ $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s -run '^TestLambdaTestLink(Executor|Reply)_' ./spinifex/runtime/roles/awsgw/ $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink,lambdatestlinkseam -timeout 120s -run '^TestLambdaTestLinkSeam_' ./spinifex/runtime/roles/awsgw/ $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s -run '^TestLambdaTestLink' ./spinifex/runtime/daemon/ $(_RACEQ)
 
 # Unit tests for in-repo GitHub Actions (e.g. .github/actions/e2e-analyze).
 # Kept out of `test-cover` so coverage % isn't diluted by CI-only tooling.
@@ -540,7 +590,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: test-package-check build build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-worker-microvm-image build-lambda-runtime-microvm-image install-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
+.PHONY: test-package-check build build-lambda-testlink build-lambda-testlink-image build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-worker-microvm-image build-lambda-runtime-microvm-image install-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \

cmd/lambda-agent/connect.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 
+	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
 )
 
@@ -29,20 +30,20 @@ func defaultStartup() startup {
 
 // connect loads the identity, seals fw_cfg whether or not the load succeeded,
 // then dials the host. Under TLS 1.3 a host refusal is seen only on first read.
-func (s startup) connect(ctx context.Context) (*tls.Conn, int, error) {
+func (s startup) connect(ctx context.Context) (*tls.Conn, lambdav1.Scope, int, error) {
 	id, loadErr := hostlink.LoadIdentity(s.fwCfgDir)
 	if err := hostlink.SealFwCfg(s.fwCfgRoot, s.unload); err != nil {
-		return nil, exitRefused, errors.Join(loadErr, err)
+		return nil, "", exitRefused, errors.Join(loadErr, err)
 	}
 	if loadErr != nil {
-		return nil, exitRefused, loadErr
+		return nil, "", exitRefused, loadErr
 	}
 	conn, err := hostlink.Dial(ctx, id)
 	if err != nil {
 		if ctx.Err() != nil {
 			err = fmt.Errorf("%w: %w", err, context.Cause(ctx))
 		}
-		return nil, exitLink, err
+		return nil, "", exitLink, err
 	}
-	return conn, 0, nil
+	return conn, id.Scope, 0, nil
 }

cmd/lambda-agent/grant_testlink.go

@@ -0,0 +1,66 @@
+//go:build lambdatestlink
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
+)
+
+const entryGrant = "test-grant"
+
+// testGrant is the grant the host launched this VM under.
+type testGrant struct {
+	grantID, attemptID, environmentID string
+	generation                        uint64
+}
+
+// readGrant reads the test-grant fw_cfg entry: grant ID, attempt ID,
+// environment ID and generation, one per line.
+func readGrant(fwCfgDir string) (testGrant, error) {
+	b, err := os.ReadFile(filepath.Join(fwCfgDir, entryGrant, "raw"))
+	if err != nil {
+		return testGrant{}, fmt.Errorf("read %s: %w", entryGrant, err)
+	}
+	lines := strings.Split(strings.TrimSuffix(string(b), "\n"), "\n")
+	if len(lines) != 4 {
+		return testGrant{}, errors.New(entryGrant + " entry malformed")
+	}
+	g := testGrant{grantID: lines[0], attemptID: lines[1], environmentID: lines[2]}
+	g.generation, err = strconv.ParseUint(lines[3], 10, 64)
+	if err != nil || g.generation == 0 || g.grantID == "" || g.attemptID == "" || g.environmentID == "" {
+		return testGrant{}, errors.New(entryGrant + " entry malformed")
+	}
+	return g, nil
+}
+
+// checkBinding refuses a run unless the grant names the authenticated environment
+// generation, the request carries that grant and its deadline is still ahead. The
+// clock check is a local guard only: it cannot detect skew or a superseded grant.
+func checkBinding(scope lambdav1.Scope, g testGrant, req hostlink.Request, now time.Time) error {
+	parts := strings.Split(string(scope), "/")
+	if len(parts) != 5 {
+		return errors.New("authenticated scope malformed")
+	}
+	if parts[3] != g.environmentID || parts[4] != strconv.FormatUint(g.generation, 10) {
+		return errors.New("grant is not for the authenticated environment generation")
+	}
+	if req.GrantID != g.grantID || req.AttemptID != g.attemptID {
+		return errors.New("request does not carry this VM's grant")
+	}
+	if req.DeadlineMs <= 0 {
+		return errors.New("request has no deadline")
+	}
+	if !now.Before(time.UnixMilli(req.DeadlineMs)) {
+		return errors.New("request deadline already passed")
+	}
+	return nil
+}

cmd/lambda-agent/helpers_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-const scope lambdav1.Scope = "ap-southeast-2/000000000001/fn-1/env-1/gen-1"
+const scope lambdav1.Scope = "ap-southeast-2/000000000001/fn-1/env-1/1"
 
 // guest is a fake fw_cfg tree; unload removes it as unloading the module would.
 type guest struct {

cmd/lambda-agent/package_testlink.go

@@ -18,6 +18,7 @@ import (
 	"strconv"
 	"strings"
 	"time"
+	"unicode"
 
 	"golang.org/x/sys/unix"
 )
@@ -28,6 +29,7 @@ import (
 const (
 	entryPackageSize   = "test-package-size"
 	entryPackageSHA256 = "test-package-sha256"
+	entryHandler       = "test-handler"
 
 	devMaxZipBytes    = 16 << 20
 	devMaxUnzipBytes  = 64 << 20
@@ -37,8 +39,9 @@ const (
 )
 
 type packageSpec struct {
-	size   int64
-	sha256 [sha256.Size]byte
+	size    int64
+	sha256  [sha256.Size]byte
+	handler string
 }
 
 type unpackLimits struct {
@@ -76,6 +79,13 @@ func readPackageSpec(fwCfgDir string) (packageSpec, error) {
 		return spec, errors.New("package sha256 is not 64 hex digits")
 	}
 	copy(spec.sha256[:], b)
+	if spec.handler, err = read(entryHandler); err != nil {
+		return spec, err
+	}
+	if spec.handler == "" || len(spec.handler) > 128 || strings.IndexFunc(spec.handler, unicode.IsSpace) >= 0 ||
+		strings.ContainsRune(spec.handler, 0) {
+		return spec, errors.New("handler must be 1..128 characters with no whitespace")
+	}
 	return spec, nil
 }
 

cmd/lambda-agent/package_testlink_test.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strings"
 	"syscall"
 	"testing"
 
@@ -165,19 +166,23 @@ func TestUnpackChecksSizeAndHash(t *testing.T) {
 func TestReadPackageSpecBounds(t *testing.T) {
 	sum := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
 	cases := map[string]struct {
-		size, hash string
-		want       string
+		size, hash, handler string
+		want                string
 	}{
-		"valid":     {"10", sum, ""},
-		"zero":      {"0", sum, "outside"},
-		"too large": {"16777217", sum, "outside"},
-		"not hex":   {"10", "zz", "64 hex digits"},
-		"short":     {"10", sum[:62], "64 hex digits"},
+		"valid":            {"10", sum, "bootstrap.h", ""},
+		"zero":             {"0", sum, "bootstrap.h", "outside"},
+		"too large":        {"16777217", sum, "bootstrap.h", "outside"},
+		"not hex":          {"10", "zz", "bootstrap.h", "64 hex digits"},
+		"short":            {"10", sum[:62], "bootstrap.h", "64 hex digits"},
+		"empty handler":    {"10", sum, "", "handler"},
+		"handler space":    {"10", sum, "a b", "handler"},
+		"128-char handler": {"10", sum, strings.Repeat("h", 128), ""},
+		"129-char handler": {"10", sum, strings.Repeat("h", 129), "handler"},
 	}
 	for name, tc := range cases {
 		t.Run(name, func(t *testing.T) {
 			dir := t.TempDir()
-			for n, v := range map[string]string{entryPackageSize: tc.size, entryPackageSHA256: tc.hash} {
+			for n, v := range map[string]string{entryPackageSize: tc.size, entryPackageSHA256: tc.hash, entryHandler: tc.handler} {
 				require.NoError(t, os.MkdirAll(filepath.Join(dir, n), 0o700))
 				require.NoError(t, os.WriteFile(filepath.Join(dir, n, "raw"), []byte(v), 0o600))
 			}
@@ -185,6 +190,7 @@ func TestReadPackageSpecBounds(t *testing.T) {
 			if tc.want == "" {
 				require.NoError(t, err)
 				assert.Equal(t, int64(10), spec.size)
+				assert.Equal(t, tc.handler, spec.handler)
 				return
 			}
 			assert.ErrorContains(t, err, tc.want)

cmd/lambda-agent/start_tcp.go

@@ -25,7 +25,7 @@ func start(ctx context.Context) int {
 
 // run authenticates to the host and holds the connection until it closes.
 func (s startup) run(ctx context.Context) (int, error) {
-	conn, code, err := s.connect(ctx)
+	conn, _, code, err := s.connect(ctx)
 	if err != nil {
 		return code, err
 	}

cmd/lambda-agent/start_testlink.go

@@ -5,7 +5,6 @@ package main
 import (
 	"context"
 	"errors"
-	"flag"
 	"fmt"
 	"log/slog"
 	"net"
@@ -24,8 +23,6 @@ const (
 	taskRoot    = "/var/task"
 )
 
-var handler = flag.String("handler", os.Getenv("_HANDLER"), "function handler passed to the bootstrap")
-
 // testlink relays one invocation over the test carrier on the authenticated
 // connection; the carrier is not the guest protocol.
 type testlink struct {
@@ -54,13 +51,14 @@ func start(ctx context.Context) int {
 func (t testlink) run(parent context.Context) (int, error) {
 	// The package values are fw_cfg entries, so read them before connect seals it.
 	spec, specErr := readPackageSpec(t.fwCfgDir)
-	conn, code, err := t.connect(parent)
+	grant, grantErr := readGrant(t.fwCfgDir)
+	conn, scope, code, err := t.connect(parent)
 	if err != nil {
-		return code, errors.Join(err, specErr)
+		return code, errors.Join(err, specErr, grantErr)
 	}
 	defer conn.Close()
-	if specErr != nil {
-		return exitRefused, specErr
+	if err := errors.Join(specErr, grantErr); err != nil {
+		return exitRefused, err
 	}
 	link := hostlink.NewTestStream(conn)
 
@@ -76,6 +74,10 @@ func (t testlink) run(parent context.Context) (int, error) {
 	if parent.Err() != nil {
 		return exitLink, context.Cause(parent)
 	}
+	// Nothing of the tenant's is installed or run for an unbound request.
+	if err := checkBinding(scope, grant, first, time.Now()); err != nil {
+		return exitRefused, err
+	}
 	if err := t.pkg.install(spec); err != nil {
 		return exitRefused, fmt.Errorf("install package: %w", err)
 	}
@@ -93,7 +95,7 @@ func (t testlink) run(parent context.Context) (int, error) {
 	defer cancel(nil)
 	cmd, err := t.startBootstrap(ctx, []string{
 		"AWS_LAMBDA_RUNTIME_API=" + ln.Addr().String(),
-		"_HANDLER=" + *handler,
+		"_HANDLER=" + spec.handler,
 		"LAMBDA_TASK_ROOT=" + taskRoot,
 		"PATH=/bin:/sbin:/usr/bin",
 	})

cmd/lambda-agent/start_testlink_test.go

@@ -25,12 +25,18 @@ import (
 
 const helperEnv = "LAMBDA_AGENT_TEST_BOOTSTRAP"
 
+// helperHandlerOut names the file the helper bootstrap writes its _HANDLER to.
+const helperHandlerOut = "LAMBDA_AGENT_TEST_HANDLER_OUT"
+
 // TestHelperBootstrap is the fake function bootstrap, run as a subprocess: it
 // echoes one invocation through the Runtime API, then waits for the next.
 func TestHelperBootstrap(t *testing.T) {
 	if os.Getenv(helperEnv) != "1" {
 		return
 	}
+	if out := os.Getenv(helperHandlerOut); out != "" {
+		require.NoError(t, os.WriteFile(out, []byte(os.Getenv("_HANDLER")), 0o600))
+	}
 	base := "http://" + os.Getenv("AWS_LAMBDA_RUNTIME_API") + "/2018-06-01/runtime/invocation/"
 	resp, err := http.Get(base + "next")
 	require.NoError(t, err)
@@ -44,11 +50,19 @@ func TestHelperBootstrap(t *testing.T) {
 	_, _ = http.Get(base + "next")
 }
 
+// testGrantEntry grants g-1 for attempt a-1 on the fixture scope's env-1/1.
+const testGrantEntry = "g-1\na-1\nenv-1\n1\n"
+
+// testHandler is the handler the fixture's host configures.
+const testHandler = "bootstrap.fixture"
+
 type bootstrapStarts struct {
 	n       int
 	started chan struct{}
 	task    string
 	sealed  bool
+	// handlerOut receives the _HANDLER the bootstrap process itself saw.
+	handlerOut string
 }
 
 // testlink builds a guest whose fw_cfg also carries a controlled package,
@@ -61,9 +75,16 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 	sum := sha256.Sum256(zipBytes)
 	entries[entryPackageSize] = []byte(strconv.Itoa(len(zipBytes)))
 	entries[entryPackageSHA256] = []byte(hex.EncodeToString(sum[:]))
+	if _, ok := entries[entryHandler]; !ok {
+		entries[entryHandler] = []byte(testHandler)
+	}
+	if _, ok := entries[entryGrant]; !ok {
+		entries[entryGrant] = []byte(testGrantEntry)
+	}
 	g := newGuest(t, entries)
 	b.started = make(chan struct{}, 1)
 	b.task = t.TempDir()
+	b.handlerOut = filepath.Join(t.TempDir(), "handler")
 	return testlink{
 		startup: g.startup(),
 		pkg: taskPackage{
@@ -77,7 +98,7 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 			b.n++
 			b.started <- struct{}{}
 			cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestHelperBootstrap$")
-			cmd.Env = append(env, helperEnv+"=1")
+			cmd.Env = append(env, helperEnv+"=1", helperHandlerOut+"="+b.handlerOut)
 			return cmd, cmd.Start()
 		},
 	}
@@ -132,6 +153,9 @@ func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
 		assert.Error(t, err, "agent answered a second invocation")
 	}
 	assert.Equal(t, 1, b.n)
+	seen, err := os.ReadFile(b.handlerOut)
+	require.NoError(t, err, "bootstrap process did not report its handler")
+	assert.Equal(t, testHandler, string(seen), "bootstrap process not given the configured handler")
 	assert.True(t, b.sealed, "task dir not made read-only")
 	assert.FileExists(t, filepath.Join(b.task, "lib", "data.txt"))
 }
@@ -218,7 +242,8 @@ func TestTestlinkBadPackageNeverStartsBootstrap(t *testing.T) {
 
 	conn := h.guest(t)
 	defer conn.Close()
-	require.NoError(t, hostlink.NewTestStream(conn).HostSend(hostlink.Request{RequestID: "r-1"}))
+	require.NoError(t, hostlink.NewTestStream(conn).HostSend(hostlink.Request{GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1",
+		DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli()}))
 	res := wait(t, done, "agent to stop")
 	assert.Equal(t, exitRefused, res.code)
 	assert.ErrorContains(t, res.err, "install package")
@@ -244,3 +269,87 @@ func TestTestlinkMissingPackageSpecRefuses(t *testing.T) {
 	_, err := os.Stat(tl.fwCfgRoot)
 	assert.ErrorIs(t, err, os.ErrNotExist, "fw_cfg not sealed")
 }
+
+// A request that does not carry this VM's grant, or a grant for another
+// environment generation than the one authenticated, installs and runs
+// nothing of the tenant's.
+func TestTestlinkUnboundRequestNeverStartsBootstrap(t *testing.T) {
+	cases := map[string]struct {
+		grant            string
+		grantID, attempt string
+		deadlineMs       int64
+		want             string
+	}{
+		"frame with no deadline":      {testGrantEntry, "g-1", "a-1", -1, "no deadline"},
+		"frame with expired deadline": {testGrantEntry, "g-1", "a-1", time.Now().Add(-time.Second).UnixMilli(), "deadline already passed"},
+		"frame for another grant":     {testGrantEntry, "g-2", "a-1", 0, "does not carry this VM's grant"},
+		"frame for another attempt":   {testGrantEntry, "g-1", "a-2", 0, "does not carry this VM's grant"},
+		"frame with no grant":         {testGrantEntry, "", "", 0, "does not carry this VM's grant"},
+		"grant for another env":       {"g-1\na-1\nenv-2\n1\n", "g-1", "a-1", 0, "not for the authenticated environment"},
+		"grant for another gen":       {"g-1\na-1\nenv-1\n2\n", "g-1", "a-1", 0, "not for the authenticated environment"},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer cancel()
+			h := newHost(t)
+			h.accept(ctx)
+			var b bootstrapStarts
+			entries := h.entries()
+			entries[entryGrant] = []byte(tc.grant)
+			done := runTestlinkAsync(ctx, b.testlink(t, entries))
+			conn := h.guest(t)
+			defer conn.Close()
+			deadline := time.Now().Add(5 * time.Second).UnixMilli()
+			switch {
+			case tc.deadlineMs < 0:
+				deadline = 0
+			case tc.deadlineMs > 0:
+				deadline = tc.deadlineMs
+			}
+			req := hostlink.Request{GrantID: tc.grantID, AttemptID: tc.attempt, RequestID: "r-1",
+				DeadlineMs: deadline, Payload: []byte(`{}`)}
+			require.NoError(t, hostlink.NewTestStream(conn).HostSend(req))
+
+			res := wait(t, done, "agent to stop")
+			assert.Equal(t, exitRefused, res.code)
+			assert.ErrorContains(t, res.err, tc.want)
+			assert.Zero(t, b.n)
+			assert.False(t, b.sealed)
+			assert.NoFileExists(t, filepath.Join(b.task, "bootstrap"), "package installed")
+		})
+	}
+}
+
+// A missing or malformed grant entry refuses before any frame is awaited.
+func TestTestlinkBadGrantEntryRefuses(t *testing.T) {
+	cases := map[string]*string{
+		"missing":       nil,
+		"too few lines": ptr("g-1\na-1\nenv-1\n"),
+		"generation 0":  ptr("g-1\na-1\nenv-1\n0\n"),
+		"empty grant":   ptr("\na-1\nenv-1\n1\n"),
+	}
+	for name, grant := range cases {
+		t.Run(name, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer cancel()
+			h := newHost(t)
+			h.accept(ctx)
+			var b bootstrapStarts
+			entries := h.entries()
+			if grant != nil {
+				entries[entryGrant] = []byte(*grant)
+			}
+			tl := b.testlink(t, entries)
+			if grant == nil {
+				require.NoError(t, os.RemoveAll(filepath.Join(tl.fwCfgDir, entryGrant)))
+			}
+			res := wait(t, runTestlinkAsync(ctx, tl), "agent to stop")
+			assert.Equal(t, exitRefused, res.code)
+			assert.ErrorContains(t, res.err, entryGrant)
+			assert.Zero(t, b.n)
+		})
+	}
+}
+
+func ptr(s string) *string { return &s }

cmd/lambda-image-agent-tag/main.go

@@ -0,0 +1,221 @@
+// Command lambda-image-agent-tag checks whether the lambda-agent archived in a
+// built Lambda guest initramfs carries the lambdatestlink build tag. Build
+// information is recorded by the build; it is not proof of how the agent behaves.
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"compress/gzip"
+	"debug/buildinfo"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"slices"
+	"strconv"
+	"strings"
+)
+
+const (
+	agentPath    = "usr/local/bin/lambda-agent"
+	maxArchive   = 1 << 30
+	maxAgent     = 256 << 20
+	maxName      = 4096
+	newcHeader   = 110
+	trailerName  = "TRAILER!!!"
+	modeTypeMask = 0o170000
+	modeDir      = 0o040000
+	modeRegular  = 0o100000
+)
+
+var agentParents = []string{"usr", "usr/local", "usr/local/bin"}
+
+func main() {
+	if len(os.Args) != 3 || (os.Args[2] != "present" && os.Args[2] != "absent") {
+		fmt.Fprintln(os.Stderr, "usage: lambda-image-agent-tag <initramfs.cpio.gz> present|absent")
+		os.Exit(2)
+	}
+	if err := run(os.Args[1], os.Args[2] == "present"); err != nil {
+		fmt.Fprintf(os.Stderr, "%s: %v\n", os.Args[1], err)
+		os.Exit(1)
+	}
+}
+
+func run(path string, want bool) error {
+	fi, err := os.Lstat(path)
+	if err != nil {
+		return err
+	}
+	if !fi.Mode().IsRegular() {
+		return errors.New("not a regular file")
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	agent, err := archivedAgent(f)
+	if err != nil {
+		return err
+	}
+	return checkTag(agent, want)
+}
+
+// archivedAgent returns the single agent entry after checking listed names,
+// entry types, the agent's link count and zero-only padding after the trailer.
+// It is not a full kernel-equivalent parser; 070702 CRCs, for one, are unchecked.
+func archivedAgent(r io.Reader) ([]byte, error) {
+	zr, err := gzip.NewReader(bufio.NewReader(r))
+	if err != nil {
+		return nil, fmt.Errorf("not a readable gzip archive: %w", err)
+	}
+	raw, err := io.ReadAll(io.LimitReader(zr, maxArchive+1))
+	if err != nil {
+		return nil, fmt.Errorf("not a readable gzip archive: %w", err)
+	}
+	if len(raw) > maxArchive {
+		return nil, fmt.Errorf("archive larger than %d bytes", maxArchive)
+	}
+
+	seen := map[string]bool{}
+	var agent []byte
+	off := 0
+	for {
+		e, next, err := parseEntry(raw, off)
+		if err != nil {
+			return nil, fmt.Errorf("entry at offset %d: %w", off, err)
+		}
+		off = next
+		if e.name == trailerName {
+			break
+		}
+		if !canonical(e.name) {
+			return nil, fmt.Errorf("entry name %q is not canonical", e.name)
+		}
+		if seen[e.name] {
+			return nil, fmt.Errorf("entry %q appears more than once", e.name)
+		}
+		seen[e.name] = true
+		for _, p := range agentParents {
+			if e.name == p && e.mode&modeTypeMask != modeDir {
+				return nil, fmt.Errorf("%s is not a directory", p)
+			}
+		}
+		if e.name == agentPath {
+			for _, p := range agentParents {
+				if !seen[p] {
+					return nil, fmt.Errorf("%s has no %s directory entry before it", agentPath, p)
+				}
+			}
+			if e.mode&modeTypeMask != modeRegular {
+				return nil, fmt.Errorf("%s is not a regular file", agentPath)
+			}
+			if e.nlink != 1 {
+				return nil, fmt.Errorf("%s has %d links, want 1", agentPath, e.nlink)
+			}
+			if len(e.data) > maxAgent {
+				return nil, fmt.Errorf("%s is larger than %d bytes", agentPath, maxAgent)
+			}
+			agent = e.data
+		}
+	}
+	if i := bytes.IndexFunc(raw[off:], func(r rune) bool { return r != 0 }); i >= 0 {
+		return nil, fmt.Errorf("data after the archive trailer at offset %d", off+i)
+	}
+	if agent == nil {
+		return nil, fmt.Errorf("no %s entry", agentPath)
+	}
+	return agent, nil
+}
+
+type entry struct {
+	name  string
+	mode  uint64
+	nlink uint64
+	data  []byte
+}
+
+func parseEntry(raw []byte, off int) (entry, int, error) {
+	if len(raw)-off < newcHeader {
+		return entry{}, 0, errors.New("truncated header")
+	}
+	h := raw[off : off+newcHeader]
+	if m := string(h[:6]); m != "070701" && m != "070702" {
+		return entry{}, 0, fmt.Errorf("magic %q is not newc", m)
+	}
+	field := func(i int) (uint64, error) {
+		return strconv.ParseUint(string(h[6+8*i:14+8*i]), 16, 32)
+	}
+	mode, err1 := field(1)
+	nlink, err2 := field(4)
+	size, err3 := field(6)
+	nameSize, err4 := field(11)
+	if err := errors.Join(err1, err2, err3, err4); err != nil {
+		return entry{}, 0, fmt.Errorf("bad header field: %w", err)
+	}
+	if nameSize < 2 || nameSize > maxName {
+		return entry{}, 0, fmt.Errorf("name size %d out of range", nameSize)
+	}
+	nameEnd := off + newcHeader + int(nameSize)
+	dataOff := align4(nameEnd)
+	dataEnd := dataOff + int(size)
+	if dataEnd > len(raw) || align4(dataEnd) > len(raw) {
+		return entry{}, 0, errors.New("truncated entry")
+	}
+	name := raw[off+newcHeader : nameEnd]
+	if name[len(name)-1] != 0 || bytes.IndexByte(name[:len(name)-1], 0) >= 0 {
+		return entry{}, 0, errors.New("name is not one NUL-terminated string")
+	}
+	return entry{name: string(name[:len(name)-1]), mode: mode, nlink: nlink, data: raw[dataOff:dataEnd]}, align4(dataEnd), nil
+}
+
+func align4(n int) int { return (n + 3) &^ 3 }
+
+// canonical accepts "." and relative names with no empty, "." or ".." component.
+func canonical(name string) bool {
+	if name == "." {
+		return true
+	}
+	if name == "" || strings.HasPrefix(name, "/") {
+		return false
+	}
+	for c := range strings.SplitSeq(name, "/") {
+		if c == "" || c == "." || c == ".." {
+			return false
+		}
+	}
+	return true
+}
+
+func checkTag(agent []byte, want bool) error {
+	info, err := buildinfo.Read(bytes.NewReader(agent))
+	if err != nil {
+		return fmt.Errorf("no Go build information in %s: %w", agentPath, err)
+	}
+	return decide(info, want)
+}
+
+// decide refuses build information with no recorded settings, since their
+// absence does not show the tag was absent.
+func decide(info *buildinfo.BuildInfo, want bool) error {
+	if len(info.Settings) == 0 {
+		return fmt.Errorf("%s records no build settings", agentPath)
+	}
+	if hasTag(info, "lambdatestlink") != want {
+		return fmt.Errorf("%s: lambdatestlink build tag present=%v, want %v", agentPath, !want, want)
+	}
+	return nil
+}
+
+func hasTag(info *buildinfo.BuildInfo, tag string) bool {
+	for _, s := range info.Settings {
+		if s.Key != "-tags" {
+			continue
+		}
+		if slices.Contains(strings.Split(s.Value, ","), tag) {
+			return true
+		}
+	}
+	return false
+}

cmd/lambda-image-agent-tag/main_test.go

@@ -0,0 +1,154 @@
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"debug/buildinfo"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime/debug"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+type fileEntry struct {
+	name  string
+	mode  uint32
+	nlink uint32
+	data  string
+}
+
+func dir(name string) fileEntry { return fileEntry{name: name, mode: modeDir | 0o755, nlink: 2} }
+func file(name, data string) fileEntry {
+	return fileEntry{name: name, mode: modeRegular | 0o755, nlink: 1, data: data}
+}
+func symlink(name, target string) fileEntry {
+	return fileEntry{name: name, mode: 0o120777, nlink: 1, data: target}
+}
+
+// newc writes entries and a trailer in the format the kernel unpacks.
+func newc(entries ...fileEntry) []byte {
+	var b bytes.Buffer
+	for i, e := range append(entries, fileEntry{name: trailerName, nlink: 1}) {
+		fmt.Fprintf(&b, "070701%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X%08X",
+			i+1, e.mode, 0, 0, e.nlink, 0, len(e.data), 0, 0, 0, 0, len(e.name)+1, 0)
+		b.WriteString(e.name)
+		b.WriteByte(0)
+		pad(&b)
+		b.WriteString(e.data)
+		pad(&b)
+	}
+	return b.Bytes()
+}
+
+func pad(b *bytes.Buffer) {
+	for b.Len()%4 != 0 {
+		b.WriteByte(0)
+	}
+}
+
+func gz(parts ...[]byte) []byte {
+	var b bytes.Buffer
+	for _, p := range parts {
+		w := gzip.NewWriter(&b)
+		_, _ = w.Write(p)
+		_ = w.Close()
+	}
+	return b.Bytes()
+}
+
+func withAgent(agent fileEntry) []byte {
+	return newc(dir("."), dir("usr"), dir("usr/local"), dir("usr/local/bin"), agent, file("init", "#!/bin/sh"))
+}
+
+func TestArchivedAgent_ReturnsTheOneArchivedAgent(t *testing.T) {
+	got, err := archivedAgent(bytes.NewReader(gz(withAgent(file(agentPath, "agent-bytes")))))
+	require.NoError(t, err)
+	require.Equal(t, "agent-bytes", string(got))
+
+	padded := append(withAgent(file(agentPath, "agent-bytes")), make([]byte, 1024)...)
+	got, err = archivedAgent(bytes.NewReader(gz(padded)))
+	require.NoError(t, err, "zero padding after the trailer")
+	require.Equal(t, "agent-bytes", string(got))
+}
+
+func TestArchivedAgent_Refuses(t *testing.T) {
+	parents := []fileEntry{dir("."), dir("usr"), dir("usr/local"), dir("usr/local/bin")}
+	agent := file(agentPath, "untagged")
+	small := newc(append(parents, agent)...)
+	second := newc(append(parents[1:], file(agentPath, "tagged"))...)
+	// The second archive starts inside what would be the first one's final block.
+	sameBlock := append(append([]byte{}, small...), second...)
+	for len(sameBlock)%512 != 0 {
+		sameBlock = append(sameBlock, 0)
+	}
+
+	for name, archive := range map[string][]byte{
+		"aliased duplicate":            gz(newc(append(parents, agent, file("././"+agentPath, "tagged"))...)),
+		"leading slash alias":          gz(newc(append(parents, agent, file("/"+agentPath, "tagged"))...)),
+		"dot-dot component":            gz(newc(append(parents, agent, file("usr/local/../local/bin/lambda-agent", "x"))...)),
+		"empty component":              gz(newc(append(parents, agent, file("usr//local/bin/lambda-agent", "x"))...)),
+		"exact duplicate":              gz(newc(append(parents, agent, file(agentPath, "tagged"))...)),
+		"duplicate directory":          gz(newc(dir("usr"), dir("usr"), dir("usr/local"), dir("usr/local/bin"), agent)),
+		"missing directory entry":      gz(newc(dir("usr"), dir("usr/local/bin"), agent)),
+		"directory after the agent":    gz(newc(dir("usr"), dir("usr/local"), agent, dir("usr/local/bin"))),
+		"parent is a symlink":          gz(newc(dir("usr"), dir("usr/local"), symlink("usr/local/bin", "elsewhere"), agent)),
+		"agent is a symlink":           gz(newc(append(parents, symlink(agentPath, "/bin/sh"))...)),
+		"hard-linked agent":            gz(newc(append(parents, fileEntry{name: agentPath, mode: modeRegular | 0o755, nlink: 2, data: "x"})...)),
+		"no agent":                     gz(newc(parents...)),
+		"second archive appended":      gz(small, second),
+		"second archive in same block": gz(sameBlock),
+		"non-zero byte after trailer":  gz(append(append([]byte{}, small...), 0, 0, 1)),
+		"truncated archive":            gz(small[:len(small)-20]),
+		"not newc":                     gz([]byte("not cpio at all, but long enough to fill one newc header: " + string(make([]byte, 120)))),
+		"not gzip":                     small,
+	} {
+		t.Run(name, func(t *testing.T) {
+			_, err := archivedAgent(bytes.NewReader(archive))
+			require.Error(t, err)
+		})
+	}
+}
+
+func TestRun_RefusesASymlinkedImagePath(t *testing.T) {
+	d := t.TempDir()
+	img := filepath.Join(d, "initramfs.cpio.gz")
+	require.NoError(t, os.WriteFile(img, gz(withAgent(file(agentPath, "x"))), 0o644))
+	require.NoError(t, os.Symlink(img, filepath.Join(d, "link")))
+	require.ErrorContains(t, run(filepath.Join(d, "link"), false), "not a regular file")
+}
+
+func TestHasTag_MatchesWholeTagsOnly(t *testing.T) {
+	info := func(tags string) *buildinfo.BuildInfo {
+		return &buildinfo.BuildInfo{Settings: []debug.BuildSetting{{Key: "-buildmode", Value: "exe"}, {Key: "-tags", Value: tags}}}
+	}
+	require.True(t, hasTag(info("lambdatestlink,fips140v1.0"), "lambdatestlink"))
+	require.False(t, hasTag(info("fips140v1.0"), "lambdatestlink"))
+	require.False(t, hasTag(info("lambdatestlinkx,xlambdatestlink"), "lambdatestlink"))
+	require.False(t, hasTag(&buildinfo.BuildInfo{}, "lambdatestlink"))
+}
+
+// The test binary carries real build information, so checkTag reads it and
+// refuses the opposite answer; bytes with no build information are refused.
+func TestCheckTag_ReadsRealBuildInformation(t *testing.T) {
+	exe, err := os.Executable()
+	require.NoError(t, err)
+	raw, err := os.ReadFile(exe)
+	require.NoError(t, err)
+	info, err := buildinfo.Read(bytes.NewReader(raw))
+	require.NoError(t, err)
+	tagged := hasTag(info, "lambdatestlink")
+	require.NoError(t, checkTag(raw, tagged))
+	require.Error(t, checkTag(raw, !tagged))
+	require.Error(t, checkTag([]byte("not a Go binary"), false))
+}
+
+func TestDecide_RefusesMissingBuildSettings(t *testing.T) {
+	require.ErrorContains(t, decide(&buildinfo.BuildInfo{}, false), "no build settings")
+	require.ErrorContains(t, decide(&buildinfo.BuildInfo{}, true), "no build settings")
+	untagged := &buildinfo.BuildInfo{Settings: []debug.BuildSetting{{Key: "-buildmode", Value: "exe"}}}
+	require.NoError(t, decide(untagged, false))
+	require.Error(t, decide(untagged, true))
+}

cmd/lambda-image-digest/main.go

@@ -0,0 +1,26 @@
+//go:build lambdatestlink
+
+// Command lambda-image-digest prints the test-link runtime image digest of one
+// directory, selecting the boot files the same way the daemon does. The digest
+// identifies the bytes; it does not authenticate where they came from.
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/testlink"
+)
+
+func main() {
+	if len(os.Args) != 2 {
+		fmt.Fprintln(os.Stderr, "usage: lambda-image-digest <image-dir>")
+		os.Exit(2)
+	}
+	digest, err := testlink.DirDigest(os.Args[1])
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+	fmt.Println(digest)
+}

scripts/build-lambda-runtime-microvm-image.sh

@@ -3,24 +3,28 @@
 # + initramfs: a guest running the Lambda guest agent, with virtio-net for its
 # host control connection. It is not shipped.
 #
+# Usage: build-lambda-runtime-microvm-image.sh [agent-binary [output-dir]]
+#
 # Inputs:
 #   $REPO_ROOT/build/microvm/lambda-runtime/{init.sh,inittab,run-agent}
-#   $REPO_ROOT/bin/lambda-agent  (make bin/lambda-agent)
+#   agent binary, default $REPO_ROOT/bin/lambda-agent (make bin/lambda-agent)
 #
-# Outputs:
-#   $REPO_ROOT/build/microvm/lambda-runtime/vmlinuz
-#   $REPO_ROOT/build/microvm/lambda-runtime/initramfs.cpio.gz
+# Outputs, in output-dir (default $REPO_ROOT/build/microvm/lambda-runtime):
+#   vmlinuz
+#   initramfs.cpio.gz
 #
 # Rootfs strategy and module stripping: scripts/lib/microvm-image.sh.
 # shellcheck disable=SC2034 # variables are consumed by the sourced library
 set -euo pipefail
 
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
-BUILD_DIR="$REPO_ROOT/build/microvm/lambda-runtime"
-INIT_SH="$BUILD_DIR/init.sh"
-INITTAB="$BUILD_DIR/inittab"
-RUN_AGENT="$BUILD_DIR/run-agent"
-LAMBDA_AGENT_BIN="$REPO_ROOT/bin/lambda-agent"
+INPUT_DIR="$REPO_ROOT/build/microvm/lambda-runtime"
+INIT_SH="$INPUT_DIR/init.sh"
+INITTAB="$INPUT_DIR/inittab"
+RUN_AGENT="$INPUT_DIR/run-agent"
+LAMBDA_AGENT_BIN="${1:-$REPO_ROOT/bin/lambda-agent}"
+# The library stages beside BUILD_DIR, so it names the output, not the inputs.
+BUILD_DIR="${2:-$INPUT_DIR}"
 TAG="[build-lambda-runtime-microvm-image]"
 
 ALPINE_VERSION="${ALPINE_VERSION:-3.24}"
@@ -38,7 +42,8 @@ MICROVM_GATE_MODULES="qemu_fw_cfg virtio_net"
 source "$REPO_ROOT/scripts/lib/microvm-image.sh"
 
 echo "$TAG repo root: $REPO_ROOT"
-echo "$TAG build dir: $BUILD_DIR"
+echo "$TAG input dir: $INPUT_DIR"
+echo "$TAG output dir: $BUILD_DIR"
 
 microvm_check_tools
 
@@ -57,6 +62,9 @@ if ! file -b "$LAMBDA_AGENT_BIN" | grep -q "statically linked"; then
     exit 1
 fi
 
+# Absolute, because packing the initramfs runs from inside the chroot.
+mkdir -p "$BUILD_DIR"
+BUILD_DIR=$(cd "$BUILD_DIR" && pwd)
 microvm_init_dirs
 microvm_make_rootfs
 microvm_empty_dev

scripts/build-lambda-runtime-microvm-image_test.sh

@@ -0,0 +1,81 @@
+#!/bin/sh
+# build-lambda-runtime-microvm-image_test.sh — a build into a chosen output
+# directory stages and publishes there, and leaves the default input directory
+# untouched.
+#
+# The script runs in a fixture repo against the real library, with only the
+# rootfs, kernel and packing steps replaced so no image is downloaded.
+set -eu
+
+SCRIPTS_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT
+
+fails=0
+pass() { echo "  ok   - $1"; }
+fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
+
+echo "-- build-lambda-runtime-microvm-image.sh output directory"
+
+REPO="$TMP/repo"
+IN="$REPO/build/microvm/lambda-runtime"
+mkdir -p "$REPO/scripts/lib" "$IN" "$REPO/bin" "$TMP/bin"
+cp "$SCRIPTS_DIR/build-lambda-runtime-microvm-image.sh" "$REPO/scripts/"
+for f in init.sh inittab run-agent; do printf '%s\n' "$f" > "$IN/$f"; done
+printf 'default image\n' > "$IN/vmlinuz"
+printf '#!/bin/sh\n' > "$REPO/bin/agent-testlink"
+chmod +x "$REPO/bin/agent-testlink"
+printf '#!/bin/sh\necho "ELF 64-bit, statically linked"\n' > "$TMP/bin/file"
+chmod +x "$TMP/bin/file"
+BEFORE=$(cd "$IN" && ls -A | cksum)
+
+# The real library, with the steps that need a rootfs or kernel replaced.
+# Packing runs from inside the chroot as the real step does, and records where
+# the stage is while it still exists.
+cat > "$REPO/scripts/lib/microvm-image.sh" <<EOF
+. "$SCRIPTS_DIR/lib/microvm-image.sh"
+microvm_check_tools() { :; }
+microvm_make_rootfs() { mkdir -p "\$CHROOT_DIR/etc"; }
+microvm_empty_dev() { :; }
+microvm_locate_kernel() { :; }
+microvm_strip_modules() { :; }
+microvm_strip_rootfs() { :; }
+microvm_stage_kernel() { printf 'kernel\n' > "\$STAGE_DIR/vmlinuz"; }
+microvm_pack_initramfs() {
+    (cd "\$CHROOT_DIR" && printf 'initramfs\n' | gzip > "\$STAGE_DIR/initramfs.cpio.gz")
+    dirname "\$STAGE_DIR" > "$TMP/stage-parent"
+    ls -A "$IN" > "$TMP/input-during"
+}
+EOF
+
+# The output is given relative to the repo root, as make passes it.
+OUT="$REPO/build/microvm/lambda-testlink-runtime"
+if ! (cd "$REPO" && PATH="$TMP/bin:$PATH" bash scripts/build-lambda-runtime-microvm-image.sh bin/agent-testlink build/microvm/lambda-testlink-runtime) > "$TMP/log" 2>&1; then
+    fail "build into output dir: $(tr '\n' ' ' < "$TMP/log")"
+else
+    if [ "$(cat "$TMP/stage-parent")" = "$OUT" ]; then
+        pass "stage created in the output directory"
+    else
+        fail "stage created in $(cat "$TMP/stage-parent"), want $OUT"
+    fi
+    if grep -q '^\.stage\.' "$TMP/input-during"; then
+        fail "stage present in the default input directory during the build"
+    else
+        pass "no stage in the default input directory during the build"
+    fi
+    if [ "$(cd "$IN" && ls -A | cksum)" = "$BEFORE" ] && grep -qx 'default image' "$IN/vmlinuz"; then
+        pass "default input directory unchanged"
+    else
+        fail "default input directory changed: $(cd "$IN" && ls -A | tr '\n' ' ')"
+    fi
+    if grep -qx kernel "$OUT/vmlinuz" && gzip -t "$OUT/initramfs.cpio.gz" && [ "$(ls -A "$OUT" | wc -l)" -eq 2 ]; then
+        pass "output directory holds exactly the published pair"
+    else
+        fail "output directory: $(ls -A "$OUT" | tr '\n' ' ')"
+    fi
+fi
+
+if [ "$fails" -ne 0 ]; then
+    echo "$fails failure(s)"
+    exit 1
+fi

scripts/lib/lambda-testlink-build_test.sh

@@ -0,0 +1,111 @@
+#!/bin/sh
+# lambda-testlink-build_test.sh — the test-link make targets refuse a malformed
+# image digest and an unsupported host or GOARCH before touching any output.
+#
+# Every case is refused on the recipe's first lines, so nothing is compiled.
+set -eu
+
+LIB_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+REPO_ROOT=$(dirname -- "$(dirname -- "$LIB_DIR")")
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT
+
+fails=0
+pass() { echo "  ok   - $1"; }
+fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
+
+echo "-- make build-lambda-testlink / build-lambda-testlink-image guards"
+
+PWN="$TMP/pwn"
+OUT="$TMP/out"
+HEX=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
+mkdir -p "$OUT"
+printf 'old daemon\n' > "$OUT/spx"
+printf 'old initializer\n' > "$OUT/hardkill.so"
+PAIR=$(cat "$OUT/spx" "$OUT/hardkill.so" | cksum)
+
+# unchanged <name> — the existing pair is intact, no scratch was made and no
+# injected command ran.
+unchanged() {
+    if [ -e "$PWN" ]; then
+        fail "$1: injected command ran"
+        rm -f "$PWN"
+    elif [ -e "$OUT.tmp" ]; then
+        fail "$1: scratch directory created"
+    elif [ "$(cat "$OUT/spx" "$OUT/hardkill.so" | cksum)" != "$PAIR" ]; then
+        fail "$1: existing pair replaced"
+    else
+        pass "$1"
+    fi
+}
+
+# refused <name> <want> <log> <status> — make failed with the expected message.
+refused() {
+    if [ "$4" -eq 0 ]; then
+        fail "$1: make succeeded"
+    elif ! grep -q "$2" "$3"; then
+        fail "$1: wrong refusal: $(tr '\n' ' ' < "$3")"
+    else
+        unchanged "$1"
+    fi
+}
+
+testlink() {
+    make -s -C "$REPO_ROOT" --no-print-directory build-lambda-testlink LAMBDA_TESTLINK_DIR="$OUT" "$@" > "$TMP/log" 2>&1
+}
+
+DIGEST_MSG="LAMBDA_IMAGE_DIGEST must be 64 lowercase hex digits"
+NL='
+'
+for v in '' "\$(shell touch $PWN)" "\$\$(touch $PWN)" "\`touch $PWN\`" "$HEX; touch $PWN" \
+    "$HEX\" -X main.x=\"y" "$HEX -X main.x=y" "$HEX${NL}touch $PWN" \
+    0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF "${HEX%?}" "${HEX}0"; do
+    # GNU make itself expands a command-line $(shell ...) while parsing, for
+    # any target, so that one literal is tested only from the environment.
+    case "$v" in
+    "\$(shell"*) ;;
+    *)
+        status=0
+        testlink LAMBDA_IMAGE_DIGEST="$v" || status=$?
+        refused "command-line digest [$v]" "$DIGEST_MSG" "$TMP/log" "$status"
+        ;;
+    esac
+    status=0
+    LAMBDA_IMAGE_DIGEST="$v" testlink || status=$?
+    refused "environment digest [$v]" "$DIGEST_MSG" "$TMP/log" "$status"
+done
+
+status=0
+env -u LAMBDA_IMAGE_DIGEST make -s -C "$REPO_ROOT" --no-print-directory build-lambda-testlink LAMBDA_TESTLINK_DIR="$OUT" > "$TMP/log" 2>&1 || status=$?
+refused "digest unset" "$DIGEST_MSG" "$TMP/log" "$status"
+
+ARCH_MSG="supports only an x86_64 host building linux/amd64"
+SHIM="$TMP/shim"
+mkdir -p "$SHIM"
+printf '#!/bin/sh\necho aarch64\n' > "$SHIM/uname"
+chmod +x "$SHIM/uname"
+
+status=0
+GOARCH=arm64 testlink LAMBDA_IMAGE_DIGEST="$HEX" || status=$?
+refused "GOARCH=arm64" "$ARCH_MSG" "$TMP/log" "$status"
+status=0
+PATH="$SHIM:$PATH" testlink LAMBDA_IMAGE_DIGEST="$HEX" || status=$?
+refused "aarch64 host" "$ARCH_MSG" "$TMP/log" "$status"
+
+# The image target refuses before building qboot or any agent binary.
+for c in "GOARCH=arm64" "PATH=$SHIM:$PATH"; do
+    status=0
+    env "$c" make -s -C "$REPO_ROOT" --no-print-directory build-lambda-testlink-image > "$TMP/log" 2>&1 || status=$?
+    if [ "$status" -eq 0 ]; then
+        fail "image target [${c%%=*}]: make succeeded"
+    elif ! grep -q "$ARCH_MSG" "$TMP/log" || grep -q "qboot\|go build" "$TMP/log"; then
+        fail "image target [${c%%=*}]: did not refuse first: $(tr '\n' ' ' < "$TMP/log")"
+    else
+        pass "image target [${c%%=*}] refused first"
+    fi
+done
+
+if [ "$fails" -ne 0 ]; then
+    echo "$fails failure(s)"
+    exit 1
+fi

spinifex/agents/lambda/runtimeapi/server.go

@@ -212,7 +212,7 @@ func (s *Server) response(w http.ResponseWriter, r *http.Request) {
 	}
 	if len(body) > MaxResponseBytes {
 		msg := "Response payload size exceeded maximum allowed payload size (" + strconv.Itoa(MaxResponseBytes) + " bytes)."
-		s.complete(id, Result{Payload: ErrorBody(msg, ErrTypeResponseTooLarge), FunctionError: ErrTypeResponseTooLarge})
+		s.complete(id, Result{Payload: ErrorBody(msg, ErrTypeResponseTooLarge), FunctionError: ErrTypeUnhandled})
 		writeError(w, http.StatusRequestEntityTooLarge, errTypeTooLarge, msg)
 		return
 	}

spinifex/agents/lambda/runtimeapi/server_test.go

@@ -170,7 +170,7 @@ func TestOversizeResponse(t *testing.T) {
 
 	got := <-out
 	require.NoError(t, got.err)
-	assert.Equal(t, "Function.ResponseSizeTooLarge", got.res.FunctionError)
+	assert.Equal(t, "Unhandled", got.res.FunctionError)
 	assert.Equal(t, "Function.ResponseSizeTooLarge", errType(t, got.res.Payload))
 }
 

spinifex/domains/lambda/awsapi/create.go

@@ -110,7 +110,7 @@ func (d Deps) createFunction(ctx context.Context, c *call) (reply, error) {
 
 // rollback makes one compare-and-swap attempt to free the name. It cannot undo
 // a create that completed, because the revision it holds has moved on. The code
-// object, if one landed, is left in place and recorded as a cleanup obligation.
+// object, if one landed, stays in place under its owed upload record.
 func (d Deps) rollback(ctx context.Context, c *call, name string, createRev uint64, ref codestore.Ref, why string, cause error) {
 	slog.ErrorContext(ctx, "Lambda: create did not complete", "reason", why, "function_name", name,
 		"function_id", ref.FunctionID, "err", cause)

spinifex/domains/lambda/awsapi/get.go

@@ -40,6 +40,12 @@ func (d Deps) lookup(ctx context.Context, c *call, query string) (function.Recor
 		}
 		q = query
 	}
+	return d.lookupTarget(ctx, c, name, q)
+}
+
+// lookupTarget reads the record for an already resolved name and qualifier;
+// only $LATEST exists.
+func (d Deps) lookupTarget(ctx context.Context, c *call, name, q string) (function.Record, uint64, error) {
 	if q != "" && q != "$LATEST" {
 		return function.Record{}, 0, notFound(c.Region, c.AccountID, name+":"+q)
 	}

spinifex/domains/lambda/awsapi/handler_test.go

@@ -45,6 +45,7 @@ type localClient struct {
 	svc *function.MetaService
 
 	mu             sync.Mutex
+	forceActive    bool
 	failCodeStored bool
 	passOwnerClock bool
 	rollbacks      int
@@ -68,6 +69,11 @@ func (c *localClient) Get(ctx context.Context, account, region, name string) (fu
 	if !r.Found {
 		return function.Record{}, 0, function.ErrNotFound
 	}
+	c.mu.Lock()
+	if c.forceActive {
+		r.Record.State = function.StateActive
+	}
+	c.mu.Unlock()
 	return r.Record, r.Revision, nil
 }
 
@@ -104,11 +110,20 @@ type memStore struct {
 	objects map[codestore.Ref][]byte
 	putErr  error
 	urlErr  error
+	// beforePut runs once, unlocked, before the next write lands.
+	beforePut func()
 }
 
 var _ awsapi.CodeStore = (*memStore)(nil)
 
 func (m *memStore) Put(_ context.Context, ref codestore.Ref, code []byte) (string, error) {
+	m.mu.Lock()
+	hook := m.beforePut
+	m.beforePut = nil
+	m.mu.Unlock()
+	if hook != nil {
+		hook()
+	}
 	m.mu.Lock()
 	defer m.mu.Unlock()
 	if m.putErr != nil {
@@ -164,6 +179,40 @@ func newHarness(t *testing.T, statements ...iamdomain.Statement) (*gwtest.Harnes
 	return gwtest.Scoped(statements...), meta, store, kv, key
 }
 
+// uploads returns every upload record in the account bucket, by function ID.
+func (e *env) uploads(t *testing.T) map[string]function.Upload {
+	t.Helper()
+	kv := e.kv(t)
+	keys, err := kv.ListKeys(t.Context())
+	require.NoError(t, err)
+	out := map[string]function.Upload{}
+	for k := range keys.Keys() {
+		if !strings.HasPrefix(k, "upload.") {
+			continue
+		}
+		entry, err := kv.Get(t.Context(), k)
+		require.NoError(t, err)
+		var up function.Upload
+		require.NoError(t, json.Unmarshal(entry.Value(), &up))
+		require.Equal(t, function.KVUploadKey(up.Region, up.FunctionID, up.RevisionID), k)
+		out[up.FunctionID] = up
+	}
+	return out
+}
+
+// requireTracked asserts every code copy in the store has an upload record.
+func (e *env) requireTracked(t *testing.T) {
+	t.Helper()
+	ups := e.uploads(t)
+	e.store.mu.Lock()
+	defer e.store.mu.Unlock()
+	for ref := range e.store.objects {
+		up, ok := ups[ref.FunctionID]
+		require.True(t, ok, "code copy %s has no upload record", ref.FunctionID)
+		require.Equal(t, ref.RevisionID, up.RevisionID)
+	}
+}
+
 type env struct {
 	h     *gwtest.Harness
 	meta  *localClient
@@ -275,6 +324,15 @@ func TestCreateFunction_Success(t *testing.T) {
 	require.NoError(t, err)
 	assert.NotContains(t, string(entry.Value()), "hunter2", "the environment is never stored in plaintext")
 	assert.Contains(t, string(entry.Value()), "envCipher")
+
+	e.requireTracked(t)
+	ups := e.uploads(t)
+	require.Len(t, ups, 1)
+	for _, up := range ups {
+		assert.Equal(t, function.UploadAttached, up.State)
+		assert.Equal(t, entry.Revision(), up.AttachedRevision)
+		assert.Equal(t, "hello", up.FunctionName)
+	}
 }
 
 func TestCreateFunction_RendersTheStoredTimestamps(t *testing.T) {
@@ -307,6 +365,8 @@ func TestCreateFunction_DuplicateNameLeavesFirstUntouched(t *testing.T) {
 	assert.Contains(t, err.Error(), "Function already exist: hello")
 
 	assert.Equal(t, 1, e.store.count(), "the loser wrote no code")
+	assert.Len(t, e.uploads(t), 1, "the refused create leaves no upload record")
+	e.requireTracked(t)
 	after, _, err := e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
 	require.NoError(t, err)
 	assert.Equal(t, first, after)
@@ -321,6 +381,11 @@ func TestCreateFunction_CodePutFailureFreesTheName(t *testing.T) {
 	assert.Equal(t, 1, e.meta.rollbacks)
 	_, _, err = e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
 	require.ErrorIs(t, err, function.ErrNotFound)
+	ups := e.uploads(t)
+	require.Len(t, ups, 1)
+	for _, up := range ups {
+		assert.Equal(t, function.UploadOwed, up.State, "a failed write may still have landed")
+	}
 
 	e.store.putErr = nil
 	rec, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
@@ -337,6 +402,51 @@ func TestCreateFunction_CodeStoredFailureRollsBackOnce(t *testing.T) {
 	assert.Equal(t, 1, e.store.count(), "the code object is left in place, never deleted here")
 	_, _, err = e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
 	require.ErrorIs(t, err, function.ErrNotFound, "the still-storing claim was withdrawn")
+	e.requireTracked(t)
+	for _, up := range e.uploads(t) {
+		assert.Equal(t, function.UploadOwed, up.State, "the unnamed copy stays owed")
+	}
+}
+
+// The first create's write stalls past its claim, a second create takes the
+// name over, then the first write lands and its completion is refused.
+func TestCreateFunction_LateWriterAfterTakeoverStaysTracked(t *testing.T) {
+	e := newEnv(t)
+	key := function.KVFunctionKey(gwtest.Region, "hello")
+	e.store.beforePut = func() {
+		entry, err := e.kv(t).Get(t.Context(), key)
+		require.NoError(t, err)
+		var rec function.Record
+		require.NoError(t, json.Unmarshal(entry.Value(), &rec))
+		rec.ClaimExpiresAt = time.Now().Add(-time.Second)
+		data, err := json.Marshal(rec)
+		require.NoError(t, err)
+		_, err = e.kv(t).Update(t.Context(), key, data, entry.Revision())
+		require.NoError(t, err)
+
+		rec2, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) {
+			b["Code"] = map[string]any{"ZipFile": mustZip(t, "second")}
+		}))
+		require.NoError(t, err)
+		require.Equal(t, http.StatusCreated, rec2.Code)
+	}
+
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
+	assert.Equal(t, 2, e.store.count(), "both copies landed")
+	e.requireTracked(t)
+
+	winner, _, err := e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
+	require.NoError(t, err)
+	ups := e.uploads(t)
+	require.Len(t, ups, 2)
+	for id, up := range ups {
+		if id == winner.FunctionID {
+			assert.Equal(t, function.UploadAttached, up.State)
+			continue
+		}
+		assert.Equal(t, function.UploadOwed, up.State, "the late writer's copy is never inferred complete")
+	}
 }
 
 func TestCreateFunction_RefusesUnsupportedInput(t *testing.T) {
@@ -655,7 +765,7 @@ func TestAuthorization(t *testing.T) {
 
 func TestRegistration_ResolvesActionsAndInventory(t *testing.T) {
 	reg := awsapi.NewRegistration(awsapi.Deps{})
-	assert.Equal(t, []string{"CreateFunction", "GetFunction", "GetFunctionCodeSigningConfig", "GetFunctionConfiguration", "ListVersionsByFunction"}, reg.Inventory.Registered)
+	assert.Equal(t, []string{"CreateFunction", "GetFunction", "GetFunctionCodeSigningConfig", "GetFunctionConfiguration", "Invoke", "ListVersionsByFunction"}, reg.Inventory.Registered)
 	assert.Equal(t, "ListVersionsByFunction", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x/versions", nil)))
 	assert.Equal(t, "GetFunctionCodeSigningConfig", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2020-06-30/functions/x/code-signing-config", nil)))
 	assert.Equal(t, "CreateFunction", reg.ResolveAction(httptest.NewRequest(http.MethodPost, "/2015-03-31/functions", nil)))

spinifex/domains/lambda/awsapi/invoke.go

@@ -0,0 +1,186 @@
+package awsapi
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"log/slog"
+	"net/http"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// maxSyncPayload is the largest synchronous request payload AWS accepts.
+const maxSyncPayload = 6 * 1024 * 1024
+
+// waitOverhead is how much longer than the function timeout a request waits
+// for its result. It bounds the wait only, never the execution.
+const waitOverhead = 60 * time.Second
+
+// SyncInvocation is a RequestResponse request handed to the execution owner,
+// which decides admission.
+type SyncInvocation struct {
+	// Function and Revision are the one resolved configuration the attempt
+	// binds; the environment stays sealed in Function.EnvCipher.
+	Function function.Record
+	Revision uint64
+	// Qualifier is the validated qualifier the request named, or "".
+	Qualifier string
+	Payload   []byte
+	// WaitUntil is when the request stops waiting. Code may still be running
+	// after it; it proves nothing about stop and releases no slot.
+	WaitUntil time.Time
+}
+
+// SyncResult is what the function returned. FunctionError is "" on success;
+// ExecutedVersion is the version that ran.
+type SyncResult struct {
+	Payload         []byte
+	FunctionError   string
+	ExecutedVersion string
+}
+
+// Executor admits and runs one synchronous attempt. Its context ends only at
+// WaitUntil, never on caller disconnect. A *RefusalError means no code was
+// dispatched; any other error means the outcome is unknown.
+type Executor interface {
+	InvokeSync(ctx context.Context, inv SyncInvocation) (SyncResult, error)
+}
+
+// RefusalError is a known pre-dispatch refusal, such as a capacity throttle. Err
+// is the AWS error returned to the caller.
+type RefusalError struct{ Err error }
+
+func (r *RefusalError) Error() string { return "invocation refused before dispatch: " + r.Err.Error() }
+
+func (r *RefusalError) Unwrap() error { return r.Err }
+
+// invokeTarget is the function and qualifier an Invoke request names, once
+// every deterministic check on the request has passed.
+type invokeTarget struct{ Name, Qualifier string }
+
+// validateInvoke runs every check that needs no stored state: options, size,
+// payload JSON, function name and qualifier. Nothing here reads metadata.
+func validateInvoke(c *call) (*invokeTarget, error) {
+	if err := checkInvokeOptions(c.Header); err != nil {
+		return nil, err
+	}
+	if len(c.Body) > maxSyncPayload {
+		return nil, awserrors.Errorf(awserrors.ErrorRequestEntityTooLargeException,
+			"Request must be smaller than %d bytes for the InvokeFunction operation", maxSyncPayload)
+	}
+	// An empty body is dispatched as it is; anything else must be JSON.
+	if len(c.Body) > 0 && !json.Valid(c.Body) {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidRequestContentException,
+			"Could not parse request body into json")
+	}
+	if len(c.Params[0]) > maxFunctionNameLen {
+		return nil, invalid("1 validation error detected: Value at 'functionName' failed to satisfy constraint: Member must have length less than or equal to 256")
+	}
+	name, q, err := resolveFunctionName(c.Region, c.AccountID, c.Params[0])
+	if err != nil {
+		return nil, err
+	}
+	if q != "" && !nameQualifierRE.MatchString(q) {
+		return nil, invalidFunctionName()
+	}
+	if query, ok := c.Query["Qualifier"]; ok {
+		if len(query) != 1 || query[0] == "" {
+			return nil, invalid("1 validation error detected: Value at 'qualifier' failed to satisfy constraint: Member must have length greater than or equal to 1")
+		}
+		if len(query[0]) > maxQualifierLen {
+			return nil, invalid("1 validation error detected: Value at 'qualifier' failed to satisfy constraint: Member must have length less than or equal to 128")
+		}
+		if !queryQualifierRE.MatchString(query[0]) {
+			return nil, invalid("1 validation error detected: Value at 'qualifier' failed to satisfy constraint: Member must satisfy regular expression pattern: \\$(LATEST(\\.PUBLISHED)?)|[a-zA-Z0-9-_$]+")
+		}
+		if q != "" && q != query[0] {
+			return nil, invalid("The derived qualifier from the function name does not match the specified qualifier.")
+		}
+		q = query[0]
+	}
+	return &invokeTarget{Name: name, Qualifier: q}, nil
+}
+
+// invoke implements Invoke for RequestResponse on $LATEST. serve has already
+// validated the request into c.Invoke and authorized it.
+func (d Deps) invoke(ctx context.Context, c *call) (reply, error) {
+	if c.Invoke == nil {
+		return reply{}, serviceError("invoke request was not validated")
+	}
+	rec, revision, err := d.lookupTarget(ctx, c, c.Invoke.Name, c.Invoke.Qualifier)
+	if err != nil {
+		return reply{}, err
+	}
+	if rec.State != function.StateActive {
+		return reply{}, awserrors.Errorf(awserrors.ErrorResourceConflictException,
+			"The operation cannot be performed at this time. The function is currently in the following state: %s", rec.State)
+	}
+	if d.Exec == nil || rec.Code.State != function.CodeStored {
+		return reply{}, serviceError("function execution is unavailable")
+	}
+
+	if err := ctx.Err(); err != nil {
+		return reply{}, err
+	}
+	// Once accepted, a disconnecting caller does not cancel the attempt; only
+	// the wait is bounded, and a late result is discarded.
+	waitUntil := time.Now().Add(time.Duration(rec.Timeout)*time.Second + waitOverhead)
+	actx, cancel := context.WithDeadline(context.WithoutCancel(ctx), waitUntil)
+	defer cancel()
+	out, err := d.Exec.InvokeSync(actx, SyncInvocation{
+		Function:  rec,
+		Revision:  revision,
+		Qualifier: c.Invoke.Qualifier,
+		Payload:   c.Body,
+		WaitUntil: waitUntil,
+	})
+	if refusal, ok := errors.AsType[*RefusalError](err); ok {
+		if _, coded := awserrors.ResolveErrorCode(refusal.Err); coded {
+			return reply{}, refusal.Err
+		}
+		err = refusal
+	}
+	if err != nil {
+		slog.ErrorContext(ctx, "Lambda: synchronous attempt outcome unknown", "function_id", rec.FunctionID, "err", err)
+		return reply{}, serviceError("the invocation outcome is unknown")
+	}
+	if out.ExecutedVersion == "" || len(out.Payload) > maxSyncPayload {
+		slog.ErrorContext(ctx, "Lambda: executor returned an invalid result", "function_id", rec.FunctionID,
+			"payload_bytes", len(out.Payload))
+		return reply{}, serviceError("the invocation outcome is unknown")
+	}
+	hdr := http.Header{}
+	hdr.Set("X-Amz-Executed-Version", out.ExecutedVersion)
+	if out.FunctionError != "" {
+		hdr.Set("X-Amz-Function-Error", out.FunctionError)
+	}
+	return reply{Status: http.StatusOK, Body: rawJSON(out.Payload), Header: hdr}, nil
+}
+
+// checkInvokeOptions refuses every request option this build does not
+// implement rather than ignoring it.
+func checkInvokeOptions(h http.Header) error {
+	switch t := h.Get("X-Amz-Invocation-Type"); t {
+	case "", "RequestResponse":
+	case "Event", "DryRun":
+		return unsupported("InvocationType " + t)
+	default:
+		return invalid("InvocationType must be RequestResponse, Event or DryRun")
+	}
+	switch l := h.Get("X-Amz-Log-Type"); l {
+	case "", "None":
+	case "Tail":
+		return unsupported("LogType Tail")
+	default:
+		return invalid("LogType must be None or Tail")
+	}
+	for _, name := range []string{"X-Amz-Client-Context", "X-Amz-Tenant-Id", "X-Amz-Durable-Execution-Name"} {
+		if _, ok := h[name]; ok {
+			return unsupported(name)
+		}
+	}
+	return nil
+}

spinifex/domains/lambda/awsapi/invoke_test.go

@@ -0,0 +1,455 @@
+package awsapi_test
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"maps"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/internal/testkit/gwtest"
+	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type fakeExec struct {
+	mu     sync.Mutex
+	calls  []awsapi.SyncInvocation
+	ctxs   []context.Context
+	result awsapi.SyncResult
+	err    error
+	// started, when set, is closed once the call begins and the call then
+	// blocks until release is closed.
+	started, release chan struct{}
+}
+
+func (f *fakeExec) InvokeSync(ctx context.Context, inv awsapi.SyncInvocation) (awsapi.SyncResult, error) {
+	f.mu.Lock()
+	f.calls = append(f.calls, inv)
+	f.ctxs = append(f.ctxs, ctx)
+	started, release := f.started, f.release
+	f.mu.Unlock()
+	if started != nil {
+		close(started)
+		<-release
+	}
+	return f.result, f.err
+}
+
+func (f *fakeExec) count() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return len(f.calls)
+}
+
+var okResult = awsapi.SyncResult{Payload: []byte(`{"sum":42}`), ExecutedVersion: "$LATEST"}
+
+// newInvokeEnv creates "hello" with its code stored and reports it Active.
+func newInvokeEnv(t *testing.T, statements ...iamdomain.Statement) (*env, *fakeExec) {
+	t.Helper()
+	e := newEnv(t, statements...)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) {
+		b["Environment"] = map[string]any{"Variables": map[string]string{"MODE": "prod"}}
+	}))
+	require.NoError(t, err)
+	e.meta.forceActive = true
+	ex := &fakeExec{result: okResult}
+	e.deps.Exec = ex
+	return e, ex
+}
+
+func (e *env) invoke(ctx context.Context, path string, header http.Header, payload []byte) (*httptest.ResponseRecorder, error) {
+	rec := httptest.NewRecorder()
+	req := httptest.NewRequestWithContext(ctx, http.MethodPost, path, bytes.NewReader(payload))
+	maps.Copy(req.Header, header)
+	err := e.h.Dispatch(awsapi.NewRegistration(e.deps), awsapi.ServiceName, rec, req)
+	return rec, err
+}
+
+const invokePath = "/2015-03-31/functions/hello/invocations"
+
+func TestInvoke_ReturnsTheExecutorPayloadAndHeaders(t *testing.T) {
+	e, ex := newInvokeEnv(t)
+	payload := []byte(`{"a":40,"b":2,"note":"caf\u00e9"}`)
+	rec, err := e.invoke(t.Context(), invokePath, nil, payload)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, rec.Code)
+	assert.Equal(t, `{"sum":42}`, rec.Body.String())
+	assert.Equal(t, "$LATEST", rec.Header().Get("X-Amz-Executed-Version"))
+	assert.NotContains(t, rec.Header(), "X-Amz-Function-Error")
+	assert.NotContains(t, rec.Header(), "X-Amz-Log-Result")
+
+	require.Equal(t, 1, ex.count())
+	got := ex.calls[0]
+	assert.Equal(t, payload, got.Payload)
+	stored, revision, err := e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
+	require.NoError(t, err)
+	assert.Equal(t, revision, got.Revision)
+	assert.Equal(t, stored, got.Function)
+	assert.NotEmpty(t, got.Function.EnvCipher)
+	assert.NotContains(t, got.Function.EnvCipher, "prod")
+}
+
+func TestInvoke_FunctionErrorHeaderComesFromTheExecutor(t *testing.T) {
+	e, ex := newInvokeEnv(t)
+	ex.result = awsapi.SyncResult{Payload: []byte(`{"errorType":"x"}`), FunctionError: "Unhandled", ExecutedVersion: "$LATEST"}
+	rec, err := e.invoke(t.Context(), invokePath, nil, nil)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, rec.Code)
+	assert.Equal(t, "Unhandled", rec.Header().Get("X-Amz-Function-Error"))
+	assert.Equal(t, `{"errorType":"x"}`, rec.Body.String())
+}
+
+func TestInvoke_PendingFunctionIsAConflictAndNeverDispatches(t *testing.T) {
+	e, ex := newInvokeEnv(t)
+	e.meta.forceActive = false
+	_, err := e.invoke(t.Context(), invokePath, nil, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
+	assert.Equal(t, 0, ex.count())
+}
+
+func TestInvoke_FailsClosedWithoutAnExecutor(t *testing.T) {
+	e, _ := newInvokeEnv(t)
+	e.deps.Exec = nil
+	_, err := e.invoke(t.Context(), invokePath, nil, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
+}
+
+func TestInvoke_DoesNotNeedTheCodeStore(t *testing.T) {
+	e, _ := newInvokeEnv(t)
+	e.deps.Codes = func(context.Context) (awsapi.CodeStore, error) { return nil, errors.New("code store down") }
+	_, err := e.invoke(t.Context(), invokePath, nil, nil)
+	require.NoError(t, err)
+	e.deps.Codes = nil
+	_, err = e.invoke(t.Context(), invokePath, nil, nil)
+	require.NoError(t, err)
+}
+
+func TestInvoke_RefusalBeforeDispatchIsKeptApartFromAnUnknownOutcome(t *testing.T) {
+	e, ex := newInvokeEnv(t)
+
+	ex.err = &awsapi.RefusalError{Err: awserrors.Errorf(awserrors.ErrorThrottling, "Rate exceeded")}
+	_, err := e.invoke(t.Context(), invokePath, nil, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorThrottling)
+
+	ex.err = &awsapi.RefusalError{Err: errors.New("uncoded")}
+	_, err = e.invoke(t.Context(), invokePath, nil, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
+
+	ex.err = errors.New("transport lost")
+	_, err = e.invoke(t.Context(), invokePath, nil, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
+	assert.Contains(t, err.Error()+errMessage(err), "outcome is unknown")
+
+	ex.err = nil
+	ex.result = awsapi.SyncResult{Payload: []byte(`{}`)}
+	_, err = e.invoke(t.Context(), invokePath, nil, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
+}
+
+func errMessage(err error) string {
+	_, msg, _ := awserrors.ResolveErrorDetail(err)
+	return msg
+}
+
+func TestInvoke_RefusesOptionsItDoesNotImplement(t *testing.T) {
+	e, ex := newInvokeEnv(t)
+	for _, h := range []http.Header{
+		{"X-Amz-Invocation-Type": {"Event"}},
+		{"X-Amz-Invocation-Type": {"DryRun"}},
+		{"X-Amz-Invocation-Type": {"Bogus"}},
+		{"X-Amz-Log-Type": {"Tail"}},
+		{"X-Amz-Log-Type": {"Bogus"}},
+		{"X-Amz-Client-Context": {"e30="}},
+		{"X-Amz-Tenant-Id": {"t1"}},
+	} {
+		_, err := e.invoke(t.Context(), invokePath, h, nil)
+		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+	}
+	assert.Equal(t, 0, ex.count())
+
+	for _, h := range []http.Header{
+		{"X-Amz-Invocation-Type": {"RequestResponse"}},
+		{"X-Amz-Log-Type": {"None"}},
+	} {
+		_, err := e.invoke(t.Context(), invokePath, h, nil)
+		require.NoError(t, err)
+	}
+}
+
+// jsonOfSize is a JSON string exactly n bytes long.
+func jsonOfSize(n int) []byte {
+	return []byte(`"` + strings.Repeat("a", n-2) + `"`)
+}
+
+func TestInvoke_PayloadLimit(t *testing.T) {
+	e, ex := newInvokeEnv(t)
+	_, err := e.invoke(t.Context(), invokePath, nil, jsonOfSize(6*1024*1024))
+	require.NoError(t, err)
+	_, err = e.invoke(t.Context(), invokePath, nil, jsonOfSize(6*1024*1024+1))
+	testutil.RequireAWSCode(t, err, awserrors.ErrorRequestEntityTooLargeException)
+	assert.Equal(t, 1, ex.count())
+}
+
+func TestInvoke_PayloadMustBeJSONUnlessEmpty(t *testing.T) {
+	e, ex := newInvokeEnv(t)
+	for _, bad := range []string{"not json", `{"a":`, "{} {}", "\x00"} {
+		_, err := e.invoke(t.Context(), invokePath, nil, []byte(bad))
+		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidRequestContentException)
+	}
+	assert.Equal(t, 0, ex.count(), "a malformed payload reached the executor")
+
+	for _, good := range []string{"", `{"a":1}`, `[1,2]`, `"s"`, `42`, `null`, " true "} {
+		_, err := e.invoke(t.Context(), invokePath, nil, []byte(good))
+		require.NoError(t, err, "payload %q", good)
+	}
+	require.Equal(t, 7, ex.count())
+	assert.Empty(t, ex.calls[0].Payload, "an empty body is dispatched as it is")
+	assert.Equal(t, []byte(" true "), ex.calls[6].Payload)
+}
+
+func TestInvoke_MissingAndUnsupportedFunctions(t *testing.T) {
+	e, ex := newInvokeEnv(t)
+	_, err := e.invoke(t.Context(), "/2015-03-31/functions/ghost/invocations", nil, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+	_, err = e.invoke(t.Context(), invokePath+"?Qualifier=1", nil, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+	_, err = e.invoke(t.Context(), invokePath+"?Qualifier=%24LATEST", nil, nil)
+	require.NoError(t, err)
+	assert.Equal(t, 1, ex.count())
+}
+
+// Each form follows its own documented Invoke pattern; a documented value this
+// prototype does not serve passes validation and fails closed at lookup.
+func TestInvoke_QualifierFormsAndLengths(t *testing.T) {
+	fn := "/2015-03-31/functions/"
+	long := func(n int) string { return strings.Repeat("a", n) }
+	cases := []struct {
+		name, path, code string
+	}{
+		{"path $LATEST.PUBLISHED", fn + "hello:$LATEST.PUBLISHED/invocations", awserrors.ErrorResourceNotFoundException},
+		{"query $LATEST.PUBLISHED", invokePath + "?Qualifier=%24LATEST.PUBLISHED", awserrors.ErrorResourceNotFoundException},
+		{"query $foo", invokePath + "?Qualifier=%24foo", awserrors.ErrorResourceNotFoundException},
+		{"query of 128 characters", invokePath + "?Qualifier=" + long(128), awserrors.ErrorResourceNotFoundException},
+		{"query of 129 characters", invokePath + "?Qualifier=" + long(129), awserrors.ErrorInvalidParameterValueException},
+		{"empty query", invokePath + "?Qualifier=", awserrors.ErrorInvalidParameterValueException},
+		{"query with a dot", invokePath + "?Qualifier=a.b", awserrors.ErrorInvalidParameterValueException},
+		{"query $LATEST.OTHER", invokePath + "?Qualifier=%24LATEST.OTHER", awserrors.ErrorInvalidParameterValueException},
+		{"path $foo", fn + "hello:$foo/invocations", awserrors.ErrorInvalidParameterValueException},
+		{"path with a dot", fn + "hello:a.b/invocations", awserrors.ErrorInvalidParameterValueException},
+		{"function name of 256 characters", fn + "hello:" + long(250) + "/invocations", awserrors.ErrorResourceNotFoundException},
+		{"function name of 257 characters", fn + "hello:" + long(251) + "/invocations", awserrors.ErrorInvalidParameterValueException},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			e, ex := newInvokeEnv(t)
+			_, err := e.invoke(t.Context(), tc.path, nil, nil)
+			testutil.RequireAWSCode(t, err, tc.code)
+			assert.Equal(t, 0, ex.count())
+		})
+	}
+}
+
+func TestInvoke_CancelledBeforeDispatchNeverReachesTheExecutor(t *testing.T) {
+	e, ex := newInvokeEnv(t)
+	ctx, cancel := context.WithCancel(t.Context())
+	cancel()
+	_, err := e.invoke(ctx, invokePath, nil, nil)
+	require.Error(t, err)
+	assert.Equal(t, 0, ex.count())
+}
+
+func TestInvoke_CallerDisconnectDoesNotCancelAnAcceptedAttempt(t *testing.T) {
+	e, ex := newInvokeEnv(t)
+	ex.started, ex.release = make(chan struct{}), make(chan struct{})
+	ctx, cancel := context.WithCancel(t.Context())
+	done := make(chan error, 1)
+	go func() {
+		_, err := e.invoke(ctx, invokePath, nil, nil)
+		done <- err
+	}()
+	<-ex.started
+	cancel()
+
+	actx := ex.ctxs[0]
+	assert.NoError(t, actx.Err(), "the caller's disconnect reached the attempt")
+	deadline, ok := actx.Deadline()
+	require.True(t, ok, "the wait must be bounded")
+	assert.Equal(t, ex.calls[0].WaitUntil, deadline)
+	assert.WithinDuration(t, time.Now().Add(3*time.Second+60*time.Second), deadline, 5*time.Second)
+
+	close(ex.release)
+	select {
+	case <-done:
+	case <-time.After(10 * time.Second):
+		t.Fatal("the handler did not return after the attempt finished")
+	}
+}
+
+func TestInvoke_Authorization(t *testing.T) {
+	arn := "arn:aws:lambda:" + gwtest.Region + ":" + gwtest.AccountID + ":function:hello"
+	run := func(t *testing.T, statements ...iamdomain.Statement) (*fakeExec, error) {
+		return runPath(t, invokePath, statements...)
+	}
+	t.Run("lambda:GetFunction alone is not enough", func(t *testing.T) {
+		ex, err := run(t, gwtest.Statement("Allow", "lambda:GetFunction", "*"))
+		gwtest.AssertDenied(t, err)
+		assert.Equal(t, 0, ex.count())
+	})
+	t.Run("a deny on the function ARN applies", func(t *testing.T) {
+		ex, err := run(t,
+			gwtest.Statement("Allow", "lambda:*", "*"),
+			gwtest.Statement("Deny", "lambda:InvokeFunction", arn))
+		gwtest.AssertDenied(t, err)
+		assert.Equal(t, 0, ex.count())
+	})
+	t.Run("an allow on another function ARN does not apply", func(t *testing.T) {
+		_, err := run(t, gwtest.Statement("Allow", "lambda:InvokeFunction", arn+"-other"))
+		gwtest.AssertDenied(t, err)
+	})
+	t.Run("an allow of lambda:InvokeFunction on the function ARN passes", func(t *testing.T) {
+		ex, err := run(t, gwtest.Statement("Allow", "lambda:InvokeFunction", arn))
+		require.NoError(t, err)
+		assert.Equal(t, 1, ex.count())
+	})
+	t.Run("an unqualified call does not match a qualified-only allow", func(t *testing.T) {
+		ex, err := run(t, gwtest.Statement("Allow", "lambda:InvokeFunction", arn+":$LATEST"))
+		gwtest.AssertDenied(t, err)
+		assert.Equal(t, 0, ex.count())
+	})
+	t.Run("an unqualified call does not match an allow on any qualified ARN", func(t *testing.T) {
+		_, err := run(t, gwtest.Statement("Allow", "lambda:InvokeFunction", arn+":*"))
+		gwtest.AssertDenied(t, err)
+	})
+	t.Run("an unqualified call matches a wildcard allow", func(t *testing.T) {
+		ex, err := run(t, gwtest.Statement("Allow", "lambda:InvokeFunction", arn+"*"))
+		require.NoError(t, err)
+		assert.Equal(t, 1, ex.count())
+	})
+
+	// Lambda compares the policy resource with both the name's qualifier and
+	// the Qualifier parameter, so both forms are checked the same way.
+	qualified := map[string]string{
+		"path qualifier":  "/2015-03-31/functions/hello:$LATEST/invocations",
+		"query qualifier": invokePath + "?Qualifier=%24LATEST",
+	}
+	for form, path := range qualified {
+		t.Run(form, func(t *testing.T) {
+			t.Run("a qualified allow passes", func(t *testing.T) {
+				ex, err := runPath(t, path, gwtest.Statement("Allow", "lambda:InvokeFunction", arn+":$LATEST"))
+				require.NoError(t, err)
+				assert.Equal(t, 1, ex.count())
+			})
+			t.Run("an explicit qualified deny overrides the matching qualified allow", func(t *testing.T) {
+				ex, err := runPath(t, path,
+					gwtest.Statement("Allow", "lambda:InvokeFunction", arn+":$LATEST"),
+					gwtest.Statement("Deny", "lambda:InvokeFunction", arn+":$LATEST"))
+				gwtest.AssertDenied(t, err)
+				assert.Equal(t, 0, ex.count())
+			})
+			t.Run("an unqualified-only allow does not match", func(t *testing.T) {
+				ex, err := runPath(t, path, gwtest.Statement("Allow", "lambda:InvokeFunction", arn))
+				gwtest.AssertDenied(t, err)
+				assert.Equal(t, 0, ex.count())
+			})
+			t.Run("an allow on any qualified ARN passes", func(t *testing.T) {
+				ex, err := runPath(t, path, gwtest.Statement("Allow", "lambda:InvokeFunction", arn+":*"))
+				require.NoError(t, err)
+				assert.Equal(t, 1, ex.count())
+			})
+			t.Run("a wildcard allow passes", func(t *testing.T) {
+				ex, err := runPath(t, path, gwtest.Statement("Allow", "lambda:InvokeFunction", arn+"*"))
+				require.NoError(t, err)
+				assert.Equal(t, 1, ex.count())
+			})
+		})
+	}
+}
+
+func runPath(t *testing.T, path string, statements ...iamdomain.Statement) (*fakeExec, error) {
+	t.Helper()
+	e := newEnv(t, statements...)
+	sum := sha256.Sum256(testZip)
+	rev, err := e.meta.Create(t.Context(), gwtest.AccountID, gwtest.Region, function.Record{
+		FunctionID: "f1", FunctionName: "hello", Role: roleARN, Runtime: "provided.al2023", Handler: "h",
+		Timeout: 3, MemorySize: 128, Code: function.Code{RevisionID: "r1"},
+	})
+	require.NoError(t, err)
+	_, _, err = e.meta.CodeStored(t.Context(), gwtest.AccountID, gwtest.Region, "hello", rev, hex.EncodeToString(sum[:]), int64(len(testZip)))
+	require.NoError(t, err)
+	e.meta.forceActive = true
+	ex := &fakeExec{result: okResult}
+	e.deps.Exec = ex
+	_, err = e.invoke(t.Context(), path, nil, nil)
+	return ex, err
+}
+
+// countingMeta counts metadata reads, to show a request never got that far.
+type countingMeta struct {
+	function.Client
+
+	gets atomic.Int32
+}
+
+func (m *countingMeta) Get(ctx context.Context, account, region, name string) (function.Record, uint64, error) {
+	m.gets.Add(1)
+	return m.Client.Get(ctx, account, region, name)
+}
+
+// Every deterministic check runs before authorization, so a caller with no
+// Invoke access still gets the validation error, and neither the metadata
+// store nor the executor is reached. Whether AWS orders these the same way
+// is not verified.
+func TestInvoke_ValidationPrecedesAuthorization(t *testing.T) {
+	big := append([]byte(`"`), bytes.Repeat([]byte("a"), 6*1024*1024)...)
+	cases := []struct {
+		name    string
+		path    string
+		header  http.Header
+		payload []byte
+		code    string
+	}{
+		{"unsupported invocation type", invokePath, http.Header{"X-Amz-Invocation-Type": {"Event"}}, nil, awserrors.ErrorInvalidParameterValueException},
+		{"invalid invocation type", invokePath, http.Header{"X-Amz-Invocation-Type": {"Later"}}, nil, awserrors.ErrorInvalidParameterValueException},
+		{"payload too large", invokePath, nil, append(big, '"'), awserrors.ErrorRequestEntityTooLargeException},
+		{"payload not JSON", invokePath, nil, []byte(`{`), awserrors.ErrorInvalidRequestContentException},
+		{"malformed function name", "/2015-03-31/functions/bad%20name/invocations", nil, nil, awserrors.ErrorInvalidParameterValueException},
+		{"malformed path qualifier", "/2015-03-31/functions/hello:a.b/invocations", nil, nil, awserrors.ErrorInvalidParameterValueException},
+		{"malformed query qualifier", invokePath + "?Qualifier=a.b", nil, nil, awserrors.ErrorInvalidParameterValueException},
+		{"empty query qualifier", invokePath + "?Qualifier=", nil, nil, awserrors.ErrorInvalidParameterValueException},
+		{"conflicting qualifiers", "/2015-03-31/functions/hello:$LATEST/invocations?Qualifier=2", nil, nil, awserrors.ErrorInvalidParameterValueException},
+		{"malformed query encoding", invokePath + "?Qualifier=%ZZ", nil, nil, awserrors.ErrorInvalidParameterValueException},
+		{"malformed pair beside a valid qualifier", invokePath + "?Qualifier=%24LATEST&x=%ZZ", nil, nil, awserrors.ErrorInvalidParameterValueException},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			for access, statement := range map[string]iamdomain.Statement{
+				"denied":  gwtest.Statement("Allow", "lambda:GetFunction", "*"),
+				"allowed": gwtest.Statement("Allow", "lambda:InvokeFunction", "*"),
+			} {
+				e := newEnv(t, statement)
+				meta := &countingMeta{Client: e.meta}
+				e.deps.Meta = meta
+				ex := &fakeExec{result: okResult}
+				e.deps.Exec = ex
+				_, err := e.invoke(t.Context(), tc.path, tc.header, tc.payload)
+				testutil.RequireAWSCode(t, err, tc.code)
+				assert.Zero(t, meta.gets.Load(), "%s: metadata was read", access)
+				assert.Equal(t, 0, ex.count(), "%s: the executor ran", access)
+			}
+		})
+	}
+}

spinifex/domains/lambda/awsapi/registration.go

@@ -8,7 +8,9 @@ import (
 	"errors"
 	"io"
 	"log/slog"
+	"maps"
 	"net/http"
+	"net/url"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -50,6 +52,8 @@ type Deps struct {
 	Trusts func(accountID, roleARN string) (bool, error)
 	// AEAD seals function environment variables; it is the master key's cipher.
 	AEAD cipher.AEAD
+	// Exec runs synchronous attempts; nil refuses every Invoke.
+	Exec Executor
 }
 
 // NewRegistration returns the Lambda REST-JSON registration.
@@ -89,6 +93,17 @@ func functionARN(region, accountID, name string) string {
 	return arn.FormatLambdaFunction(region, accountID, name)
 }
 
+// qualifiedFunctionARN names the version or alias a validated request selects,
+// since Lambda matches a qualified request only against a qualified policy
+// resource.
+func qualifiedFunctionARN(region, accountID, name, qualifier string) string {
+	base := functionARN(region, accountID, name)
+	if base == "*" || qualifier == "" {
+		return base
+	}
+	return base + ":" + qualifier
+}
+
 // serve resolves the route, authorizes the request, then runs the action.
 func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 	r := inv.Request
@@ -104,7 +119,7 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 	if err != nil {
 		return err
 	}
-	c := &call{AccountID: inv.AccountID, Region: inv.Region, Params: params, Query: r.URL.Query(), Body: body}
+	c := &call{AccountID: inv.AccountID, Region: inv.Region, Params: params, Query: r.URL.Query(), Body: body, Header: r.Header}
 
 	// The region is the gateway's own, never the caller-supplied credential
 	// scope, so a caller cannot sign for another region to slide out from
@@ -121,8 +136,19 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 		if name, _, err := resolveFunctionName(inv.Region, inv.AccountID, params[0]); err == nil {
 			resource = functionARN(inv.Region, inv.AccountID, name)
 		}
+	case "Invoke":
+		// Invoke validates deterministically before authorizing, and its
+		// resource comes from that same validated target. URL.Query drops
+		// malformed pairs, so the raw query is parsed strictly here.
+		if c.Query, err = url.ParseQuery(r.URL.RawQuery); err != nil {
+			return invalid("The request query string could not be parsed.")
+		}
+		if c.Invoke, err = validateInvoke(c); err != nil {
+			return err
+		}
+		resource = qualifiedFunctionARN(inv.Region, inv.AccountID, c.Invoke.Name, c.Invoke.Qualifier)
 	}
-	if err := inv.Authorize(ServiceName, action, []string{resource}, nil); err != nil {
+	if err := inv.Authorize(ServiceName, iamAction(action), []string{resource}, nil); err != nil {
 		return err
 	}
 	if decodeErr != nil {
@@ -133,9 +159,9 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 			return err
 		}
 	}
-	// Only GetFunction presigns code; the other reads are metadata only.
+	// Only CreateFunction and GetFunction touch the code store.
 	needsCodes := action != "GetFunctionConfiguration" && action != "GetFunctionCodeSigningConfig" &&
-		action != "ListVersionsByFunction"
+		action != "ListVersionsByFunction" && action != "Invoke"
 	if d.Meta == nil || (d.Codes == nil && needsCodes) {
 		return errors.New(awserrors.ErrorServiceException)
 	}
@@ -148,6 +174,14 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 	return nil
 }
 
+// iamAction is the IAM action an API operation is authorized as.
+func iamAction(action string) string {
+	if action == "Invoke" {
+		return "InvokeFunction"
+	}
+	return action
+}
+
 func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 	return d.serve(w, inv)
 }
@@ -197,6 +231,7 @@ func writeJSON(w http.ResponseWriter, out reply) {
 		_, _ = w.Write(envelope.JSONBody(awserrors.ErrorServiceException, "failed to marshal response"))
 		return
 	}
+	maps.Copy(w.Header(), out.Header)
 	w.Header().Set("Content-Type", "application/json")
 	w.WriteHeader(out.Status)
 	if _, err := w.Write(body); err != nil {

spinifex/domains/lambda/awsapi/routes.go

@@ -2,6 +2,7 @@ package awsapi
 
 import (
 	"context"
+	"net/http"
 	"net/url"
 
 	"github.com/aws/aws-sdk-go/service/lambda"
@@ -15,9 +16,13 @@ type call struct {
 	Params    []string
 	Query     url.Values
 	Body      []byte
+	Header    http.Header
 	// Create is the decoded CreateFunction body, decoded once ahead of the
 	// authorization gate and the PassRole check that both read it.
 	Create *lambda.CreateFunctionInput
+	// Invoke is Invoke's validated target, resolved ahead of the
+	// authorization gate whose resource it names.
+	Invoke *invokeTarget
 }
 
 // rawJSON is a response body written as is, for a shape the SDK encoder
@@ -28,6 +33,7 @@ type rawJSON []byte
 type reply struct {
 	Status int
 	Body   any
+	Header http.Header
 }
 
 type route = rest.Route[routeHandler]
@@ -56,6 +62,10 @@ var routes = []route{
 		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
 			return d.getFunctionCodeSigningConfig(ctx, c)
 		}},
+	{Method: "POST", Pattern: "/2015-03-31/functions/{FunctionName}/invocations", Action: "Invoke",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.invoke(ctx, c)
+		}},
 }
 
 var router = rest.NewRouter("lambda", routes)

spinifex/domains/lambda/awsapi/validate.go

@@ -30,6 +30,20 @@ const (
 
 var nameRE = regexp.MustCompile(`^[a-zA-Z0-9-_]{1,64}$`)
 
+// nameQualifierRE is the qualifier suffix of Invoke's documented FunctionName
+// pattern; queryQualifierRE is its Qualifier pattern, bounded by maxQualifierLen.
+var (
+	nameQualifierRE  = regexp.MustCompile(`^(\$LATEST(\.PUBLISHED)?|[a-zA-Z0-9-_]+)$`)
+	queryQualifierRE = regexp.MustCompile(`^(\$LATEST(\.PUBLISHED)?|[a-zA-Z0-9-_$]+)$`)
+)
+
+// maxQualifierLen bounds the Qualifier query; maxFunctionNameLen bounds the
+// whole FunctionName, qualifier suffix included.
+const (
+	maxQualifierLen    = 128
+	maxFunctionNameLen = 256
+)
+
 func invalid(format string, args ...any) error {
 	return awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, format, args...)
 }

spinifex/domains/lambda/execution/grant.go

@@ -0,0 +1,396 @@
+// Package execution owns Lambda attempt grants: who may run an attempt and
+// report its result. This is a provisional single-node owner; it is not the
+// ADR-0009 M1 mechanism, which is still open.
+package execution
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"regexp"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// KVBucket holds one record per node: its grant sequence and current attempt.
+const KVBucket = "lambda-execution"
+
+const formatVersion = 1
+
+// Attempt states. The attempt lives in its node's record, and every change is
+// a conditional write on that one key, so a start, a withdrawal and the next
+// grant can never all succeed.
+const (
+	// StateGranted: recorded; nothing may have started.
+	StateGranted = "granted"
+	// StateRunning: the daemon may have launched the attempt.
+	StateRunning = "running"
+	// StateAccepted: a result was accepted under the grant; stop not yet proven.
+	StateAccepted = "accepted"
+	// StateWithdrawn: the attempt never started, and now never can.
+	StateWithdrawn = "withdrawn"
+	// StateReleased: the VM stop is proven and its resources released, but the
+	// daemon's own record of it is not yet gone; the node stays held.
+	StateReleased = "released"
+	// StateEnded: the stop is proven and nothing of the attempt remains.
+	StateEnded = "ended"
+	// StateAmbiguous: the end is unproven; the attempt keeps the node's slot.
+	StateAmbiguous = "ambiguous"
+)
+
+var (
+	// ErrNodeBusy: the node holds an attempt that has not ended.
+	ErrNodeBusy = errors.New("execution: node holds an unended attempt")
+	// ErrNotCurrent: the grant is not the node's current grant.
+	ErrNotCurrent = errors.New("execution: grant is not current")
+	// ErrWrongState: the attempt is not in a state that allows the change.
+	ErrWrongState = errors.New("execution: attempt is in the wrong state")
+	// ErrExpired: the grant's launch deadline has passed.
+	ErrExpired = errors.New("execution: grant expired before launch")
+	// ErrUnknownRecord: a record this owner cannot read or did not write.
+	ErrUnknownRecord = errors.New("execution: unrecognised record")
+	// ErrMismatch: the request or code differs from what the grant authorises.
+	ErrMismatch = errors.New("execution: request does not match the grant")
+	// ErrOutcomeUnknown: a write's outcome is unknown and may have landed.
+	ErrOutcomeUnknown = errors.New("execution: write outcome unknown")
+)
+
+var segment = regexp.MustCompile(`^[A-Za-z0-9-]{1,64}$`)
+
+// Grant is the identity that authorises one attempt on one environment
+// generation. Seq orders grants on a node and never changes for the attempt.
+type Grant struct {
+	GrantID       string `json:"grantId"`
+	Seq           uint64 `json:"seq"`
+	AttemptID     string `json:"attemptId"`
+	EnvironmentID string `json:"environmentId"`
+	Generation    uint64 `json:"generation"`
+	Node          string `json:"node"`
+}
+
+// Identity is the exact request and code a grant authorises.
+type Identity struct {
+	RequestID  string `json:"requestId"`
+	AccountID  string `json:"accountId"`
+	Region     string `json:"region"`
+	FunctionID string `json:"functionId"`
+	RevisionID string `json:"revisionId"`
+	CodeSHA256 string `json:"codeSha256"`
+}
+
+// Attempt is the owner's record of one attempt and its grant.
+type Attempt struct {
+	Grant     Grant     `json:"grant"`
+	Identity  Identity  `json:"identity"`
+	State     string    `json:"state"`
+	LaunchBy  time.Time `json:"launchBy"`
+	CreatedAt time.Time `json:"createdAt"`
+	Launch    *Launch   `json:"launch,omitempty"`
+}
+
+// Launch identifies an attempt's VM process, its launcher and what it holds,
+// so a restarted daemon can prove the stop. The pidfs inodes are comparable
+// only within BootID on MachineID; they do not identify the host.
+type Launch struct {
+	MachineID     string `json:"machineId"`
+	BootID        string `json:"bootId"`
+	PIDNamespace  uint64 `json:"pidNamespace"`
+	LauncherPID   int    `json:"launcherPid"`
+	LauncherInode uint64 `json:"launcherInode"`
+	PID           int    `json:"pid"`
+	Inode         uint64 `json:"inode"`
+	TapName       string `json:"tapName"`
+	TapIndex      int    `json:"tapIndex"`
+	TapOwner      string `json:"tapOwner"`
+}
+
+func (l Launch) valid() bool {
+	return l.MachineID != "" && l.BootID != "" && l.PIDNamespace != 0 && l.LauncherPID > 0 &&
+		l.LauncherInode != 0 && l.PID > 0 && l.Inode != 0
+}
+
+// nodeRecord is a node's one slot. Seq survives every attempt so each later
+// grant orders after each earlier one.
+type nodeRecord struct {
+	FormatVersion int      `json:"formatVersion"`
+	Seq           uint64   `json:"seq"`
+	Attempt       *Attempt `json:"attempt,omitempty"`
+}
+
+// Store is the only writer of node records.
+type Store struct {
+	kv  jetstream.KeyValue
+	now func() time.Time
+	// beforeWrite runs between a read and its conditional write; tests use it.
+	beforeWrite func()
+}
+
+// Open opens (or creates) the execution bucket.
+func Open(ctx context.Context, js jetstream.KeyValueManager) (*Store, error) {
+	kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucket, 1)
+	if err != nil {
+		return nil, fmt.Errorf("execution: open bucket: %w", err)
+	}
+	return &Store{kv: kv, now: time.Now}, nil
+}
+
+func nodeKey(node string) string { return "node." + node }
+
+// Issue records a new attempt and its grant before anything is dispatched.
+// It succeeds only if the node's current attempt ended, was withdrawn, or
+// provably never started before its launch deadline.
+func (s *Store) Issue(ctx context.Context, a Attempt) (Attempt, error) {
+	g := a.Grant
+	for _, seg := range []string{g.AttemptID, g.EnvironmentID, g.Node, a.Identity.FunctionID} {
+		if !segment.MatchString(seg) {
+			return Attempt{}, fmt.Errorf("execution: invalid identifier %q", seg)
+		}
+	}
+	id := a.Identity
+	if id.RequestID == "" || id.AccountID == "" || id.Region == "" || id.RevisionID == "" || id.CodeSHA256 == "" {
+		return Attempt{}, errors.New("execution: request and code identity are required")
+	}
+	if g.Generation == 0 {
+		return Attempt{}, errors.New("execution: environment generation is required")
+	}
+	grantID, err := randomID()
+	if err != nil {
+		return Attempt{}, err
+	}
+	a.Grant.GrantID, a.State, a.CreatedAt = grantID, StateGranted, s.now()
+
+	rec, rev, err := s.read(ctx, g.Node)
+	missing := errors.Is(err, jetstream.ErrKeyNotFound)
+	if err != nil && !missing {
+		return Attempt{}, err
+	}
+	if !missing && rec.Attempt != nil {
+		if rec.Attempt.Grant.AttemptID == g.AttemptID {
+			return Attempt{}, fmt.Errorf("%w: attempt %s already granted", ErrWrongState, g.AttemptID)
+		}
+		if !s.supersedable(*rec.Attempt) {
+			return Attempt{}, ErrNodeBusy
+		}
+	}
+	a.Grant.Seq = rec.Seq + 1
+	next := nodeRecord{FormatVersion: formatVersion, Seq: a.Grant.Seq, Attempt: &a}
+	if missing {
+		err = s.create(ctx, g.Node, next)
+	} else {
+		err = s.write(ctx, g.Node, next, rev)
+	}
+	if errors.Is(err, ErrNotCurrent) {
+		return Attempt{}, ErrNodeBusy
+	}
+	if err != nil {
+		return Attempt{}, err
+	}
+	return a, nil
+}
+
+// supersedable reports whether a new grant may replace cur. A granted attempt
+// past its deadline qualifies because Start refuses it then, and both writes
+// are conditional on the same record revision.
+func (s *Store) supersedable(cur Attempt) bool {
+	switch cur.State {
+	case StateEnded, StateWithdrawn:
+		return true
+	case StateGranted:
+		return !s.now().Before(cur.LaunchBy)
+	}
+	return false
+}
+
+// Start moves a current, unexpired grant from granted to running, only for
+// the exact request and code it authorises. The daemon calls it before
+// allocating anything; a repeat, replay or mismatch is refused unchanged.
+func (s *Store) Start(ctx context.Context, g Grant, want Identity) error {
+	return s.transition(ctx, g, func(a *Attempt) error {
+		if a.Identity != want {
+			return ErrMismatch
+		}
+		if a.State != StateGranted {
+			return fmt.Errorf("%w: %s", ErrWrongState, a.State)
+		}
+		if !s.now().Before(a.LaunchBy) {
+			return ErrExpired
+		}
+		a.State = StateRunning
+		return nil
+	})
+}
+
+// Accept records that a result under g is accepted. Only one result is ever
+// accepted, and none once the attempt has ended or been held as ambiguous.
+func (s *Store) Accept(ctx context.Context, g Grant) error {
+	return s.transition(ctx, g, func(a *Attempt) error {
+		if a.State != StateRunning {
+			return fmt.Errorf("%w: %s", ErrWrongState, a.State)
+		}
+		a.State = StateAccepted
+		return nil
+	})
+}
+
+// Launched records the running attempt's launch identity, once, before the
+// daemon sends the guest anything.
+func (s *Store) Launched(ctx context.Context, g Grant, l Launch) error {
+	if !l.valid() {
+		return errors.New("execution: incomplete launch identity")
+	}
+	return s.transition(ctx, g, func(a *Attempt) error {
+		if a.State != StateRunning || a.Launch != nil {
+			return fmt.Errorf("%w: %s", ErrWrongState, a.State)
+		}
+		a.Launch = &l
+		return nil
+	})
+}
+
+// Withdraw ends a grant that never started, for a caller that knows it sent
+// nothing. It fails once the daemon has started the attempt.
+func (s *Store) Withdraw(ctx context.Context, g Grant) error {
+	return s.transition(ctx, g, func(a *Attempt) error {
+		if a.State != StateGranted {
+			return fmt.Errorf("%w: %s", ErrWrongState, a.State)
+		}
+		a.State = StateWithdrawn
+		return nil
+	})
+}
+
+// Released records the caller's proof that the attempt's VM has stopped and
+// everything it held is released. The node stays held until Stopped.
+func (s *Store) Released(ctx context.Context, g Grant) error {
+	return s.transition(ctx, g, func(a *Attempt) error {
+		switch a.State {
+		case StateRunning, StateAccepted, StateAmbiguous:
+			a.State = StateReleased
+			return nil
+		}
+		return fmt.Errorf("%w: %s", ErrWrongState, a.State)
+	})
+}
+
+// Stopped records that nothing of a released attempt remains, which frees the
+// node for the next grant.
+func (s *Store) Stopped(ctx context.Context, g Grant) error {
+	return s.transition(ctx, g, func(a *Attempt) error {
+		if a.State != StateReleased {
+			return fmt.Errorf("%w: %s", ErrWrongState, a.State)
+		}
+		a.State = StateEnded
+		return nil
+	})
+}
+
+// Ambiguous records that the attempt's end is unproven. The node stays held
+// until Stopped; no amount of time releases it.
+func (s *Store) Ambiguous(ctx context.Context, g Grant) error {
+	return s.transition(ctx, g, func(a *Attempt) error {
+		switch a.State {
+		case StateRunning, StateAccepted, StateAmbiguous:
+			a.State = StateAmbiguous
+			return nil
+		}
+		return fmt.Errorf("%w: %s", ErrWrongState, a.State)
+	})
+}
+
+// Current returns the node's current attempt, read-only.
+func (s *Store) Current(ctx context.Context, node string) (Attempt, error) {
+	rec, _, err := s.read(ctx, node)
+	if errors.Is(err, jetstream.ErrKeyNotFound) || err == nil && rec.Attempt == nil {
+		return Attempt{}, ErrNotCurrent
+	}
+	if err != nil {
+		return Attempt{}, err
+	}
+	return *rec.Attempt, nil
+}
+
+// transition applies change to the attempt under g, written only at the
+// revision it was read, so a concurrent change refuses it.
+func (s *Store) transition(ctx context.Context, g Grant, change func(*Attempt) error) error {
+	rec, rev, err := s.read(ctx, g.Node)
+	if errors.Is(err, jetstream.ErrKeyNotFound) {
+		return ErrNotCurrent
+	}
+	if err != nil {
+		return err
+	}
+	if rec.Attempt == nil || rec.Attempt.Grant != g {
+		return ErrNotCurrent
+	}
+	a := *rec.Attempt
+	if err := change(&a); err != nil {
+		return err
+	}
+	rec.Attempt = &a
+	return s.write(ctx, g.Node, rec, rev)
+}
+
+func (s *Store) read(ctx context.Context, node string) (nodeRecord, uint64, error) {
+	e, err := s.kv.Get(ctx, nodeKey(node))
+	if errors.Is(err, jetstream.ErrKeyNotFound) {
+		return nodeRecord{}, 0, err
+	}
+	if err != nil {
+		return nodeRecord{}, 0, fmt.Errorf("execution: read node record: %w", err)
+	}
+	var rec nodeRecord
+	if err := json.Unmarshal(e.Value(), &rec); err != nil || rec.FormatVersion != formatVersion || rec.Seq == 0 ||
+		(rec.Attempt != nil && (rec.Attempt.Grant.Node != node || rec.Attempt.Grant.Seq != rec.Seq)) {
+		return nodeRecord{}, 0, ErrUnknownRecord
+	}
+	return rec, e.Revision(), nil
+}
+
+func (s *Store) create(ctx context.Context, node string, rec nodeRecord) error {
+	body, err := json.Marshal(rec)
+	if err != nil {
+		return err
+	}
+	if s.beforeWrite != nil {
+		s.beforeWrite()
+	}
+	if _, err := s.kv.Create(ctx, nodeKey(node), body); err != nil {
+		if errors.Is(err, jetstream.ErrKeyExists) {
+			return ErrNodeBusy
+		}
+		return fmt.Errorf("%w: %w", ErrOutcomeUnknown, err)
+	}
+	return nil
+}
+
+// write replaces the node record only at rev. A revision conflict means
+// another change won; any other error may have landed.
+func (s *Store) write(ctx context.Context, node string, rec nodeRecord, rev uint64) error {
+	body, err := json.Marshal(rec)
+	if err != nil {
+		return err
+	}
+	if s.beforeWrite != nil {
+		s.beforeWrite()
+	}
+	if _, err := s.kv.Update(ctx, nodeKey(node), body, rev); err != nil {
+		if errors.Is(err, jetstream.ErrKeyExists) {
+			return fmt.Errorf("%w: record changed concurrently", ErrNotCurrent)
+		}
+		return fmt.Errorf("%w: %w", ErrOutcomeUnknown, err)
+	}
+	return nil
+}
+
+func randomID() (string, error) {
+	b := make([]byte, 16)
+	if _, err := rand.Read(b); err != nil {
+		return "", fmt.Errorf("execution: grant ID: %w", err)
+	}
+	return hex.EncodeToString(b), nil
+}

spinifex/domains/lambda/execution/grant_test.go

@@ -0,0 +1,376 @@
+package execution
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/require"
+)
+
+type clock struct {
+	mu sync.Mutex
+	t  time.Time
+}
+
+func (c *clock) now() time.Time { c.mu.Lock(); defer c.mu.Unlock(); return c.t }
+
+func (c *clock) add(d time.Duration) { c.mu.Lock(); c.t = c.t.Add(d); c.mu.Unlock() }
+
+func newStore(t *testing.T) (*Store, *clock) {
+	t.Helper()
+	_, nc, _ := testutil.StartTestJetStream(t)
+	s, err := Open(context.Background(), testutil.NewJetStream(t, nc))
+	require.NoError(t, err)
+	c := &clock{t: time.Date(2026, 11, 3, 0, 0, 0, 0, time.UTC)}
+	s.now = c.now
+	return s, c
+}
+
+func attempt(c *clock, id string) Attempt {
+	return Attempt{
+		Grant:    Grant{AttemptID: id, EnvironmentID: "env-" + id, Generation: 1, Node: "node1"},
+		Identity: identity(id), LaunchBy: c.now().Add(time.Minute),
+	}
+}
+
+func identity(id string) Identity {
+	return Identity{RequestID: "req-" + id, AccountID: "123456789012", Region: "ap-southeast-2",
+		FunctionID: "fn1", RevisionID: "rev1", CodeSHA256: "aa"}
+}
+
+func state(t *testing.T, s *Store, id string) string {
+	t.Helper()
+	rec, _, err := s.read(context.Background(), "node1")
+	require.NoError(t, err)
+	require.NotNil(t, rec.Attempt)
+	require.Equal(t, id, rec.Attempt.Grant.AttemptID)
+	return rec.Attempt.State
+}
+
+func started(t *testing.T, s *Store, c *clock, id string) Attempt {
+	t.Helper()
+	a, err := s.Issue(context.Background(), attempt(c, id))
+	require.NoError(t, err)
+	require.NoError(t, s.Start(context.Background(), a.Grant, a.Identity))
+	return a
+}
+
+func TestGrant_LifecycleFreesTheSlotOnlyOnProvenStop(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	a := started(t, s, c, "a1")
+	require.Equal(t, uint64(1), a.Grant.Seq)
+	require.NotEmpty(t, a.Grant.GrantID)
+
+	_, err := s.Issue(ctx, attempt(c, "a2"))
+	require.ErrorIs(t, err, ErrNodeBusy, "no second grant while the first runs")
+
+	require.NoError(t, s.Accept(ctx, a.Grant))
+	require.ErrorIs(t, s.Accept(ctx, a.Grant), ErrWrongState, "one result per attempt")
+	_, err = s.Issue(ctx, attempt(c, "a2"))
+	require.ErrorIs(t, err, ErrNodeBusy, "an accepted result does not free the slot")
+
+	require.NoError(t, s.Released(ctx, a.Grant))
+	require.NoError(t, s.Stopped(ctx, a.Grant))
+	require.Equal(t, StateEnded, state(t, s, "a1"))
+	require.ErrorIs(t, s.Accept(ctx, a.Grant), ErrWrongState, "no result accepted after the end")
+
+	b, err := s.Issue(ctx, attempt(c, "a2"))
+	require.NoError(t, err)
+	require.Equal(t, uint64(2), b.Grant.Seq, "later grants order after earlier ones")
+}
+
+func TestGrant_AmbiguousHoldsTheSlotUntilStopIsProven(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	a := started(t, s, c, "a1")
+	require.NoError(t, s.Ambiguous(ctx, a.Grant))
+	require.NoError(t, s.Ambiguous(ctx, a.Grant), "holding again is a no-op")
+
+	c.add(24 * time.Hour)
+	_, err := s.Issue(ctx, attempt(c, "a2"))
+	require.ErrorIs(t, err, ErrNodeBusy, "time alone never releases an ambiguous attempt")
+	require.ErrorIs(t, s.Accept(ctx, a.Grant), ErrWrongState, "no result accepted while ambiguous")
+	require.ErrorIs(t, s.Withdraw(ctx, a.Grant), ErrWrongState, "a started attempt cannot be withdrawn")
+
+	require.NoError(t, s.Released(ctx, a.Grant))
+	require.NoError(t, s.Stopped(ctx, a.Grant))
+	_, err = s.Issue(ctx, attempt(c, "a2"))
+	require.NoError(t, err)
+}
+
+func TestGrant_StartRefusesReplayAndEveryMismatchedIdentity(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	a, err := s.Issue(ctx, attempt(c, "a1"))
+	require.NoError(t, err)
+
+	mismatched := map[string]func(g *Grant){
+		"grant ID":    func(g *Grant) { g.GrantID = "other" },
+		"older seq":   func(g *Grant) { g.Seq-- },
+		"environment": func(g *Grant) { g.EnvironmentID = "env-other" },
+		"generation":  func(g *Grant) { g.Generation++ },
+		"attempt":     func(g *Grant) { g.AttemptID = "a9" },
+		"node":        func(g *Grant) { g.Node = "node2" },
+	}
+	for name, mutate := range mismatched {
+		g := a.Grant
+		mutate(&g)
+		require.ErrorIs(t, s.Start(ctx, g, a.Identity), ErrNotCurrent, name)
+		require.ErrorIs(t, s.Stopped(ctx, g), ErrNotCurrent, name)
+	}
+
+	require.NoError(t, s.Start(ctx, a.Grant, a.Identity))
+	require.ErrorIs(t, s.Start(ctx, a.Grant, a.Identity), ErrWrongState, "a replayed start is refused")
+}
+
+func TestGrant_ExpiredGrantIsSupersededOnlyIfItNeverStarted(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	old, err := s.Issue(ctx, attempt(c, "a1"))
+	require.NoError(t, err)
+
+	c.add(2 * time.Minute)
+	require.ErrorIs(t, s.Start(ctx, old.Grant, old.Identity), ErrExpired)
+
+	next, err := s.Issue(ctx, attempt(c, "a2"))
+	require.NoError(t, err, "a holder that can never start does not block the node")
+	require.Greater(t, next.Grant.Seq, old.Grant.Seq)
+	require.ErrorIs(t, s.Start(ctx, old.Grant, old.Identity), ErrNotCurrent, "the superseded grant stays refused")
+
+	// A started holder is never superseded, however late.
+	require.NoError(t, s.Start(ctx, next.Grant, next.Identity))
+	c.add(24 * time.Hour)
+	_, err = s.Issue(ctx, attempt(c, "a3"))
+	require.ErrorIs(t, err, ErrNodeBusy)
+	require.Equal(t, StateRunning, state(t, s, "a2"))
+}
+
+func TestGrant_WithdrawFreesOnlyANeverStartedGrant(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	a, err := s.Issue(ctx, attempt(c, "a1"))
+	require.NoError(t, err)
+	require.NoError(t, s.Withdraw(ctx, a.Grant))
+	require.ErrorIs(t, s.Start(ctx, a.Grant, a.Identity), ErrWrongState, "a withdrawn grant can never start")
+	_, err = s.Issue(ctx, attempt(c, "a2"))
+	require.NoError(t, err)
+}
+
+func TestGrant_UnexpiredGrantedHolderBlocks(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	_, err := s.Issue(ctx, attempt(c, "a1"))
+	require.NoError(t, err)
+	_, err = s.Issue(ctx, attempt(c, "a2"))
+	require.ErrorIs(t, err, ErrNodeBusy)
+}
+
+func TestGrant_ConcurrentIssueGrantsExactlyOne(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	// Seed the slot so racers contend on the conditional update, not Create.
+	seed, err := s.Issue(ctx, attempt(c, "seed"))
+	require.NoError(t, err)
+	require.NoError(t, s.Withdraw(ctx, seed.Grant))
+
+	const racers = 16
+	var wg sync.WaitGroup
+	results := make(chan error, racers)
+	for i := range racers {
+		wg.Go(func() {
+			_, err := s.Issue(ctx, attempt(c, fmt.Sprintf("r%d", i)))
+			results <- err
+		})
+	}
+	wg.Wait()
+	close(results)
+	granted := 0
+	for err := range results {
+		if err == nil {
+			granted++
+		} else {
+			require.ErrorIs(t, err, ErrNodeBusy)
+		}
+	}
+	require.Equal(t, 1, granted)
+}
+
+// TestGrant_StartLosesToASupersedingGrant forces the interleaving where a
+// Start has read an expiring grant and a new grant replaces it before Start
+// writes: Start must fail, leaving only the new grant able to run.
+func TestGrant_StartLosesToASupersedingGrant(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	old, err := s.Issue(ctx, attempt(c, "a1"))
+	require.NoError(t, err)
+
+	// Start reads at a time the old grant is still valid; the new grant is
+	// issued after expiry, between Start's read and its write.
+	var next Attempt
+	s.beforeWrite = func() {
+		s.beforeWrite = nil
+		c.add(2 * time.Minute)
+		next, err = s.Issue(ctx, attempt(c, "a2"))
+		require.NoError(t, err)
+	}
+	require.ErrorIs(t, s.Start(ctx, old.Grant, old.Identity), ErrNotCurrent)
+	c.add(-2 * time.Minute)
+	require.NoError(t, s.Start(ctx, next.Grant, next.Identity))
+	require.ErrorIs(t, s.Start(ctx, old.Grant, old.Identity), ErrNotCurrent)
+	require.Equal(t, StateRunning, state(t, s, "a2"))
+}
+
+func TestGrant_AcceptLosesToAConcurrentHold(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	a := started(t, s, c, "a1")
+	s.beforeWrite = func() {
+		s.beforeWrite = nil
+		require.NoError(t, s.Ambiguous(ctx, a.Grant))
+	}
+	require.ErrorIs(t, s.Accept(ctx, a.Grant), ErrNotCurrent, "the result was not accepted")
+	require.Equal(t, StateAmbiguous, state(t, s, "a1"))
+}
+
+func TestGrant_DuplicateAttemptIsRefused(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	a := started(t, s, c, "a1")
+	require.NoError(t, s.Released(ctx, a.Grant))
+	require.NoError(t, s.Stopped(ctx, a.Grant))
+
+	_, err := s.Issue(ctx, attempt(c, "a1"))
+	require.ErrorIs(t, err, ErrWrongState, "an attempt ID is not granted twice in a row")
+	_, err = s.Issue(ctx, attempt(c, "a2"))
+	require.NoError(t, err, "the refused duplicate does not take the slot")
+}
+
+// lostAck lands attempt Creates and then reports a failure, as a lost
+// acknowledgement would.
+type lostAck struct{ jetstream.KeyValue }
+
+func (l lostAck) Create(ctx context.Context, key string, v []byte, opts ...jetstream.KVCreateOpt) (uint64, error) {
+	if _, err := l.KeyValue.Create(ctx, key, v, opts...); err != nil {
+		return 0, err
+	}
+	return 0, errors.New("ack lost")
+}
+
+func TestGrant_AmbiguousRecordWriteKeepsTheSlot(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	inner := s.kv
+	s.kv = lostAck{inner}
+	_, err := s.Issue(ctx, attempt(c, "a1"))
+	require.ErrorIs(t, err, ErrOutcomeUnknown)
+	s.kv = inner
+
+	_, err = s.Issue(ctx, attempt(c, "a2"))
+	require.ErrorIs(t, err, ErrNodeBusy, "a grant that may be recorded keeps the slot")
+	require.Equal(t, StateGranted, state(t, s, "a1"))
+	c.add(2 * time.Minute)
+	_, err = s.Issue(ctx, attempt(c, "a2"))
+	require.NoError(t, err, "past its deadline the landed grant can never start")
+}
+
+func TestGrant_UnrecognisedRecordIsRefusedUnchanged(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	a, err := s.Issue(ctx, attempt(c, "a1"))
+	require.NoError(t, err)
+	require.NoError(t, s.Withdraw(ctx, a.Grant))
+
+	e, err := s.kv.Get(ctx, nodeKey("node1"))
+	require.NoError(t, err)
+	var raw map[string]any
+	require.NoError(t, json.Unmarshal(e.Value(), &raw))
+	raw["formatVersion"] = formatVersion + 1
+	body, _ := json.Marshal(raw)
+	rev, err := s.kv.Update(ctx, nodeKey("node1"), body, e.Revision())
+	require.NoError(t, err)
+
+	_, err = s.Issue(ctx, attempt(c, "a2"))
+	require.ErrorIs(t, err, ErrUnknownRecord)
+	require.ErrorIs(t, s.Start(ctx, a.Grant, a.Identity), ErrUnknownRecord)
+	after, err := s.kv.Get(ctx, nodeKey("node1"))
+	require.NoError(t, err)
+	require.Equal(t, rev, after.Revision())
+	require.Equal(t, body, after.Value())
+}
+
+func TestGrant_StartRefusesAMismatchedRequestUnchanged(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	a, err := s.Issue(ctx, attempt(c, "a1"))
+	require.NoError(t, err)
+	mismatched := map[string]func(id *Identity){
+		"request":  func(id *Identity) { id.RequestID = "req-other" },
+		"account":  func(id *Identity) { id.AccountID = "210987654321" },
+		"region":   func(id *Identity) { id.Region = "us-east-1" },
+		"function": func(id *Identity) { id.FunctionID = "fn2" },
+		"revision": func(id *Identity) { id.RevisionID = "rev2" },
+		"code":     func(id *Identity) { id.CodeSHA256 = "bb" },
+	}
+	for name, mutate := range mismatched {
+		want := a.Identity
+		mutate(&want)
+		require.ErrorIs(t, s.Start(ctx, a.Grant, want), ErrMismatch, name)
+		require.Equal(t, StateGranted, state(t, s, "a1"), name)
+	}
+	require.NoError(t, s.Start(ctx, a.Grant, a.Identity))
+}
+
+// A released attempt still holds the node; only Stopped, and only after
+// Released, frees it for the next grant.
+func TestReleasedHoldsTheNodeUntilStopped(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	a, err := s.Issue(ctx, attempt(c, "a1"))
+	require.NoError(t, err)
+	require.NoError(t, s.Start(ctx, a.Grant, a.Identity))
+	require.ErrorIs(t, s.Stopped(ctx, a.Grant), ErrWrongState)
+
+	require.NoError(t, s.Released(ctx, a.Grant))
+	_, err = s.Issue(ctx, attempt(c, "a2"))
+	require.ErrorIs(t, err, ErrNodeBusy)
+
+	require.NoError(t, s.Stopped(ctx, a.Grant))
+	_, err = s.Issue(ctx, attempt(c, "a2"))
+	require.NoError(t, err)
+}
+
+// The launch identity is recorded once, complete, and only while running, so
+// recovery never acts on a partial or replaced identity.
+func TestGrant_LaunchedRecordsACompleteIdentityOnce(t *testing.T) {
+	s, c := newStore(t)
+	ctx := context.Background()
+	l := Launch{MachineID: "m", BootID: "b", PIDNamespace: 4026531836, LauncherPID: 10,
+		LauncherInode: 11, PID: 20, Inode: 21}
+
+	granted, err := s.Issue(ctx, attempt(c, "a0"))
+	require.NoError(t, err)
+	require.ErrorIs(t, s.Launched(ctx, granted.Grant, l), ErrWrongState, "not yet running")
+	require.NoError(t, s.Withdraw(ctx, granted.Grant))
+
+	a := started(t, s, c, "a1")
+	partial := l
+	partial.Inode = 0
+	require.Error(t, s.Launched(ctx, a.Grant, partial))
+	require.NoError(t, s.Launched(ctx, a.Grant, l))
+	other := l
+	other.PID = 99
+	require.ErrorIs(t, s.Launched(ctx, a.Grant, other), ErrWrongState, "recorded once")
+
+	rec, err := s.Current(ctx, "node1")
+	require.NoError(t, err)
+	require.NotNil(t, rec.Launch)
+	require.Equal(t, l, *rec.Launch)
+}

spinifex/domains/lambda/function/meta.go

@@ -26,6 +26,7 @@ var (
 	nameRE   = regexp.MustCompile(`^[a-zA-Z0-9-_]{1,64}$`)
 	regionRE = regexp.MustCompile(`^[a-z0-9-]{1,32}$`)
 	digestRE = regexp.MustCompile(`^[0-9a-f]{64}$`)
+	idRE     = regexp.MustCompile(`^[a-zA-Z0-9_-]{1,64}$`)
 )
 
 // ValidName reports whether name is a legal function name.
@@ -81,17 +82,15 @@ func decode(data []byte) (Record, error) {
 	return rec, nil
 }
 
-// Create binds the name by writing the record with kv.Create. The record starts
-// Pending with its code storing and a claim that expires after ClaimTTL. A name
-// held only by an expired, still-storing claim is taken over once.
-//
-// Orphaned code objects from rolled-back or taken-over claims are not cleaned up yet.
+// Create records the code upload, then claims the name with kv.Create: Pending,
+// code storing, claim expiring after ClaimTTL. A name held only by an expired,
+// still-storing claim is taken over once; the taken-over upload stays owed.
 func (s *MetaService) Create(ctx context.Context, req *CreateRequest, accountID string) (*CreateResponse, error) {
 	rec := req.Record
 	if !regionRE.MatchString(req.Region) {
 		return nil, badRegion()
 	}
-	if !ValidName(rec.FunctionName) || rec.FunctionID == "" || rec.Code.RevisionID == "" || accountID == "" {
+	if !ValidName(rec.FunctionName) || !idRE.MatchString(rec.FunctionID) || !idRE.MatchString(rec.Code.RevisionID) || accountID == "" {
 		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, "function record is incomplete")
 	}
 	kv, err := s.bucket(ctx, accountID)
@@ -112,6 +111,21 @@ func (s *MetaService) Create(ctx context.Context, req *CreateRequest, accountID
 	}
 	key := KVFunctionKey(req.Region, rec.FunctionName)
 
+	// The upload is recorded before the claim exists, so no copy can be written
+	// for a claim that has no upload record.
+	upKey := KVUploadKey(req.Region, rec.FunctionID, rec.Code.RevisionID)
+	upData, err := json.Marshal(Upload{
+		FormatVersion: FormatVersion, AccountID: accountID, Region: req.Region, FunctionName: rec.FunctionName,
+		FunctionID: rec.FunctionID, RevisionID: rec.Code.RevisionID, State: UploadOwed, CreatedAt: now,
+	})
+	if err != nil {
+		return nil, err
+	}
+	upRev, err := kv.Create(ctx, upKey, upData)
+	if err != nil {
+		return nil, fmt.Errorf("lambda: record code upload: %w", err)
+	}
+
 	rev, err := kv.Create(ctx, key, data)
 	if err == nil {
 		return &CreateResponse{Revision: rev}, nil
@@ -123,27 +137,85 @@ func (s *MetaService) Create(ctx context.Context, req *CreateRequest, accountID
 	exists := conflict("Function already exist: %s", rec.FunctionName)
 	entry, err := kv.Get(ctx, key)
 	if err != nil {
-		return nil, exists
+		return nil, s.refused(ctx, kv, upKey, upRev, exists)
 	}
 	old, err := decode(entry.Value())
 	if err != nil || old.Code.State != CodeStoring || !s.now().After(old.ClaimExpiresAt) {
-		return nil, exists
+		return nil, s.refused(ctx, kv, upKey, upRev, exists)
 	}
 	if err := kv.Delete(ctx, key, jetstream.LastRevision(entry.Revision())); err != nil {
-		return nil, exists
+		return nil, s.refused(ctx, kv, upKey, upRev, exists)
 	}
 	slog.WarnContext(ctx, "Lambda: took over an expired function name claim",
 		"function_name", rec.FunctionName, "old_function_id", old.FunctionID)
 	rev, err = kv.Create(ctx, key, data)
 	if err != nil {
 		if errors.Is(err, jetstream.ErrKeyExists) {
-			return nil, exists
+			return nil, s.refused(ctx, kv, upKey, upRev, exists)
 		}
 		return nil, fmt.Errorf("lambda: create function record: %w", err)
 	}
 	return &CreateResponse{Revision: rev}, nil
 }
 
+// refused drops the upload record of a create that certainly holds no claim:
+// the gateway writes code only after Create succeeds. A failed drop leaves the
+// upload owed, which is safe.
+func (s *MetaService) refused(ctx context.Context, kv jetstream.KeyValue, upKey string, upRev uint64, err error) error {
+	if derr := kv.Delete(ctx, upKey, jetstream.LastRevision(upRev)); derr != nil {
+		slog.WarnContext(ctx, "Lambda: upload record of a refused create was not dropped", "key", upKey, "err", derr)
+	}
+	return err
+}
+
+// checkUpload returns the owed upload record for rec. Any other record, or
+// none, refuses the caller; nothing is rewritten, because the state of a record
+// this owner cannot vouch for is unknown.
+func checkUpload(ctx context.Context, kv jetstream.KeyValue, rec Record) (Upload, uint64, error) {
+	refuse := func(why string, args ...any) (Upload, uint64, error) {
+		slog.WarnContext(ctx, "Lambda: code upload record refused; its state is unknown",
+			"function_id", rec.FunctionID, "revision_id", rec.Code.RevisionID, "reason", fmt.Sprintf(why, args...))
+		return Upload{}, 0, awserrors.Errorf(awserrors.ErrorServiceException, "function code upload is not tracked")
+	}
+	entry, err := kv.Get(ctx, KVUploadKey(rec.Region, rec.FunctionID, rec.Code.RevisionID))
+	if errors.Is(err, jetstream.ErrKeyNotFound) {
+		return refuse("missing")
+	}
+	if err != nil {
+		return Upload{}, 0, fmt.Errorf("lambda: get code upload record: %w", err)
+	}
+	var up Upload
+	if err := json.Unmarshal(entry.Value(), &up); err != nil {
+		return refuse("unreadable")
+	}
+	if up.FormatVersion != FormatVersion {
+		return refuse("format version %d", up.FormatVersion)
+	}
+	if up.AccountID != rec.AccountID || up.Region != rec.Region || up.FunctionName != rec.FunctionName ||
+		up.FunctionID != rec.FunctionID || up.RevisionID != rec.Code.RevisionID {
+		return refuse("identity mismatch")
+	}
+	if up.State != UploadOwed {
+		return refuse("state %s", up.State)
+	}
+	return up, entry.Revision(), nil
+}
+
+// attach marks the checked upload as named by the function record at revision.
+// KV has no multi-key write, so a change since the check fails this CAS, and
+// the record is then left as found, its state unknown.
+func attach(ctx context.Context, kv jetstream.KeyValue, up Upload, upRev, revision uint64) {
+	up.State, up.AttachedRevision = UploadAttached, revision
+	data, err := json.Marshal(up)
+	if err == nil {
+		_, err = kv.Update(ctx, KVUploadKey(up.Region, up.FunctionID, up.RevisionID), data, upRev)
+	}
+	if err != nil {
+		slog.WarnContext(ctx, "Lambda: upload record not marked attached; its state is unknown",
+			"function_id", up.FunctionID, "revision_id", up.RevisionID, "err", err)
+	}
+}
+
 // Get returns the record, or Found false when the name is unbound.
 func (s *MetaService) Get(ctx context.Context, req *GetRequest, accountID string) (*GetResponse, error) {
 	if !regionRE.MatchString(req.Region) {
@@ -190,8 +262,9 @@ func (s *MetaService) readStoring(ctx context.Context, kv jetstream.KeyValue, re
 	return rec, nil
 }
 
-// CodeStored marks the code stored. It is refused, leaving the record unchanged,
-// when the revision moved, the code is not storing, or the claim has expired.
+// CodeStored marks the code stored. It is refused, leaving both records unchanged,
+// when the revision moved, the code is not storing, the claim has expired, or
+// the upload record is not the owed one this create wrote.
 func (s *MetaService) CodeStored(ctx context.Context, req *CodeStoredRequest, accountID string) (*CodeStoredResponse, error) {
 	if !regionRE.MatchString(req.Region) {
 		return nil, badRegion()
@@ -214,6 +287,10 @@ func (s *MetaService) CodeStored(ctx context.Context, req *CodeStoredRequest, ac
 	if now.After(rec.ClaimExpiresAt) {
 		return nil, conflict("Function creation claim expired: %s", req.FunctionName)
 	}
+	up, upRev, err := checkUpload(ctx, kv, rec)
+	if err != nil {
+		return nil, err
+	}
 	rec.Code.State, rec.Code.SHA256, rec.Code.Size = CodeStored, req.SHA256, req.Size
 	rec.State, rec.StateReasonCode, rec.StateReason = StatePending, ReasonCodeCreating, ReasonCreating
 	rec.LastModified = now.UTC()
@@ -228,12 +305,13 @@ func (s *MetaService) CodeStored(ctx context.Context, req *CodeStoredRequest, ac
 		}
 		return nil, fmt.Errorf("lambda: update function record: %w", err)
 	}
+	attach(ctx, kv, up, upRev, rev)
 	return &CodeStoredResponse{Record: rec, Revision: rev}, nil
 }
 
 // RollbackCreate frees the name of a create whose own code write failed. It
 // deletes only a record still storing at exactly the revision create returned;
-// an already-absent name is success.
+// an already-absent name is success. The upload stays owed: the write may have landed.
 func (s *MetaService) RollbackCreate(ctx context.Context, req *RollbackCreateRequest, accountID string) (*RollbackCreateResponse, error) {
 	if !regionRE.MatchString(req.Region) {
 		return nil, badRegion()

spinifex/domains/lambda/function/record.go

@@ -62,8 +62,37 @@ const (
 	KVBucketAccountPrefix  = "lambda-account-"
 	KVBucketAccountHistory = 1
 	kvFunctionKeyPrefix    = "fn."
+	kvUploadKeyPrefix      = "upload."
 )
 
+// KVUploadKey returns the key of the record that tracks one code upload. The
+// bucket is shared across Regions, so the Region is part of the key.
+func KVUploadKey(region, functionID, revisionID string) string {
+	return kvUploadKeyPrefix + region + "." + functionID + "." + revisionID
+}
+
+// Upload states. An owed upload may have a code copy that no function record
+// names; it stays owed until the owner records otherwise.
+const (
+	UploadOwed     = "owed"
+	UploadAttached = "attached"
+)
+
+// Upload tracks one code upload. A create begun through this owner writes it
+// before the copy; records and copies from older owners have none.
+type Upload struct {
+	FormatVersion int       `json:"formatVersion"`
+	AccountID     string    `json:"accountId"`
+	Region        string    `json:"region"`
+	FunctionName  string    `json:"functionName"`
+	FunctionID    string    `json:"functionId"`
+	RevisionID    string    `json:"revisionId"`
+	State         string    `json:"state"`
+	CreatedAt     time.Time `json:"createdAt"`
+	// AttachedRevision is the function record revision that names the copy.
+	AttachedRevision uint64 `json:"attachedRevision,omitempty"`
+}
+
 // KVAccountBucket returns the per-account JetStream KV bucket name.
 func KVAccountBucket(accountID string) string { return KVBucketAccountPrefix + accountID }
 

spinifex/domains/lambda/function/upload_test.go

@@ -0,0 +1,212 @@
+package function
+
+import (
+	"encoding/json"
+	"errors"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// upload reads the tracking record for functionID, or reports it absent.
+func upload(t *testing.T, svc *MetaService, functionID string) (Upload, bool) {
+	t.Helper()
+	return uploadIn(t, svc, testRegion, functionID)
+}
+
+func uploadIn(t *testing.T, svc *MetaService, region, functionID string) (Upload, bool) {
+	t.Helper()
+	kv, err := svc.bucket(t.Context(), testAccount)
+	require.NoError(t, err)
+	entry, err := kv.Get(t.Context(), KVUploadKey(region, functionID, "rev-"+functionID))
+	if errors.Is(err, jetstream.ErrKeyNotFound) {
+		return Upload{}, false
+	}
+	require.NoError(t, err)
+	var up Upload
+	require.NoError(t, json.Unmarshal(entry.Value(), &up))
+	return up, true
+}
+
+func requireOwed(t *testing.T, svc *MetaService, functionID string) {
+	t.Helper()
+	up, ok := upload(t, svc, functionID)
+	require.True(t, ok, "upload record for %s is missing", functionID)
+	assert.Equal(t, UploadOwed, up.State)
+}
+
+func TestUpload_CreateRecordsItOwedAndCodeStoredAttachesIt(t *testing.T) {
+	svc, c := newTestService(t)
+	rev := mustCreate(t, svc, "hello", "f1")
+	up, ok := upload(t, svc, "f1")
+	require.True(t, ok)
+	assert.Equal(t, Upload{FormatVersion: FormatVersion, AccountID: testAccount, Region: testRegion, FunctionName: "hello",
+		FunctionID: "f1", RevisionID: "rev-f1", State: UploadOwed, CreatedAt: c.t}, up)
+
+	stored, err := svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: rev, SHA256: testDigest, Size: 9}, testAccount)
+	require.NoError(t, err)
+	up, ok = upload(t, svc, "f1")
+	require.True(t, ok)
+	assert.Equal(t, UploadAttached, up.State)
+	assert.Equal(t, stored.Revision, up.AttachedRevision)
+}
+
+func TestUpload_IsRecordedBeforeTheNameIsClaimed(t *testing.T) {
+	svc, _ := newTestService(t)
+	kv, err := svc.bucket(t.Context(), testAccount)
+	require.NoError(t, err)
+	_, err = kv.Create(t.Context(), KVUploadKey(testRegion, "f1", "rev-f1"), []byte(`{}`))
+	require.NoError(t, err)
+
+	_, err = svc.Create(t.Context(), createReq("hello", "f1"), testAccount)
+	require.Error(t, err)
+	assert.False(t, mustGet(t, svc, "hello").Found, "no claim exists when the upload could not be recorded")
+}
+
+func TestUpload_RefusedDuplicateLeavesNoRecord(t *testing.T) {
+	svc, _ := newTestService(t)
+	mustCreate(t, svc, "hello", "f1")
+	_, err := svc.Create(t.Context(), createReq("hello", "f2"), testAccount)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
+	_, ok := upload(t, svc, "f2")
+	assert.False(t, ok, "a create refused before any claim can never be written to")
+	requireOwed(t, svc, "f1")
+}
+
+func TestUpload_RollbackKeepsItOwed(t *testing.T) {
+	svc, _ := newTestService(t)
+	rev := mustCreate(t, svc, "hello", "f1")
+	_, err := svc.RollbackCreate(t.Context(), &RollbackCreateRequest{Region: testRegion, FunctionName: "hello", Revision: rev}, testAccount)
+	require.NoError(t, err)
+	assert.False(t, mustGet(t, svc, "hello").Found)
+	requireOwed(t, svc, "f1")
+}
+
+func TestUpload_ExpiredClaimIsNotTakenAsCompletion(t *testing.T) {
+	svc, c := newTestService(t)
+	rev := mustCreate(t, svc, "hello", "f1")
+	c.t = c.t.Add(ClaimTTL + time.Second)
+	_, err := svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: rev, SHA256: testDigest, Size: 9}, testAccount)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
+	requireOwed(t, svc, "f1")
+}
+
+// The original writer stalls past its claim, a second create takes the name,
+// and the original then finishes its write and reports it.
+func TestUpload_TakeoverAndLateWriterLeaveTheOriginalOwed(t *testing.T) {
+	svc, c := newTestService(t)
+	oldRev := mustCreate(t, svc, "hello", "f1")
+	c.t = c.t.Add(ClaimTTL + time.Second)
+	newRev := mustCreate(t, svc, "hello", "f2")
+	requireOwed(t, svc, "f1")
+	requireOwed(t, svc, "f2")
+
+	_, err := svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: oldRev, SHA256: testDigest, Size: 9}, testAccount)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
+	_, err = svc.RollbackCreate(t.Context(), &RollbackCreateRequest{Region: testRegion, FunctionName: "hello", Revision: oldRev}, testAccount)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
+	requireOwed(t, svc, "f1")
+
+	_, err = svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: newRev, SHA256: testDigest, Size: 9}, testAccount)
+	require.NoError(t, err)
+	up, _ := upload(t, svc, "f2")
+	assert.Equal(t, UploadAttached, up.State)
+	requireOwed(t, svc, "f1")
+}
+
+func TestUpload_RefusesIdentifiersThatCannotBeKeys(t *testing.T) {
+	svc, _ := newTestService(t)
+	for _, id := range []string{"a.b", "a b", "a*"} {
+		_, err := svc.Create(t.Context(), createReq("hello", id), testAccount)
+		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+	}
+	assert.False(t, mustGet(t, svc, "hello").Found)
+}
+
+func TestUpload_SameIdentifiersInTwoRegionsAreTrackedApart(t *testing.T) {
+	svc, _ := newTestService(t)
+	rev := mustCreate(t, svc, "hello", "f1")
+	other := createReq("hello", "f1")
+	other.Region = "us-east-1"
+	_, err := svc.Create(t.Context(), other, testAccount)
+	require.NoError(t, err)
+
+	_, err = svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: rev, SHA256: testDigest, Size: 9}, testAccount)
+	require.NoError(t, err)
+	here, ok := uploadIn(t, svc, testRegion, "f1")
+	require.True(t, ok)
+	assert.Equal(t, UploadAttached, here.State)
+	there, ok := uploadIn(t, svc, "us-east-1", "f1")
+	require.True(t, ok)
+	assert.Equal(t, UploadOwed, there.State)
+	assert.Equal(t, "us-east-1", there.Region)
+}
+
+// An upload record the owner cannot vouch for refuses CodeStored, and neither
+// it nor the function record changes.
+func TestUpload_CodeStoredRefusesAnUnrecognisedRecord(t *testing.T) {
+	asJSON := func(mutate func(*Upload)) func([]byte) []byte {
+		return func(b []byte) []byte {
+			var up Upload
+			if err := json.Unmarshal(b, &up); err != nil {
+				panic(err)
+			}
+			mutate(&up)
+			out, err := json.Marshal(up)
+			if err != nil {
+				panic(err)
+			}
+			return out
+		}
+	}
+	for name, rewrite := range map[string]func([]byte) []byte{
+		"missing":          nil,
+		"unreadable":       func([]byte) []byte { return []byte("not json") },
+		"future format":    asJSON(func(u *Upload) { u.FormatVersion = FormatVersion + 1 }),
+		"other function":   asJSON(func(u *Upload) { u.FunctionName = "other" }),
+		"other account":    asJSON(func(u *Upload) { u.AccountID = "000000000002" }),
+		"already attached": asJSON(func(u *Upload) { u.State, u.AttachedRevision = UploadAttached, 7 }),
+	} {
+		t.Run(name, func(t *testing.T) {
+			svc, _ := newTestService(t)
+			rev := mustCreate(t, svc, "hello", "f1")
+			kv, err := svc.bucket(t.Context(), testAccount)
+			require.NoError(t, err)
+			key := KVUploadKey(testRegion, "f1", "rev-f1")
+			entry, err := kv.Get(t.Context(), key)
+			require.NoError(t, err)
+			if rewrite == nil {
+				require.NoError(t, kv.Delete(t.Context(), key))
+			} else {
+				_, err = kv.Update(t.Context(), key, rewrite(entry.Value()), entry.Revision())
+				require.NoError(t, err)
+			}
+			before, err := kv.Get(t.Context(), key)
+			if rewrite == nil {
+				require.ErrorIs(t, err, jetstream.ErrKeyNotFound)
+			} else {
+				require.NoError(t, err)
+			}
+
+			_, err = svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: rev, SHA256: testDigest, Size: 9}, testAccount)
+			testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
+
+			fn := mustGet(t, svc, "hello")
+			assert.Equal(t, rev, fn.Revision, "the function record is unchanged")
+			assert.Equal(t, CodeStoring, fn.Record.Code.State)
+			after, err := kv.Get(t.Context(), key)
+			if rewrite == nil {
+				require.ErrorIs(t, err, jetstream.ErrKeyNotFound, "no record is invented")
+				return
+			}
+			require.NoError(t, err)
+			assert.Equal(t, before.Revision(), after.Revision())
+			assert.Equal(t, before.Value(), after.Value())
+		})
+	}
+}

spinifex/domains/network/host/owned_tap.go

@@ -0,0 +1,128 @@
+//go:build lambdatestlink
+
+package host
+
+import (
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	hostcommand "github.com/mulgadc/spinifex/spinifex/runtime/host/command"
+)
+
+// ErrTapExists means a link with the requested name already exists; nothing
+// was changed.
+var ErrTapExists = errors.New("tap name already in use")
+
+// ErrTapNotCreated wraps a CreateOwnedTap failure that left no link behind.
+// Any other failure may have left one.
+var ErrTapNotCreated = errors.New("no tap created")
+
+// ownedTapSysfs is a var so tests can stand in a fake /sys/class/net.
+var ownedTapSysfs = "/sys/class/net"
+
+const ownerExternalID = "spinifex-owner"
+
+// OwnedTap identifies a tap CreateOwnedTap made: a later remove acts only on a
+// link with the same ifindex and alias, and an OVS port with the same owner.
+type OwnedTap struct {
+	Name    string
+	IfIndex int
+	Owner   string
+}
+
+// CreateOwnedTap creates and attaches a new tap, refusing to replace a link or
+// OVS port of the same name. The check-then-create gap remains, so callers use
+// names no other creator would pick.
+func (p *OVSPlumber) CreateOwnedTap(spec vm.TapSpec, owner string) (OwnedTap, error) {
+	if owner == "" || strings.ContainsAny(owner, " \t\n\"'") {
+		return OwnedTap{}, fmt.Errorf("%w: invalid owner", ErrTapNotCreated)
+	}
+	if _, err := os.Lstat(filepath.Join(ownedTapSysfs, spec.Name)); err == nil {
+		return OwnedTap{}, fmt.Errorf("%w: %w: %s", ErrTapNotCreated, ErrTapExists, spec.Name)
+	}
+	addArgs := []string{"tuntap", "add", "dev", spec.Name, "mode", "tap"}
+	if uid := os.Geteuid(); uid != 0 {
+		addArgs = append(addArgs, "user", strconv.Itoa(uid), "group", strconv.Itoa(os.Getegid()))
+	}
+	if out, err := hostcommand.SudoCommand("ip", addArgs...).CombinedOutput(); err != nil {
+		// The add may have taken effect before failing, so this is not ErrTapNotCreated.
+		return OwnedTap{}, fmt.Errorf("create tap %s: %s: %w", spec.Name, strings.TrimSpace(string(out)), err)
+	}
+	idx, err := readIfIndex(spec.Name)
+	if err != nil {
+		// Without its ifindex the new link cannot be told apart, so it stays.
+		return OwnedTap{}, fmt.Errorf("created tap %s but cannot read its ifindex; left in place: %w", spec.Name, err)
+	}
+	tap := OwnedTap{Name: spec.Name, IfIndex: idx, Owner: owner}
+	upArgs := []string{"link", "set", "dev", spec.Name, "alias", owner, "up"}
+	if spec.MTU > 0 {
+		upArgs = append(upArgs, "mtu", strconv.Itoa(spec.MTU))
+	}
+	if out, err := hostcommand.SudoCommand("ip", upArgs...).CombinedOutput(); err != nil {
+		return tap, errors.Join(fmt.Errorf("bring up tap %s: %s: %w", spec.Name, strings.TrimSpace(string(out)), err), p.RemoveOwnedTap(tap))
+	}
+	// No --may-exist: a port record of the same name fails the add.
+	addPort := []string{"add-port", spec.Bridge, spec.Name, "--", "set", "Interface", spec.Name,
+		fmt.Sprintf("external_ids:%s=%s", ownerExternalID, owner)}
+	if out, err := hostcommand.SudoCommand("ovs-vsctl", addPort...).CombinedOutput(); err != nil {
+		return tap, errors.Join(fmt.Errorf("add tap %s to %s: %s: %w", spec.Name, spec.Bridge, strings.TrimSpace(string(out)), err), p.removeOwnedLink(tap))
+	}
+	return tap, nil
+}
+
+// RemoveOwnedTap removes the OVS port and kernel link only where each still
+// carries tap's owner; anything else under that name is left alone. A link
+// already gone is success.
+func (p *OVSPlumber) RemoveOwnedTap(tap OwnedTap) error {
+	out, err := hostcommand.SudoCommand("ovs-vsctl", "--if-exists", "get", "Interface", tap.Name,
+		"external_ids:"+ownerExternalID).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("read OVS owner of %s: %s: %w", tap.Name, strings.TrimSpace(string(out)), err)
+	}
+	switch got := strings.Trim(strings.TrimSpace(string(out)), `"`); got {
+	case tap.Owner:
+		if out, err := hostcommand.SudoCommand("ovs-vsctl", "--if-exists", "del-port", tap.Name).CombinedOutput(); err != nil {
+			return fmt.Errorf("remove OVS port %s: %s: %w", tap.Name, strings.TrimSpace(string(out)), err)
+		}
+	case "":
+	default:
+		return fmt.Errorf("OVS port %s belongs to another owner; left in place", tap.Name)
+	}
+	return p.removeOwnedLink(tap)
+}
+
+func (p *OVSPlumber) removeOwnedLink(tap OwnedTap) error {
+	idx, err := readIfIndex(tap.Name)
+	if errors.Is(err, os.ErrNotExist) {
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("read ifindex of %s: %w", tap.Name, err)
+	}
+	alias, err := os.ReadFile(filepath.Join(ownedTapSysfs, tap.Name, "ifalias"))
+	if err != nil {
+		return fmt.Errorf("read alias of %s: %w", tap.Name, err)
+	}
+	if idx != tap.IfIndex || strings.TrimSpace(string(alias)) != tap.Owner {
+		return fmt.Errorf("link %s is not the tap this owner created; left in place", tap.Name)
+	}
+	if out, err := deleteTapLink(tap.Name).CombinedOutput(); err != nil {
+		return fmt.Errorf("delete tap %s: %s: %w", tap.Name, strings.TrimSpace(string(out)), err)
+	}
+	slog.Info("Owned tap removed", "tap", tap.Name)
+	return nil
+}
+
+func readIfIndex(name string) (int, error) {
+	b, err := os.ReadFile(filepath.Join(ownedTapSysfs, name, "ifindex"))
+	if err != nil {
+		return 0, err
+	}
+	return strconv.Atoi(strings.TrimSpace(string(b)))
+}

spinifex/domains/network/host/owned_tap_test.go

@@ -0,0 +1,164 @@
+//go:build lambdatestlink
+
+package host
+
+import (
+	"errors"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"slices"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	hostcommand "github.com/mulgadc/spinifex/spinifex/runtime/host/command"
+	"github.com/stretchr/testify/require"
+)
+
+// fakeHost stands in for sysfs and the privileged commands: ip creates and
+// deletes link directories, ovs-vsctl reports a port owner.
+type fakeHost struct {
+	sysfs     string
+	calls     []string
+	ovsOwner  string
+	failCmd   string
+	nextIndex string
+}
+
+func newFakeHost(t *testing.T) *fakeHost {
+	f := &fakeHost{sysfs: t.TempDir(), nextIndex: "41"}
+	orig := ownedTapSysfs
+	ownedTapSysfs = f.sysfs
+	t.Cleanup(func() { ownedTapSysfs = orig })
+	t.Cleanup(hostcommand.SetSudoCommandForTest(func(name string, args ...string) *exec.Cmd {
+		call := name + " " + strings.Join(args, " ")
+		f.calls = append(f.calls, call)
+		if f.failCmd != "" && strings.Contains(call, f.failCmd) {
+			return exec.Command("/bin/false")
+		}
+		switch {
+		case name == "ip" && args[0] == "tuntap":
+			f.link(t, args[3], f.nextIndex, "")
+		case name == "ip" && slices.Contains(args, "alias"):
+			i := slices.Index(args, "alias")
+			require.NoError(t, os.WriteFile(filepath.Join(f.sysfs, args[3], "ifalias"), []byte(args[i+1]+"\n"), 0o600))
+		case name == "ip" && args[0] == "link" && args[1] == "del":
+			require.NoError(t, os.RemoveAll(filepath.Join(f.sysfs, args[3])))
+		case name == "ovs-vsctl" && slices.Contains(args, "get"):
+			return exec.Command("/bin/echo", f.ovsOwner)
+		}
+		return exec.Command("/bin/true")
+	}))
+	return f
+}
+
+func (f *fakeHost) link(t *testing.T, name, index, alias string) {
+	dir := filepath.Join(f.sysfs, name)
+	require.NoError(t, os.MkdirAll(dir, 0o700))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "ifindex"), []byte(index+"\n"), 0o600))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "ifalias"), []byte(alias+"\n"), 0o600))
+}
+
+func (f *fakeHost) ran(sub string) bool {
+	return slices.ContainsFunc(f.calls, func(c string) bool { return strings.Contains(c, sub) })
+}
+
+func TestCreateOwnedTap_RefusesAnExistingLinkWithoutTouchingIt(t *testing.T) {
+	f := newFakeHost(t)
+	f.link(t, "lm1", "7", "someone-else")
+	_, err := NewOVSPlumber().CreateOwnedTap(vm.TapSpec{Name: "lm1", Bridge: "br-mgmt"}, "lambda-a")
+	require.ErrorIs(t, err, ErrTapExists)
+	require.ErrorIs(t, err, ErrTapNotCreated)
+	require.Empty(t, f.calls, "an existing link must not be deleted, replaced or detached")
+}
+
+func TestCreateOwnedTap_CreatesWithoutReplacing(t *testing.T) {
+	f := newFakeHost(t)
+	tap, err := NewOVSPlumber().CreateOwnedTap(vm.TapSpec{Name: "lm1", Bridge: "br-mgmt", MTU: 1400}, "lambda-a")
+	require.NoError(t, err)
+	require.Equal(t, OwnedTap{Name: "lm1", IfIndex: 41, Owner: "lambda-a"}, tap)
+	require.True(t, f.ran("add-port br-mgmt lm1 -- set Interface lm1 external_ids:spinifex-owner=lambda-a"))
+	require.True(t, f.ran("alias lambda-a up mtu 1400"))
+	require.False(t, f.ran("del-port"), "creation never removes an existing port")
+	require.False(t, f.ran("--may-exist"))
+	require.False(t, f.ran("link del"))
+}
+
+func TestCreateOwnedTap_OnlyARefusalClaimsNothingCreated(t *testing.T) {
+	f := newFakeHost(t)
+	_, err := NewOVSPlumber().CreateOwnedTap(vm.TapSpec{Name: "lm1", Bridge: "br-mgmt"}, "bad owner")
+	require.ErrorIs(t, err, ErrTapNotCreated)
+	require.Empty(t, f.calls)
+
+	f = newFakeHost(t)
+	f.failCmd = "tuntap add"
+	_, err = NewOVSPlumber().CreateOwnedTap(vm.TapSpec{Name: "lm1", Bridge: "br-mgmt"}, "lambda-a")
+	require.Error(t, err)
+	require.NotErrorIs(t, err, ErrTapNotCreated, "a failed add may still have created the link")
+
+	f = newFakeHost(t)
+	f.nextIndex = "not-a-number"
+	tap, err := NewOVSPlumber().CreateOwnedTap(vm.TapSpec{Name: "lm2", Bridge: "br-mgmt"}, "lambda-a")
+	require.Error(t, err)
+	require.NotErrorIs(t, err, ErrTapNotCreated, "a link exists that the caller cannot identify")
+	require.Empty(t, tap.Name)
+	require.False(t, f.ran("link del"))
+}
+
+func TestCreateOwnedTap_FailedAttachRemovesOnlyItsOwnLink(t *testing.T) {
+	f := newFakeHost(t)
+	f.failCmd = "add-port"
+	_, err := NewOVSPlumber().CreateOwnedTap(vm.TapSpec{Name: "lm1", Bridge: "br-mgmt"}, "lambda-a")
+	require.Error(t, err)
+	require.True(t, f.ran("link del dev lm1"))
+	require.False(t, f.ran("del-port"), "a port this call failed to add is never removed")
+}
+
+func TestRemoveOwnedTap_LeavesAnythingItDidNotCreate(t *testing.T) {
+	cases := map[string]struct {
+		index, alias, ovsOwner string
+		wantDelPort            bool
+	}{
+		"other OVS owner":  {index: "41", alias: "lambda-a", ovsOwner: `"other"`},
+		"replaced link":    {index: "99", alias: "lambda-a", ovsOwner: `"lambda-a"`, wantDelPort: true},
+		"other link alias": {index: "41", alias: "other", ovsOwner: ""},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			f := newFakeHost(t)
+			f.link(t, "lm1", tc.index, tc.alias)
+			f.ovsOwner = tc.ovsOwner
+			err := NewOVSPlumber().RemoveOwnedTap(OwnedTap{Name: "lm1", IfIndex: 41, Owner: "lambda-a"})
+			require.Error(t, err)
+			require.False(t, f.ran("link del"))
+			require.Equal(t, tc.wantDelPort, f.ran("del-port"))
+		})
+	}
+}
+
+func TestRemoveOwnedTap_RemovesItsOwnTap(t *testing.T) {
+	f := newFakeHost(t)
+	f.link(t, "lm1", "41", "lambda-a")
+	f.ovsOwner = `"lambda-a"`
+	require.NoError(t, NewOVSPlumber().RemoveOwnedTap(OwnedTap{Name: "lm1", IfIndex: 41, Owner: "lambda-a"}))
+	require.True(t, f.ran("del-port lm1"))
+	require.True(t, f.ran("link del dev lm1"))
+
+	// Already gone is success, and touches nothing.
+	f.calls, f.ovsOwner = nil, ""
+	require.NoError(t, NewOVSPlumber().RemoveOwnedTap(OwnedTap{Name: "lm1", IfIndex: 41, Owner: "lambda-a"}))
+	require.False(t, f.ran("del-port"))
+	require.False(t, f.ran("link del"))
+}
+
+func TestRemoveOwnedTap_OVSReadFailureRemovesNothing(t *testing.T) {
+	f := newFakeHost(t)
+	f.link(t, "lm1", "41", "lambda-a")
+	f.failCmd = "get Interface"
+	err := NewOVSPlumber().RemoveOwnedTap(OwnedTap{Name: "lm1", IfIndex: 41, Owner: "lambda-a"})
+	require.Error(t, err)
+	require.False(t, f.ran("del-port"))
+	require.False(t, f.ran("link del"))
+	require.False(t, errors.Is(err, ErrTapExists))
+}

spinifex/foundation/aws/errors/awserrors.go

@@ -625,8 +625,10 @@ var (
 	ErrorSerializationException            = "SerializationException"
 	ErrorUnknownOperationException         = "UnknownOperationException"
 	// Lambda wire codes that no other service shares.
-	ErrorResourceConflictException = "ResourceConflictException"
-	ErrorServiceException          = "ServiceException"
+	ErrorResourceConflictException      = "ResourceConflictException"
+	ErrorServiceException               = "ServiceException"
+	ErrorRequestEntityTooLargeException = "RequestEntityTooLargeException"
+	ErrorInvalidRequestContentException = "InvalidRequestContentException"
 )
 
 // ResolveErrorCode returns the first registered AWS error code in err's unwrap tree.
@@ -1426,6 +1428,8 @@ var ErrorLookup = map[string]ErrorMessage{
 	ErrorUnknownOperationException:         {HTTPCode: 400, Message: "The requested operation is not recognized."},
 	ErrorResourceConflictException:         {HTTPCode: 409, Message: "The resource already exists or is in a state that does not allow this operation."},
 	ErrorServiceException:                  {HTTPCode: 500, Message: "The Lambda service encountered an internal error."},
+	ErrorRequestEntityTooLargeException:    {HTTPCode: 413, Message: "The request payload exceeded the Invoke request body JSON input quota."},
+	ErrorInvalidRequestContentException:    {HTTPCode: 400, Message: "The request body could not be parsed as JSON."},
 
 	// Route 53.
 	ErrorNoSuchHostedZone:          {HTTPCode: 404, Message: "No hosted zone exists with the ID that you specified."},

spinifex/ingress/aws/pipeline/lambda_invoke_wire_test.go

@@ -0,0 +1,122 @@
+package pipeline
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/testkit/gwfix"
+	lambdaawsapi "github.com/mulgadc/spinifex/spinifex/domains/lambda/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// activeMeta serves one Active function "hello" with its code stored.
+type activeMeta struct{}
+
+var _ function.Client = activeMeta{}
+
+func (activeMeta) Get(_ context.Context, account, region, name string) (function.Record, uint64, error) {
+	if name != "hello" {
+		return function.Record{}, 0, function.ErrNotFound
+	}
+	return function.Record{
+		FunctionID: "f1", AccountID: account, FunctionName: name, Runtime: "provided.al2023",
+		Timeout: 3, MemorySize: 128, State: function.StateActive,
+		Code: function.Code{RevisionID: "r1", State: function.CodeStored},
+	}, 1, nil
+}
+
+func (activeMeta) Create(context.Context, string, string, function.Record) (uint64, error) {
+	return 0, errors.New("unused")
+}
+
+func (activeMeta) CodeStored(context.Context, string, string, string, uint64, string, int64) (function.Record, uint64, error) {
+	return function.Record{}, 0, errors.New("unused")
+}
+
+func (activeMeta) RollbackCreate(context.Context, string, string, string, uint64) error {
+	return errors.New("unused")
+}
+
+// countingExec records how many invocations reached it.
+type countingExec struct{ calls atomic.Int32 }
+
+func (e *countingExec) InvokeSync(context.Context, lambdaawsapi.SyncInvocation) (lambdaawsapi.SyncResult, error) {
+	e.calls.Add(1)
+	return lambdaawsapi.SyncResult{Payload: []byte(`{"ok":true}`), ExecutedVersion: "$LATEST"}, nil
+}
+
+func signedInvoke(t *testing.T, app http.Handler, body []byte) *http.Response {
+	t.Helper()
+	req := httptest.NewRequest(http.MethodPost, "/2015-03-31/functions/hello/invocations", bytes.NewReader(body))
+	req.Host = "localhost:9999"
+	gwfix.SignRequestAs(t, req, body, gwfix.AccessKey, gwfix.SecretKey, lambdaawsapi.ServiceName, time.Now().UTC())
+	return doRequest(app, req)
+}
+
+// paddedJSON is a valid JSON string exactly n bytes long.
+func paddedJSON(n int) []byte {
+	return []byte(`"` + strings.Repeat("a", n-2) + `"`)
+}
+
+// restJSONError decodes the Lambda REST-JSON error body: Type and message,
+// with the code carried only in X-Amzn-Errortype.
+func restJSONError(t *testing.T, resp *http.Response) map[string]string {
+	t.Helper()
+	var out map[string]string
+	require.NoError(t, json.NewDecoder(resp.Body).Decode(&out))
+	assert.Equal(t, "User", out["Type"])
+	assert.NotEmpty(t, out["message"])
+	return out
+}
+
+func lambdaInvokeApp(t *testing.T) (http.Handler, *countingExec) {
+	t.Helper()
+	ex := &countingExec{}
+	_, app := dispatchSeamApp(t, allowAllECRDocs(), lambdaawsapi.NewRegistration(lambdaawsapi.Deps{Meta: activeMeta{}, Exec: ex}))
+	return app, ex
+}
+
+func TestLambdaInvokeWire_PayloadSizeLimit(t *testing.T) {
+	app, ex := lambdaInvokeApp(t)
+
+	resp := signedInvoke(t, app, paddedJSON(6291456))
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Equal(t, int32(1), ex.calls.Load())
+
+	for _, size := range []int{6291457, 6291556, 7000000} {
+		resp := signedInvoke(t, app, paddedJSON(size))
+		require.Equal(t, http.StatusRequestEntityTooLarge, resp.StatusCode, "size %d", size)
+		assert.Equal(t, "RequestEntityTooLargeException", resp.Header.Get("X-Amzn-Errortype"), "size %d", size)
+		body := restJSONError(t, resp)
+		if size == 7000000 {
+			assert.Equal(t, "Request must be smaller than 6291456 bytes for the InvokeFunction operation", body["message"])
+		}
+	}
+	assert.Equal(t, int32(1), ex.calls.Load(), "an oversized payload reached the executor")
+}
+
+func TestLambdaInvokeWire_MalformedPayload(t *testing.T) {
+	app, ex := lambdaInvokeApp(t)
+
+	resp := signedInvoke(t, app, []byte("not json"))
+	require.Equal(t, http.StatusBadRequest, resp.StatusCode)
+	assert.Equal(t, "InvalidRequestContentException", resp.Header.Get("X-Amzn-Errortype"))
+	restJSONError(t, resp)
+	assert.Equal(t, int32(0), ex.calls.Load(), "a malformed payload reached the executor")
+
+	for _, good := range []string{`{"a":1}`, `[1]`, `"s"`, `7`, `null`} {
+		resp := signedInvoke(t, app, []byte(good))
+		assert.Equal(t, http.StatusOK, resp.StatusCode, "payload %s", good)
+	}
+	assert.Equal(t, int32(5), ex.calls.Load())
+}

spinifex/runtime/compute/lambda/attempt/attempt.go

@@ -0,0 +1,218 @@
+// Package attempt runs one Lambda guest VM for one invocation inside the
+// daemon: the daemon owns its deadline, and the kernel kills the VM if the
+// daemon dies first.
+package attempt
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"runtime"
+	"syscall"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+// StopBound is how long Run waits for a killed VM to be reaped before it
+// reports the stop unproven.
+const StopBound = 10 * time.Second
+
+var (
+	// ErrStopUnproven means there is no evidence the VM exited. Anything the
+	// attempt holds must be kept until Outcome.Reaped closes, which may be never.
+	ErrStopUnproven = errors.New("lambda attempt: VM stop not proven")
+	// ErrTimedOut means the deadline passed before the session returned.
+	ErrTimedOut = errors.New("lambda attempt: deadline passed")
+	// ErrVMExited means the VM exited before the session returned.
+	ErrVMExited = errors.New("lambda attempt: VM exited before the session returned")
+	// ErrNoPidfd means the kernel gave no pidfd, so the VM is never signalled.
+	ErrNoPidfd = errors.New("lambda attempt: no pidfd for the VM")
+)
+
+// waitVM waits for the VM; tests replace it to simulate a failed wait.
+var waitVM = (*exec.Cmd).Wait
+
+// pidfdSupported reports whether pidfds can be opened and signalled; tests
+// replace it to simulate a kernel or seccomp policy without them.
+var pidfdSupported = func() bool {
+	fd, err := unix.PidfdOpen(os.Getpid(), 0)
+	if err != nil {
+		return false
+	}
+	defer func() { _ = unix.Close(fd) }()
+	return unix.PidfdSendSignal(fd, 0, nil, 0) == nil
+}
+
+// Outcome reports how the VM ended.
+type Outcome struct {
+	// TimedOut is true when the deadline passed before the attempt ended.
+	TimedOut bool
+	// Stopped is true only once the process is known to have exited.
+	Stopped bool
+	// Reaped closes when the process is known to have exited, possibly after
+	// Run returns; it stays open if the wait failed.
+	Reaped <-chan struct{}
+	// SessionDone closes once the session callback has returned, or at once if
+	// it never ran; Run can return while the callback still runs.
+	SessionDone <-chan struct{}
+}
+
+// Run starts cmd, runs session with the VM's clone-time pidfd while the VM lives,
+// then kills and reaps it. It returns nil only when the session returned nil before
+// the deadline, the VM exit and any caller cancel, and the stop was then proven.
+func Run(ctx context.Context, cmd *exec.Cmd, timeout time.Duration, session func(ctx context.Context, pidfd int) error) (Outcome, error) {
+	if err := ctx.Err(); err != nil {
+		return Outcome{Stopped: true, Reaped: closedChan(), SessionDone: closedChan()}, err
+	}
+	if !pidfdSupported() {
+		// Without a stable handle the VM could only be signalled by PID.
+		return Outcome{Stopped: true, Reaped: closedChan(), SessionDone: closedChan()}, ErrNoPidfd
+	}
+	if cmd.SysProcAttr == nil {
+		cmd.SysProcAttr = &syscall.SysProcAttr{}
+	}
+	// The parent-death signal follows the thread that forked the child, so the
+	// launching goroutine stays locked to its thread until the VM is reaped.
+	cmd.SysProcAttr.Pdeathsig = syscall.SIGKILL
+	// A pidfd taken at clone is the only handle signalled, so a kill can never
+	// reach a reused PID, even if something else reaps the VM.
+	pidfd := -1
+	cmd.SysProcAttr.PidFD = &pidfd
+
+	started := make(chan error, 1)
+	reaped := make(chan struct{})
+	waitFailed := make(chan error, 1)
+	wait := func() {
+		err := waitVM(cmd)
+		if cmd.ProcessState == nil {
+			waitFailed <- err
+			return
+		}
+		close(reaped)
+	}
+	go func() {
+		runtime.LockOSThread() // never unlocked: the thread ends with this goroutine
+		if err := cmd.Start(); err != nil {
+			started <- err
+			return
+		}
+		if pidfd < 0 {
+			// Best effort, unproven: ending this thread delivers the parent-death
+			// signal unless Go parks it as the main thread. Without exit evidence
+			// the attempt is held.
+			started <- ErrNoPidfd
+			return
+		}
+		started <- nil
+		wait()
+	}()
+	if err := <-started; errors.Is(err, ErrNoPidfd) {
+		go wait()
+		out, stopErr := awaitStop(Outcome{Reaped: reaped, SessionDone: closedChan()}, reaped, waitFailed)
+		if stopErr != nil {
+			return out, stopErr
+		}
+		return out, err
+	} else if err != nil {
+		// A failed Start leaves no child: os/exec reaps a fork whose exec failed.
+		return Outcome{Stopped: true, Reaped: closedChan(), SessionDone: closedChan()}, fmt.Errorf("lambda attempt: start VM: %w", err)
+	}
+	defer func() { _ = unix.Close(pidfd) }()
+
+	if err := ctx.Err(); err != nil {
+		// Cancelled while the VM started: stop it before any session runs.
+		return stop(Outcome{Reaped: reaped, SessionDone: closedChan()}, pidfd, reaped, waitFailed, err)
+	}
+
+	deadline := time.Now().Add(timeout)
+	sessCtx, cancel := context.WithDeadline(ctx, deadline)
+	defer cancel()
+	// The callback gets its own duplicate, so a callback outliving Run never
+	// sees this descriptor's number reused.
+	sessFD, err := unix.FcntlInt(uintptr(pidfd), unix.F_DUPFD_CLOEXEC, 0)
+	if err != nil {
+		return stop(Outcome{Reaped: reaped, SessionDone: closedChan()}, pidfd, reaped, waitFailed,
+			fmt.Errorf("lambda attempt: duplicate pidfd: %w", err))
+	}
+	sessDone := make(chan error, 1)
+	sessFinished := make(chan struct{})
+	go func() {
+		// Released before the result is published, so a caller holding the
+		// result always sees SessionDone closed.
+		err := session(sessCtx, sessFD)
+		_ = unix.Close(sessFD)
+		close(sessFinished)
+		sessDone <- err
+	}()
+
+	out := Outcome{Reaped: reaped, SessionDone: sessFinished}
+	var result error
+	sessReturned := false
+	select {
+	case result = <-sessDone:
+		sessReturned = true
+	case <-reaped:
+		result = ErrVMExited
+	case err := <-waitFailed:
+		// The VM may already be someone else's to reap: signal nothing.
+		return out, fmt.Errorf("%w: wait: %w", ErrStopUnproven, err)
+	case <-sessCtx.Done():
+		result = ctx.Err()
+	}
+	// Classified by the monotonic clock and the caller's own context, not by
+	// which case won, so a result racing the deadline or a cancel never succeeds.
+	if !time.Now().Before(deadline) {
+		out.TimedOut = true
+		result = ErrTimedOut
+	} else if err := ctx.Err(); err != nil {
+		result = err
+	}
+	cancel()
+
+	out, err = stop(out, pidfd, reaped, waitFailed, result)
+	if errors.Is(err, ErrStopUnproven) {
+		return out, err
+	}
+	if !sessReturned {
+		select {
+		case <-sessDone:
+		case <-time.After(StopBound):
+		}
+	}
+	return out, result
+}
+
+// stop kills the VM through its pidfd and returns result once it is proven
+// stopped, or ErrStopUnproven.
+func stop(out Outcome, pidfd int, reaped <-chan struct{}, waitFailed <-chan error, result error) (Outcome, error) {
+	if err := unix.PidfdSendSignal(pidfd, unix.SIGKILL, nil, 0); err != nil && !errors.Is(err, unix.ESRCH) {
+		return out, fmt.Errorf("%w: kill: %w", ErrStopUnproven, err)
+	}
+	out, err := awaitStop(out, reaped, waitFailed)
+	if err != nil {
+		return out, err
+	}
+	return out, result
+}
+
+// awaitStop waits up to StopBound for evidence that the VM exited.
+func awaitStop(out Outcome, reaped <-chan struct{}, waitFailed <-chan error) (Outcome, error) {
+	select {
+	case <-reaped:
+		out.Stopped = true
+		return out, nil
+	case err := <-waitFailed:
+		return out, fmt.Errorf("%w: wait: %w", ErrStopUnproven, err)
+	case <-time.After(StopBound):
+		return out, ErrStopUnproven
+	}
+}
+
+func closedChan() <-chan struct{} {
+	c := make(chan struct{})
+	close(c)
+	return c
+}

spinifex/runtime/compute/lambda/attempt/attempt_test.go

@@ -0,0 +1,305 @@
+package attempt
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"runtime"
+	"strconv"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
+)
+
+// The test binary doubles as the fake VM and as a fake daemon, chosen by env.
+const helperEnv = "SPX_ATTEMPT_HELPER"
+
+func TestMain(m *testing.M) {
+	switch os.Getenv(helperEnv) {
+	case "hang":
+		select {}
+	case "exit":
+		os.Exit(0)
+	case "daemon":
+		os.Exit(fakeDaemon())
+	}
+	os.Exit(m.Run())
+}
+
+func helperCmd(mode string) *exec.Cmd {
+	cmd := exec.Command(os.Args[0], "-test.run=^$")
+	cmd.Env = append(os.Environ(), helperEnv+"="+mode)
+	return cmd
+}
+
+// fakeDaemon runs one attempt whose VM never exits and prints the VM's PID.
+func fakeDaemon() int {
+	vm := helperCmd("hang")
+	_, _ = Run(context.Background(), vm, time.Hour, func(ctx context.Context, _ int) error {
+		fmt.Println(vm.Process.Pid)
+		<-ctx.Done()
+		return nil
+	})
+	return 0
+}
+
+// pidfdOpen takes a handle on pid while its parent is known to be alive and
+// not reaping it, so the PID cannot have been reused yet.
+func pidfdOpen(t *testing.T, pid int) int {
+	t.Helper()
+	fd, err := unix.PidfdOpen(pid, 0)
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		_ = unix.PidfdSendSignal(fd, unix.SIGKILL, nil, 0)
+		_ = unix.Close(fd)
+	})
+	return fd
+}
+
+// exitedWithin reports whether the process behind fd exits within d; a pidfd
+// becomes readable when its process exits.
+func exitedWithin(t *testing.T, fd int, d time.Duration) bool {
+	t.Helper()
+	fds := []unix.PollFd{{Fd: int32(fd), Events: unix.POLLIN}}
+	n, err := unix.Poll(fds, int(d.Milliseconds()))
+	require.NoError(t, err)
+	return n == 1
+}
+
+func TestRun_DeadlineKillsAndReapsTheVM(t *testing.T) {
+	vm := helperCmd("hang")
+	sawDeadline := false
+	out, err := Run(t.Context(), vm, 200*time.Millisecond, func(ctx context.Context, _ int) error {
+		<-ctx.Done()
+		sawDeadline = errors.Is(ctx.Err(), context.DeadlineExceeded)
+		return nil
+	})
+	require.ErrorIs(t, err, ErrTimedOut, "a session returning nil at the deadline is not a result")
+	assert.True(t, out.TimedOut)
+	assert.True(t, out.Stopped)
+	assert.True(t, sawDeadline, "the session's context ends at the deadline")
+	require.NotNil(t, vm.ProcessState, "the VM was reaped")
+}
+
+func TestRun_SessionEndStopsTheVM(t *testing.T) {
+	vm := helperCmd("hang")
+	out, err := Run(t.Context(), vm, time.Minute, func(context.Context, int) error { return nil })
+	require.NoError(t, err)
+	assert.False(t, out.TimedOut)
+	assert.True(t, out.Stopped)
+	require.NotNil(t, vm.ProcessState)
+}
+
+func TestRun_SessionErrorIsReturnedAfterTheStop(t *testing.T) {
+	boom := errors.New("guest refused")
+	vm := helperCmd("hang")
+	out, err := Run(t.Context(), vm, time.Minute, func(context.Context, int) error { return boom })
+	require.ErrorIs(t, err, boom)
+	assert.True(t, out.Stopped)
+	require.NotNil(t, vm.ProcessState)
+}
+
+func TestRun_VMExitBeforeAResultIsAFailure(t *testing.T) {
+	vm := helperCmd("exit")
+	out, err := Run(t.Context(), vm, time.Minute, func(ctx context.Context, _ int) error {
+		<-ctx.Done()
+		return nil
+	})
+	require.ErrorIs(t, err, ErrVMExited, "a session returning nil once cancelled is not a result")
+	assert.False(t, out.TimedOut)
+	assert.True(t, out.Stopped)
+}
+
+func TestRun_CallerCancelIsNotATimeout(t *testing.T) {
+	ctx, cancel := context.WithCancel(t.Context())
+	vm := helperCmd("hang")
+	out, err := Run(ctx, vm, time.Minute, func(ctx context.Context, _ int) error {
+		cancel()
+		<-ctx.Done()
+		return nil
+	})
+	require.ErrorIs(t, err, context.Canceled)
+	assert.False(t, out.TimedOut)
+	assert.True(t, out.Stopped)
+}
+
+func TestRun_StartFailureNeverRunsTheSession(t *testing.T) {
+	called := false
+	out, err := Run(t.Context(), exec.Command("/nonexistent/qemu"), time.Minute, func(context.Context, int) error {
+		called = true
+		return nil
+	})
+	require.Error(t, err)
+	assert.True(t, out.Stopped, "a failed start leaves no process")
+	assert.False(t, called)
+}
+
+func TestRun_SetsTheParentDeathSignal(t *testing.T) {
+	vm := helperCmd("hang")
+	vm.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
+	_, err := Run(t.Context(), vm, time.Minute, func(context.Context, int) error { return nil })
+	require.NoError(t, err)
+	assert.Equal(t, syscall.SIGKILL, vm.SysProcAttr.Pdeathsig)
+	require.NotNil(t, vm.SysProcAttr.PidFD, "the VM is signalled only through its pidfd")
+	assert.True(t, vm.SysProcAttr.Setpgid, "caller attributes are kept")
+}
+
+// startFakeDaemon starts a fake daemon process and returns it with a handle on its VM.
+func startFakeDaemon(t *testing.T) (*exec.Cmd, int) {
+	t.Helper()
+	daemon := helperCmd("daemon")
+	stdout, err := daemon.StdoutPipe()
+	require.NoError(t, err)
+	require.NoError(t, daemon.Start())
+	t.Cleanup(func() {
+		_ = daemon.Process.Kill()
+		_ = daemon.Wait()
+	})
+	lines := bufio.NewScanner(stdout)
+	require.True(t, lines.Scan(), "fake daemon printed no VM PID")
+	pid, err := strconv.Atoi(lines.Text())
+	require.NoError(t, err)
+	fd := pidfdOpen(t, pid)
+	require.False(t, exitedWithin(t, fd, 0), "the VM runs while the daemon lives")
+	return daemon, fd
+}
+
+func TestRun_DaemonDeathKillsTheVM(t *testing.T) {
+	daemon, vm := startFakeDaemon(t)
+	require.NoError(t, daemon.Process.Signal(syscall.SIGKILL))
+	_ = daemon.Wait()
+	assert.True(t, exitedWithin(t, vm, 5*time.Second), "the kernel killed the VM with its daemon")
+}
+
+// A result handed over after the deadline, before the deadline's timer has
+// run, is still a timeout. One P makes the session hold it past the deadline.
+func TestRun_ResultAfterTheDeadlineIsATimeout(t *testing.T) {
+	defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(1))
+	vm := helperCmd("hang")
+	out, err := Run(t.Context(), vm, time.Millisecond, func(context.Context, int) error {
+		for start := time.Now(); time.Since(start) < 5*time.Millisecond; {
+		}
+		return nil
+	})
+	require.ErrorIs(t, err, ErrTimedOut)
+	assert.True(t, out.TimedOut)
+	assert.True(t, out.Stopped)
+}
+
+func TestRun_FailedWaitLeavesTheStopUnproven(t *testing.T) {
+	waitVM = func(*exec.Cmd) error { return errors.New("wait failed") }
+	t.Cleanup(func() { waitVM = (*exec.Cmd).Wait })
+	vm := helperCmd("hang")
+	out, err := Run(t.Context(), vm, time.Minute, func(context.Context, int) error { return nil })
+	require.ErrorIs(t, err, ErrStopUnproven)
+	assert.False(t, out.Stopped)
+	select {
+	case <-out.Reaped:
+		t.Fatal("Reaped closed without exit evidence")
+	default:
+	}
+	_ = vm.Wait()
+}
+
+// cancelOnly is a caller context whose cancellation reaches derived contexts
+// only through a goroutine, so a session result can win the race against it.
+type cancelOnly struct {
+	context.Context
+
+	done chan struct{}
+}
+
+func (c *cancelOnly) Done() <-chan struct{} { return c.done }
+
+func (c *cancelOnly) Err() error {
+	select {
+	case <-c.done:
+		return context.Canceled
+	default:
+		return nil
+	}
+}
+
+// A nil result handed over after the caller cancelled is not a success, even
+// when Run sees the result before the cancellation reaches the session.
+func TestRun_ResultRacingACallerCancelIsNotASuccess(t *testing.T) {
+	defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(1))
+	ctx := &cancelOnly{Context: t.Context(), done: make(chan struct{})}
+	vm := helperCmd("hang")
+	out, err := Run(ctx, vm, time.Minute, func(context.Context, int) error {
+		close(ctx.done)
+		return nil
+	})
+	require.ErrorIs(t, err, context.Canceled)
+	assert.False(t, out.TimedOut)
+	assert.True(t, out.Stopped)
+}
+
+func TestRun_RefusesToLaunchWithoutPidfds(t *testing.T) {
+	orig := pidfdSupported
+	pidfdSupported = func() bool { return false }
+	t.Cleanup(func() { pidfdSupported = orig })
+	called := false
+	vm := helperCmd("hang")
+	out, err := Run(t.Context(), vm, time.Minute, func(context.Context, int) error {
+		called = true
+		return nil
+	})
+	require.ErrorIs(t, err, ErrNoPidfd)
+	assert.True(t, out.Stopped)
+	assert.Nil(t, vm.Process, "QEMU was never started")
+	assert.False(t, called)
+}
+
+func TestRun_CancelledBeforeLaunchStartsNothing(t *testing.T) {
+	ctx, cancel := context.WithCancel(t.Context())
+	cancel()
+	called := false
+	vm := helperCmd("hang")
+	out, err := Run(ctx, vm, time.Minute, func(context.Context, int) error {
+		called = true
+		return nil
+	})
+	require.ErrorIs(t, err, context.Canceled)
+	assert.True(t, out.Stopped)
+	assert.Nil(t, vm.Process, "QEMU was never started")
+	assert.False(t, called)
+}
+
+// cancelOnStart cancels the caller as soon as the VM has been created.
+type cancelOnStart struct {
+	context.Context
+
+	cancel func()
+	calls  int
+}
+
+func (c *cancelOnStart) Err() error {
+	c.calls++
+	if c.calls == 2 {
+		c.cancel()
+	}
+	return c.Context.Err()
+}
+
+func TestRun_CancelledDuringStartStopsTheVMWithoutASession(t *testing.T) {
+	inner, cancel := context.WithCancel(t.Context())
+	ctx := &cancelOnStart{Context: inner, cancel: cancel}
+	called := false
+	vm := helperCmd("hang")
+	out, err := Run(ctx, vm, time.Minute, func(context.Context, int) error {
+		called = true
+		return nil
+	})
+	require.ErrorIs(t, err, context.Canceled)
+	assert.True(t, out.Stopped)
+	require.NotNil(t, vm.ProcessState, "the started VM was stopped and reaped")
+	assert.False(t, called)
+}

spinifex/runtime/compute/lambda/testlink/contract.go

@@ -0,0 +1,70 @@
+//go:build lambdatestlink
+
+// Package testlink runs one controlled-test Lambda invocation in a fresh guest
+// VM on the gateway's node. It exists only in lambdatestlink builds and is not
+// the production executor. The guest checks the grant it is given but cannot
+// verify that the grant's owner holds authority.
+package testlink
+
+import (
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/execution"
+)
+
+// Subject is the request subject for one node's test-link daemon.
+func Subject(node string) string { return "spinifex.lambda.testlink.invoke." + node }
+
+// Outcomes a daemon reply can carry.
+const (
+	// OutcomeDone means the guest returned a result and the VM stop is proven.
+	OutcomeDone = "done"
+	// OutcomeRefused means the daemon launched nothing for this request.
+	OutcomeRefused = "refused"
+	// OutcomeUnknown means a VM may have run; nothing about the result holds.
+	OutcomeUnknown = "unknown"
+)
+
+// Dev-only bounds for the test path; they are not AWS limits.
+const (
+	MaxCodeBytes = 16 << 20
+	// BootAllowance is added to the function timeout to bound the whole VM run.
+	BootAllowance = 15 * time.Second
+	CodeURLTTL    = 2 * time.Minute
+	// LaunchWindow is how long a grant may wait for the daemon to start it.
+	LaunchWindow = 30 * time.Second
+)
+
+// Request asks the daemon to run one invocation. CodeURL is a short-lived
+// download link for the exact digest; it must never be logged or stored.
+type Request struct {
+	Grant       execution.Grant `json:"grant"`
+	AttemptID   string          `json:"attempt_id"`
+	RequestID   string          `json:"request_id"`
+	FunctionARN string          `json:"function_arn"`
+	Handler     string          `json:"handler"`
+	Region      string          `json:"region"`
+	AccountID   string          `json:"account_id"`
+	FunctionID  string          `json:"function_id"`
+	RevisionID  string          `json:"revision_id"`
+	TimeoutMs   int64           `json:"timeout_ms"`
+	MemoryMiB   int64           `json:"memory_mib"`
+	CodeURL     string          `json:"code_url"`
+	CodeSize    int64           `json:"code_size"`
+	CodeSHA256  string          `json:"code_sha256"`
+	Payload     []byte          `json:"payload"`
+}
+
+// Reply is the daemon's answer. Error is a reason safe to log.
+type Reply struct {
+	Outcome       string `json:"outcome"`
+	Payload       []byte `json:"payload,omitempty"`
+	FunctionError string `json:"function_error,omitempty"`
+	Error         string `json:"error,omitempty"`
+}
+
+// Identity is the request and code identity the grant must authorise.
+func (r Request) Identity() execution.Identity {
+	return execution.Identity{RequestID: r.RequestID, AccountID: r.AccountID, Region: r.Region,
+		FunctionID: r.FunctionID, RevisionID: r.RevisionID, CodeSHA256: r.CodeSHA256}
+}

spinifex/runtime/compute/lambda/testlink/daemon.go

@@ -0,0 +1,662 @@
+//go:build lambdatestlink
+
+package testlink
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"time"
+	"unicode"
+
+	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/execution"
+	"github.com/mulgadc/spinifex/spinifex/domains/network/host"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/attempt"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/guestconn"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/nats-io/nats.go"
+)
+
+const (
+	fetchTimeout     = 60 * time.Second
+	handshakeTimeout = 10 * time.Second
+	maxTimeoutMs     = 900_000
+	fwCfgNetcfg      = "opt/spinifex/netcfg"
+	fwCfgPackage     = lambdav1.FwCfgPrefix + "test-package-"
+)
+
+const markerSuffix = ".attempt"
+
+var attemptIDPattern = regexp.MustCompile(`^[0-9a-f]{12}$`)
+
+// Taps creates the guest's tap without replacing anything, and removes only
+// a tap it created.
+type Taps interface {
+	CreateOwnedTap(spec vm.TapSpec, owner string) (host.OwnedTap, error)
+	RemoveOwnedTap(tap host.OwnedTap) error
+}
+
+// Grants is the execution owner's side of an attempt the daemon runs.
+type Grants interface {
+	Start(ctx context.Context, g execution.Grant, want execution.Identity) error
+	Launched(ctx context.Context, g execution.Grant, l execution.Launch) error
+	Accept(ctx context.Context, g execution.Grant) error
+	Released(ctx context.Context, g execution.Grant) error
+	Stopped(ctx context.Context, g execution.Grant) error
+	Ambiguous(ctx context.Context, g execution.Grant) error
+	Current(ctx context.Context, node string) (execution.Attempt, error)
+}
+
+var _ Grants = (*execution.Store)(nil)
+
+// IPs hands out and takes back management-bridge addresses.
+type IPs interface {
+	Allocate(id string) (string, error)
+	ReleaseChecked(id string) error
+}
+
+// Daemon runs one test-link invocation at a time on this node.
+type Daemon struct {
+	Node    string
+	Grants  Grants
+	Taps    Taps
+	IPs     IPs
+	Bridge  string
+	HostIP  string
+	MTU     int
+	WorkDir string
+	HTTP    *http.Client
+	// Image is the prepared runtime image every launch boots; the daemon
+	// prepares it once before serving and keeps it open for its whole life.
+	Image *Image
+	// Initializer is the hard-kill library preloaded into QEMU, and
+	// InitializerSHA256 the digest it must have.
+	Initializer       string
+	InitializerSHA256 string
+
+	busy atomic.Bool
+}
+
+// unresolved names an attempt whose marker is still on disk. A marker goes
+// only once the attempt's VM stop is proven and everything it held is
+// released, so it outlives a daemon restart; nothing resolves it on its own.
+func (d *Daemon) unresolved() (string, error) {
+	markers, err := filepath.Glob(filepath.Join(d.WorkDir, "*"+markerSuffix))
+	if err != nil {
+		return "", err
+	}
+	if len(markers) == 0 {
+		return "", nil
+	}
+	return strings.TrimSuffix(filepath.Base(markers[0]), markerSuffix), nil
+}
+
+// Serve answers requests on this node's subject until the subscription ends.
+func (d *Daemon) Serve(ctx context.Context, nc *nats.Conn, node string) (*nats.Subscription, error) {
+	return nc.Subscribe(Subject(node), func(m *nats.Msg) {
+		go func() {
+			rep := d.Handle(ctx, m.Data)
+			body, err := json.Marshal(rep)
+			if err == nil && int64(len(body)) > nc.MaxPayload() {
+				body, err = json.Marshal(Reply{Outcome: OutcomeUnknown, Error: "result too large for the test path"})
+			}
+			if err != nil {
+				slog.Error("Lambda test link: encode reply", "err", err)
+				return
+			}
+			if err := m.Respond(body); err != nil {
+				slog.Error("Lambda test link: send reply", "err", err)
+			}
+		}()
+	})
+}
+
+// Handle runs one request under its grant. It refuses before launch when it
+// can; after launch it reports done only with a result the owner accepted and
+// a proven VM stop.
+func (d *Daemon) Handle(ctx context.Context, data []byte) Reply {
+	var req Request
+	if err := json.Unmarshal(data, &req); err != nil {
+		return refused("unreadable request")
+	}
+	if err := CheckRequest(req); err != nil {
+		return refused(err.Error())
+	}
+	if req.Grant.Node != d.Node {
+		return refused("grant names another node")
+	}
+	if d.Image == nil {
+		return refused("runtime image not prepared")
+	}
+	if !d.busy.CompareAndSwap(false, true) {
+		return refused("another test-link attempt holds this node's slot")
+	}
+	// The slot stays taken while any attempt is unresolved.
+	held := false
+	defer func() {
+		if !held {
+			d.busy.Store(false)
+		}
+	}()
+	if id, err := d.unresolved(); err != nil || id != "" {
+		return refused("an unresolved test-link attempt holds this node's slot")
+	}
+	// Without pidfs a restarted daemon could not identify the VM it left.
+	if _, err := selfInode(); err != nil {
+		return refused(err.Error())
+	}
+	lib, err := d.sealInitializer()
+	if err != nil {
+		return refused(err.Error())
+	}
+	defer func() { _ = lib.Close() }()
+	// One absolute boottime deadline, fixed before the grant starts and never
+	// extended; QEMU's kill timer and every dispatch check use it.
+	boot, err := bootNow()
+	if err != nil {
+		return refused(err.Error())
+	}
+	hardDeadline := hardDeadlineFrom(boot, req)
+	rep, free := d.attempt(ctx, req, func(code []byte) (Reply, bool, string) {
+		return d.run(ctx, req, code, lib, hardDeadline)
+	})
+	held = !free
+	return rep
+}
+
+// attempt starts the grant, fetches the code, runs it with launch and tells
+// the owner how it ended. It reports whether the node is free afterwards.
+func (d *Daemon) attempt(ctx context.Context, req Request, launch func(code []byte) (Reply, bool, string)) (Reply, bool) {
+	// The owner refuses a stale, replayed or mismatched grant before anything
+	// is taken; a write it cannot confirm leaves the attempt possibly started.
+	if err := d.Grants.Start(ctx, req.Grant, req.Identity()); err != nil {
+		if errors.Is(err, execution.ErrOutcomeUnknown) {
+			slog.Error("Lambda test link: grant start outcome unknown; node held", "attempt", req.AttemptID, "err", err)
+			return Reply{Outcome: OutcomeUnknown, Error: "grant start outcome unknown"}, true
+		}
+		return refused(err.Error()), true
+	}
+
+	var rep Reply
+	resolved := true
+	marker := ""
+	if code, err := d.fetch(ctx, req); err != nil {
+		rep = refused(err.Error())
+	} else {
+		rep, resolved, marker = launch(code)
+	}
+	return rep, d.finish(ctx, req.Grant, resolved, marker)
+}
+
+// finish tells the owner how the attempt ended, ambiguous unless the VM stop
+// is proven and everything released. It reports whether the node is free.
+func (d *Daemon) finish(ctx context.Context, g execution.Grant, resolved bool, marker string) bool {
+	if !resolved {
+		if err := d.Grants.Ambiguous(ctx, g); err != nil {
+			slog.Error("Lambda test link: ambiguous record outcome unknown", "attempt", g.AttemptID, "err", err)
+		}
+		return false
+	}
+	if err := d.Grants.Released(ctx, g); err != nil {
+		// The write may have landed; the stop was proven before it either way.
+		slog.Error("Lambda test link: release record outcome unknown; marker kept", "attempt", g.AttemptID, "err", err)
+		return false
+	}
+	return d.complete(ctx, g, marker)
+}
+
+// complete removes a released attempt's marker and only then frees the node,
+// so the owner keeps the node held for as long as the marker can remain.
+func (d *Daemon) complete(ctx context.Context, g execution.Grant, marker string) bool {
+	if !removeMarker(marker) {
+		return false
+	}
+	if err := d.Grants.Stopped(ctx, g); err != nil {
+		slog.Error("Lambda test link: stop record outcome unknown; node held until recovery", "attempt", g.AttemptID, "err", err)
+		return false
+	}
+	return true
+}
+
+// removeMarker removes an attempt marker; "" names none. Already gone is success.
+func removeMarker(marker string) bool {
+	if marker == "" {
+		return true
+	}
+	if err := os.Remove(marker); err != nil && !errors.Is(err, os.ErrNotExist) {
+		slog.Error("Lambda test link: remove attempt marker", "marker", marker, "err", err)
+		return false
+	}
+	return true
+}
+
+// CheckRequest is the daemon's validation of a test-link request.
+func CheckRequest(req Request) error {
+	if !attemptIDPattern.MatchString(req.AttemptID) {
+		return errors.New("attempt ID must be 12 lowercase hex digits")
+	}
+	if req.Grant.GrantID == "" || req.Grant.AttemptID != req.AttemptID {
+		return errors.New("request carries no grant for its attempt")
+	}
+	if _, err := lambdav1.ParseScope(scope(req)); err != nil || req.Grant.Generation == 0 ||
+		strings.ContainsAny(req.Grant.GrantID+req.Grant.EnvironmentID, "/\n") {
+		return errors.New("grant names no valid environment generation")
+	}
+	if req.RequestID == "" || req.RevisionID == "" {
+		return errors.New("request and revision IDs are required")
+	}
+	if req.Handler == "" || len(req.Handler) > 128 || strings.IndexFunc(req.Handler, unicode.IsSpace) >= 0 ||
+		strings.ContainsRune(req.Handler, 0) {
+		return errors.New("handler must be 1..128 characters with no whitespace")
+	}
+	if req.TimeoutMs <= 0 || req.TimeoutMs > maxTimeoutMs {
+		return fmt.Errorf("timeout %d ms outside 1..%d", req.TimeoutMs, maxTimeoutMs)
+	}
+	if req.CodeSize <= 0 || req.CodeSize > MaxCodeBytes {
+		return fmt.Errorf("code size %d outside 1..%d bytes", req.CodeSize, MaxCodeBytes)
+	}
+	if b, err := hex.DecodeString(req.CodeSHA256); err != nil || len(b) != sha256.Size {
+		return errors.New("code digest is not 64 hex digits")
+	}
+	if _, err := lambdav1.ParseScope(scope(req)); err != nil {
+		return errors.New("region, account or function ID is not a valid scope segment")
+	}
+	u, err := url.Parse(req.CodeURL)
+	if err != nil || u.Scheme != "https" || u.Host == "" {
+		return errors.New("code URL must be HTTPS")
+	}
+	return nil
+}
+
+// scope is the environment generation the owner granted; the guest and host
+// certificates carry it, so the guest authenticates the grant's environment.
+func scope(req Request) string {
+	return req.Region + "/" + req.AccountID + "/" + req.FunctionID + "/" +
+		req.Grant.EnvironmentID + "/" + strconv.FormatUint(req.Grant.Generation, 10)
+}
+
+// grantEntry is the test-grant fw_cfg value the guest binds its one run to:
+// grant ID, attempt ID, environment ID and generation, one per line.
+func grantEntry(g execution.Grant) []byte {
+	return fmt.Appendf(nil, "%s\n%s\n%s\n%d\n", g.GrantID, g.AttemptID, g.EnvironmentID, g.Generation)
+}
+
+// fetch downloads the code copy over verified TLS and checks its exact size
+// and digest. Errors never carry the URL.
+func (d *Daemon) fetch(ctx context.Context, req Request) ([]byte, error) {
+	fctx, cancel := context.WithTimeout(ctx, fetchTimeout)
+	defer cancel()
+	hreq, err := http.NewRequestWithContext(fctx, http.MethodGet, req.CodeURL, nil)
+	if err != nil {
+		return nil, errors.New("code download: bad request")
+	}
+	client := d.HTTP
+	if client == nil {
+		client = http.DefaultClient
+	}
+	// Transport errors are dropped: a redirect's Location can echo the
+	// presigned URL inside them.
+	resp, err := client.Do(hreq)
+	if err != nil {
+		return nil, errors.New("code download: request failed")
+	}
+	defer func() { _ = resp.Body.Close() }()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("code download: status %d", resp.StatusCode)
+	}
+	code, err := io.ReadAll(io.LimitReader(resp.Body, req.CodeSize+1))
+	if err != nil {
+		return nil, errors.New("code download: read failed")
+	}
+	if int64(len(code)) != req.CodeSize {
+		return nil, fmt.Errorf("code download: got %d bytes, want %d", len(code), req.CodeSize)
+	}
+	sum := sha256.Sum256(code)
+	if hex.EncodeToString(sum[:]) != req.CodeSHA256 {
+		return nil, errors.New("code download: digest mismatch")
+	}
+	return code, nil
+}
+
+// run holds the marker, address, tap and files from before allocation until
+// the VM stop is proven, and keeps them all when it is not. resolved is true
+// only once everything is released.
+func (d *Daemon) run(ctx context.Context, req Request, code []byte, lib *os.File, hardDeadline time.Duration) (rep Reply, resolved bool, marker string) {
+	a := &held{id: "lambda-" + req.AttemptID, dir: filepath.Join(d.WorkDir, req.AttemptID),
+		marker: filepath.Join(d.WorkDir, req.AttemptID+markerSuffix)}
+	f, err := os.OpenFile(a.marker, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
+	if err != nil {
+		return refused("attempt marker unavailable"), true, ""
+	}
+	_ = f.Close()
+	marker = a.marker
+
+	ip, err := d.IPs.Allocate(a.id)
+	if err != nil {
+		// A failed allocation may still have written the address, so it stays held.
+		slog.Error("Lambda test link: address allocation outcome unknown; attempt held", "attempt", req.AttemptID, "err", err)
+		return refused("management address unavailable"), false, marker
+	}
+	a.ip = true
+	tapName := "lm" + req.AttemptID
+	a.tap, err = d.Taps.CreateOwnedTap(vm.TapSpec{Name: tapName, Bridge: d.Bridge, MTU: d.MTU}, a.id)
+	if err != nil {
+		slog.Error("Lambda test link: guest tap", "attempt", req.AttemptID, "err", err)
+		if a.tap.Name == "" && !errors.Is(err, host.ErrTapNotCreated) {
+			// A link may exist that this attempt cannot identify, so it stays held.
+			return refused("guest tap setup failed"), false, marker
+		}
+		return refused("guest tap setup failed"), d.release(a), marker
+	}
+	mac := vm.GenerateMgmtMAC(a.id)
+	cfg, gen, ln, err := d.prepare(req, code, a.dir, tapName, mac, ip)
+	if err != nil {
+		slog.Error("Lambda test link: prepare attempt", "attempt", req.AttemptID, "err", err)
+		return refused("attempt preparation failed"), d.release(a), marker
+	}
+	defer func() { _ = ln.Close() }()
+
+	cmd, err := cfg.Execute()
+	if err != nil {
+		return refused("VM command build failed"), d.release(a), marker
+	}
+	// A launch delayed past the fixed deadline is refused before QEMU starts.
+	if expired(hardDeadline) {
+		return refused(ErrDeadlineExpired.Error()), d.release(a), marker
+	}
+	hk, err := armLaunch(cmd, d.Image, lib, hardDeadline)
+	if err != nil {
+		return refused("hard-kill setup failed"), d.release(a), marker
+	}
+	defer hk.close()
+	var result Reply
+	unrecorded := false
+	out, runErr := attempt.Run(ctx, cmd, timeoutFor(req), func(sctx context.Context, pidfd int) error {
+		// The owner holds the launch identity before the guest is sent anything.
+		l, err := launchIdentity(pidfd, cmd.Process.Pid, a.tap)
+		if err != nil {
+			return err
+		}
+		if err := d.Grants.Launched(sctx, req.Grant, l); err != nil {
+			unrecorded = true
+			return fmt.Errorf("%w: %w", ErrLaunchNotRecorded, err)
+		}
+		// No frame reaches the guest until QEMU proves its own kill timer is armed.
+		if err := hk.await(sctx, cmd.Process.Pid, armWait); err != nil {
+			return err
+		}
+		r, err := d.session(sctx, req, gen, ln, a.dir, hardDeadline, d.Grants.Accept)
+		result = r
+		return err
+	})
+	if !out.Stopped {
+		slog.Error("Lambda test link: VM stop not proven; marker, tap, address and files held",
+			"attempt", req.AttemptID, "tap", tapName, "err", runErr)
+		return Reply{Outcome: OutcomeUnknown, Error: "VM stop not proven"}, false, marker
+	}
+	// What the callback set is read only once it has returned.
+	select {
+	case <-out.SessionDone:
+	default:
+		slog.Error("Lambda test link: session still running after the VM stopped; attempt held", "attempt", req.AttemptID)
+		return Reply{Outcome: OutcomeUnknown, Error: "session did not finish"}, false, marker
+	}
+	if unrecorded {
+		slog.Error("Lambda test link: launch identity write failed; attempt held as ambiguous",
+			"attempt", req.AttemptID, "err", runErr)
+		return Reply{Outcome: OutcomeUnknown, Error: ErrLaunchNotRecorded.Error()}, false, marker
+	}
+	resolved = d.release(a)
+	if runErr != nil {
+		slog.Warn("Lambda test link: attempt ended without a result", "attempt", req.AttemptID, "timed_out", out.TimedOut, "err", runErr)
+	}
+	return sessionReply(result, runErr), resolved, marker
+}
+
+// sessionReply is what the caller is told once the VM is proven stopped: the
+// session's result, or an unknown outcome carrying only a known error name.
+func sessionReply(result Reply, runErr error) Reply {
+	if runErr != nil {
+		return Reply{Outcome: OutcomeUnknown, Error: safeRunError(runErr)}
+	}
+	return result
+}
+
+func safeRunError(err error) string {
+	for _, known := range []error{ErrLaunchNotRecorded, ErrVMGone, ErrNoPidfs, ErrNotArmed, ErrResultNotAccepted, ErrFunctionTimeout, ErrTimeoutUnproven, attempt.ErrTimedOut, attempt.ErrVMExited, attempt.ErrNoPidfd, context.Canceled, context.DeadlineExceeded} {
+		if errors.Is(err, known) {
+			return known.Error()
+		}
+	}
+	return "guest session failed"
+}
+
+// QEMU reads the image only from the sealed copies run passes as fds 3 to 5.
+const (
+	sealedFirmware = "/proc/self/fd/3"
+	sealedKernel   = "/proc/self/fd/4"
+	sealedInitrd   = "/proc/self/fd/5"
+)
+
+// armLaunch gives QEMU the sealed image as fds 3 to 5 and then arms the hard
+// kill, whose ack and initializer fds follow. Any other inherited file would
+// shift those numbers, so it is refused. The daemon keeps the image fds open.
+func armLaunch(cmd *exec.Cmd, img *Image, lib *os.File, deadline time.Duration) (*hardKill, error) {
+	if img == nil || len(cmd.ExtraFiles) != 0 {
+		return nil, errors.New("VM command has unexpected inherited files")
+	}
+	cmd.ExtraFiles = []*os.File{img.Firmware, img.Kernel, img.Initrd}
+	return armCommand(cmd, lib, deadline)
+}
+
+// prepare writes the package disk and fw_cfg files and opens the listener the
+// guest dials.
+func (d *Daemon) prepare(req Request, code []byte, dir, tap, mac, ip string) (*vm.Config, *guestconn.Generation, *net.TCPListener, error) {
+	if err := os.Mkdir(dir, 0o700); err != nil {
+		return nil, nil, nil, err
+	}
+	gen, err := guestconn.NewGeneration(lambdav1.Scope(scope(req)), timeoutFor(req)+time.Hour)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	ln, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.ParseIP(d.HostIP)})
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	fail := func(err error) (*vm.Config, *guestconn.Generation, *net.TCPListener, error) {
+		_ = ln.Close()
+		return nil, nil, nil, err
+	}
+	// The guest reads exactly CodeSize bytes; the padding only makes a whole sector.
+	disk := make([]byte, (len(code)+511)/512*512)
+	copy(disk, code)
+	diskPath := filepath.Join(dir, "package.raw")
+	if err := os.WriteFile(diskPath, disk, 0o600); err != nil {
+		return fail(err)
+	}
+	entries := gen.GuestEntries(ln.Addr().String())
+	entries["test-package-size"] = []byte(strconv.FormatInt(req.CodeSize, 10))
+	entries["test-package-sha256"] = []byte(req.CodeSHA256)
+	entries["test-grant"] = grantEntry(req.Grant)
+	entries["test-handler"] = []byte(req.Handler)
+	var fwCfg []vm.FwCfgEntry
+	for name, value := range entries {
+		path := filepath.Join(dir, "fwcfg-"+name)
+		if err := os.WriteFile(path, value, 0o600); err != nil {
+			return fail(err)
+		}
+		fwCfg = append(fwCfg, vm.FwCfgEntry{Name: lambdav1.FwCfgPrefix + name, File: path})
+	}
+	netcfg := filepath.Join(dir, "netcfg")
+	if err := os.WriteFile(netcfg, fmt.Appendf(nil, "NIC0_MAC=%s\nNIC0_CIDR=%s/24\nNIC0_DEFAULT=0\n", mac, ip), 0o600); err != nil {
+		return fail(err)
+	}
+	fwCfg = append(fwCfg, vm.FwCfgEntry{Name: fwCfgNetcfg, File: netcfg})
+
+	cfg := vm.NewMicroVMConfig(vm.MicroVMSpec{
+		Name:         "lambda-" + req.AttemptID,
+		Architecture: "x86_64",
+		VCPUs:        1,
+		MemoryMiB:    int(max(req.MemoryMiB, 128)),
+		LogLevel:     3,
+		FwCfg:        fwCfg,
+		NetDevs:      []vm.NetDev{vm.TapNetDev("net0", tap, 1)},
+	})
+	cfg.BIOS, cfg.KernelImage, cfg.Initrd = sealedFirmware, sealedKernel, sealedInitrd
+	cfg.Devices = append(cfg.Devices, vm.NetDevice(cfg.MachineType, "net0", mac, 1, d.MTU))
+	cfg.Drives = append(cfg.Drives, vm.Drive{File: diskPath, Format: "raw", If: "none", ID: "pkg", ReadOnly: true})
+	blk := "virtio-blk-pci,drive=pkg"
+	if vm.IsMMIO(cfg.MachineType) {
+		blk = "virtio-blk-device,drive=pkg"
+	}
+	cfg.Devices = append(cfg.Devices, vm.Device{Value: blk})
+	return &cfg, gen, ln, nil
+}
+
+// hardDeadlineFrom is the fixed backstop for an attempt whose clock was read at
+// boot: the bounded code fetch, boot and function timeout, then the kill margin.
+func hardDeadlineFrom(boot time.Duration, req Request) time.Duration {
+	return boot + fetchTimeout + timeoutFor(req) + HardKillMargin
+}
+
+func timeoutFor(req Request) time.Duration {
+	return time.Duration(req.TimeoutMs)*time.Millisecond + BootAllowance
+}
+
+// session accepts the guest, sends the one invocation and checks the result
+// answers it before the owner records it accepted. The guest key file goes
+// once the guest has authenticated.
+func (d *Daemon) session(ctx context.Context, req Request, gen *guestconn.Generation, ln *net.TCPListener, dir string,
+	hardDeadline time.Duration, accept func(context.Context, execution.Grant) error) (Reply, error) {
+	conn, err := gen.AcceptGuest(ctx, ln, handshakeTimeout)
+	if err != nil {
+		return Reply{}, fmt.Errorf("accept guest: %w", err)
+	}
+	defer func() { _ = conn.Close() }()
+	_ = os.Remove(filepath.Join(dir, "fwcfg-"+lambdav1.EntryGuestKey))
+
+	// The function timeout runs on the host from dispatch; the boot allowance
+	// covers startup only, so a guest that stops answering is cut off here.
+	deadline := now().Add(time.Duration(req.TimeoutMs) * time.Millisecond)
+	if err := conn.SetDeadline(deadline); err != nil {
+		return Reply{}, fmt.Errorf("set guest deadline: %w", err)
+	}
+	rctx, cancel := context.WithDeadline(ctx, deadline)
+	defer cancel()
+
+	stream := hostlink.NewTestStream(conn)
+	// Accepting the guest can outlast the deadline; a pause after this check
+	// still relies on QEMU's kill timer.
+	if expired(hardDeadline) {
+		return Reply{}, ErrDeadlineExpired
+	}
+	if err := stream.HostSend(hostlink.Request{
+		GrantID:     req.Grant.GrantID,
+		AttemptID:   req.AttemptID,
+		RequestID:   req.RequestID,
+		FunctionARN: req.FunctionARN,
+		DeadlineMs:  deadline.UnixMilli(),
+		Payload:     req.Payload,
+	}); err != nil {
+		return Reply{}, pastDeadline(deadline, fmt.Errorf("send invocation: %w", err))
+	}
+	resp, err := stream.HostResult(rctx)
+	if err != nil {
+		return Reply{}, pastDeadline(deadline, fmt.Errorf("read result: %w", err))
+	}
+	// A result received once the deadline has passed is still a timeout.
+	if !now().Before(deadline) {
+		return Reply{}, ErrFunctionTimeout
+	}
+	if resp.GrantID != req.Grant.GrantID || resp.AttemptID != req.AttemptID || resp.RequestID != req.RequestID {
+		return Reply{}, errors.New("result answers a different request")
+	}
+	// Either the agent's own clock or a bootstrap can claim a timeout, and
+	// neither proves one, so the claim is never reported as a result.
+	if resp.FunctionError == errTypeTimedOut {
+		return Reply{}, ErrTimeoutUnproven
+	}
+	if err := accept(rctx, req.Grant); err != nil {
+		return Reply{}, pastDeadline(deadline, fmt.Errorf("%w: %w", ErrResultNotAccepted, err))
+	}
+	// An acceptance that lands after the deadline is still a timeout.
+	if !now().Before(deadline) {
+		return Reply{}, ErrFunctionTimeout
+	}
+	_ = stream.HostShutdown()
+	return Reply{Outcome: OutcomeDone, Payload: resp.Payload, FunctionError: resp.FunctionError}, nil
+}
+
+// now is the clock the dispatch deadline is read from; tests replace it.
+var now = time.Now
+
+// ErrResultNotAccepted reports a result the owner would not accept under the
+// attempt's grant.
+var ErrResultNotAccepted = errors.New("result not accepted under the current grant")
+
+// ErrFunctionTimeout reports that the guest did not answer within the
+// function timeout counted from dispatch.
+var ErrFunctionTimeout = errors.New("function timed out")
+
+// errTypeTimedOut is the function error a guest uses to claim a timeout.
+const errTypeTimedOut = "Sandbox.Timedout"
+
+// ErrTimeoutUnproven reports a result claiming a timeout the host cannot
+// attribute; the outcome stays unknown.
+var ErrTimeoutUnproven = errors.New("guest-reported timeout not proven")
+
+func pastDeadline(deadline time.Time, err error) error {
+	if !now().Before(deadline) {
+		return fmt.Errorf("%w: %w", ErrFunctionTimeout, err)
+	}
+	return err
+}
+
+// held is what one attempt holds, in the order it took them.
+type held struct {
+	id, dir, marker string
+	ip              bool
+	tap             host.OwnedTap
+}
+
+// release gives back what an attempt held once nothing can still use it, and
+// is safe to repeat. It stops at the first failure and reports whether it all
+// went; the marker is not its to remove.
+func (d *Daemon) release(a *held) bool {
+	if err := os.RemoveAll(a.dir); err != nil {
+		slog.Error("Lambda test link: remove attempt files; attempt held", "dir", a.dir, "err", err)
+		return false
+	}
+	if a.tap.Name != "" {
+		if err := d.Taps.RemoveOwnedTap(a.tap); err != nil {
+			slog.Error("Lambda test link: tap removal failed; attempt held", "tap", a.tap.Name, "err", err)
+			return false
+		}
+	}
+	if a.ip {
+		if err := d.IPs.ReleaseChecked(a.id); err != nil {
+			slog.Error("Lambda test link: address release unconfirmed; attempt held", "attempt", a.id, "err", err)
+			return false
+		}
+	}
+	return true
+}
+
+func refused(reason string) Reply { return Reply{Outcome: OutcomeRefused, Error: reason} }

spinifex/runtime/compute/lambda/testlink/daemon_result_test.go

@@ -0,0 +1,151 @@
+//go:build lambdatestlink
+
+package testlink
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/agent"
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/runtimeapi"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/execution"
+	"github.com/stretchr/testify/require"
+)
+
+const runtimeBase = "/2018-06-01/runtime/"
+
+// countAccept counts how often the owner is asked to accept a result.
+func countAccept(n *atomic.Int32) func(context.Context, execution.Grant) error {
+	return func(context.Context, execution.Grant) error {
+		n.Add(1)
+		return nil
+	}
+}
+
+// skewLink moves every deadline the agent sees earlier by ahead, as a guest
+// clock running that far ahead of the host's would.
+type skewLink struct {
+	hostlink.Link
+	ahead time.Duration
+}
+
+func (l skewLink) Next(ctx context.Context) (hostlink.Request, error) {
+	r, err := l.Link.Next(ctx)
+	r.DeadlineMs -= l.ahead.Milliseconds()
+	return r, err
+}
+
+// runGuest serves the session's guest end with the real agent and Runtime API
+// server; bootstrap, if set, is an HTTP client standing in for the function
+// runtime, not a launched bootstrap process.
+func runGuest(t *testing.T, ctx context.Context, guest net.Conn, ahead time.Duration, bootstrap func(base string)) {
+	t.Helper()
+	srv := runtimeapi.New()
+	ts := httptest.NewServer(srv.Handler())
+	t.Cleanup(ts.Close)
+	if bootstrap != nil {
+		go bootstrap(ts.URL + runtimeBase)
+	}
+	var link hostlink.Link = hostlink.NewTestStream(guest)
+	if ahead > 0 {
+		link = skewLink{link, ahead}
+	}
+	go func() { _ = agent.Run(ctx, link, srv) }()
+}
+
+// nextInvocation takes one invocation as a bootstrap does and returns its ID.
+func nextInvocation(base string) (string, bool) {
+	resp, err := http.Get(base + "invocation/next")
+	if err != nil {
+		return "", false
+	}
+	_, _ = io.Copy(io.Discard, resp.Body)
+	_ = resp.Body.Close()
+	return resp.Header.Get("Lambda-Runtime-Aws-Request-Id"), true
+}
+
+func post(base, path string, body []byte, header http.Header) {
+	req, err := http.NewRequest(http.MethodPost, base+path, bytes.NewReader(body))
+	if err != nil {
+		return
+	}
+	for k, v := range header {
+		req.Header[k] = v
+	}
+	if resp, err := http.DefaultClient.Do(req); err == nil {
+		_ = resp.Body.Close()
+	}
+}
+
+// A guest whose clock runs ahead gives up early and claims a timeout before
+// the host's deadline; the host does not report that as a result.
+func TestSession_GuestClockAheadTimeoutClaimIsUnknown(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	var accepts atomic.Int32
+	guest, done := startSession(t, ctx, testRequest(10_000), countAccept(&accepts))
+	// The bootstrap takes the invocation and never answers it.
+	runGuest(t, ctx, guest, 10_000*time.Millisecond-200*time.Millisecond, func(base string) { nextInvocation(base) })
+
+	e := <-done
+	require.ErrorIs(t, e.err, ErrTimeoutUnproven)
+	require.Zero(t, accepts.Load(), "the owner was asked to accept a timeout claim")
+	require.Less(t, e.took, 5*time.Second, "the claim arrived before the host's deadline")
+	require.Empty(t, e.reply.Payload)
+	require.Equal(t, ErrTimeoutUnproven.Error(), safeRunError(e.err))
+}
+
+// A bootstrap can post the timeout error type itself; that is not a timeout.
+func TestSession_BootstrapPostedTimeoutIsUnknown(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	var accepts atomic.Int32
+	guest, done := startSession(t, ctx, testRequest(10_000), countAccept(&accepts))
+	runGuest(t, ctx, guest, 0, func(base string) {
+		id, ok := nextInvocation(base)
+		if !ok {
+			return
+		}
+		post(base, "invocation/"+id+"/error", []byte(`{"errorType":"Sandbox.Timedout","errorMessage":"spoof"}`),
+			http.Header{"Lambda-Runtime-Function-Error-Type": {"Sandbox.Timedout"}})
+	})
+
+	e := <-done
+	require.ErrorIs(t, e.err, ErrTimeoutUnproven)
+	require.Zero(t, accepts.Load(), "the owner was asked to accept a timeout claim")
+	require.Empty(t, e.reply.Payload)
+}
+
+// An oversized function response comes back as an Unhandled function error
+// whose payload names Function.ResponseSizeTooLarge.
+func TestSession_OversizedResponseIsUnhandled(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	guest, done := startSession(t, ctx, testRequest(10_000), acceptAll)
+	runGuest(t, ctx, guest, 0, func(base string) {
+		id, ok := nextInvocation(base)
+		if !ok {
+			return
+		}
+		post(base, "invocation/"+id+"/response", make([]byte, runtimeapi.MaxResponseBytes+1), nil)
+	})
+
+	e := <-done
+	require.NoError(t, e.err)
+	require.Equal(t, OutcomeDone, e.reply.Outcome)
+	require.Equal(t, "Unhandled", e.reply.FunctionError)
+	var body struct {
+		ErrorType string `json:"errorType"`
+	}
+	require.NoError(t, json.Unmarshal(e.reply.Payload, &body))
+	require.Equal(t, "Function.ResponseSizeTooLarge", body.ErrorType)
+}

spinifex/runtime/compute/lambda/testlink/daemon_timeout_test.go

@@ -0,0 +1,192 @@
+//go:build lambdatestlink
+
+package testlink
+
+import (
+	"context"
+	"errors"
+	"math"
+	"net"
+	"os"
+	"path/filepath"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/execution"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/guestconn"
+	"github.com/stretchr/testify/require"
+)
+
+type sessionEnd struct {
+	reply Reply
+	err   error
+	took  time.Duration
+}
+
+// startSession runs one session against a real authenticated guest
+// connection and returns that connection and the session's end.
+func startSession(t *testing.T, ctx context.Context, req Request, accept func(context.Context, execution.Grant) error) (net.Conn, <-chan sessionEnd) {
+	t.Helper()
+	gen, err := guestconn.NewGeneration(lambdav1.Scope(scope(req)), time.Hour)
+	require.NoError(t, err)
+	ln, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1)})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = ln.Close() })
+
+	guestDir := t.TempDir()
+	for name, b := range gen.GuestEntries(ln.Addr().String()) {
+		require.NoError(t, os.MkdirAll(filepath.Join(guestDir, name), 0o700))
+		require.NoError(t, os.WriteFile(filepath.Join(guestDir, name, "raw"), b, 0o600))
+	}
+	id, err := hostlink.LoadIdentity(guestDir)
+	require.NoError(t, err)
+
+	done := make(chan sessionEnd, 1)
+	workDir := t.TempDir()
+	go func() {
+		start := time.Now()
+		r, err := (&Daemon{}).session(ctx, req, gen, ln, workDir, sessionDeadline, accept)
+		done <- sessionEnd{r, err, time.Since(start)}
+	}()
+	guest, err := hostlink.Dial(ctx, id)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = guest.Close() })
+	return guest, done
+}
+
+func testRequest(timeoutMs int64) Request {
+	return Request{Grant: execution.Grant{GrantID: "g1", AttemptID: "0123456789ab", EnvironmentID: "env-1", Generation: 1},
+		AttemptID: "0123456789ab", RequestID: "r", Region: "ap-southeast-2",
+		AccountID: "123456789012", FunctionID: "fn1", TimeoutMs: timeoutMs}
+}
+
+func acceptAll(context.Context, execution.Grant) error { return nil }
+
+// A guest that authenticates and then never answers must be cut off at the
+// function timeout from dispatch, not at the boot allowance.
+func TestSession_UnresponsiveGuestStopsAtTheFunctionTimeout(t *testing.T) {
+	// The session's own context allows far longer than the function timeout.
+	ctx, cancel := context.WithTimeout(context.Background(), BootAllowance+time.Minute)
+	defer cancel()
+	_, done := startSession(t, ctx, testRequest(300), acceptAll)
+
+	select {
+	case e := <-done:
+		require.True(t, errors.Is(e.err, ErrFunctionTimeout), "got %v", e.err)
+		require.Less(t, e.took, BootAllowance/3)
+		require.Equal(t, ErrFunctionTimeout.Error(), safeRunError(e.err))
+	case <-time.After(BootAllowance):
+		t.Fatal("session waited past the function timeout for an unresponsive guest")
+	}
+}
+
+// A valid result read once the deadline has passed is a timeout, not a result.
+func TestSession_ResultAfterTheDeadlineIsATimeout(t *testing.T) {
+	// The first reading sets the deadline; every later one is an hour on.
+	var reads atomic.Int32
+	start := time.Now()
+	now = func() time.Time {
+		if reads.Add(1) == 1 {
+			return start
+		}
+		return start.Add(time.Hour)
+	}
+	t.Cleanup(func() { now = time.Now })
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	req := testRequest(10_000)
+	guest, done := startSession(t, ctx, req, acceptAll)
+
+	stream := hostlink.NewTestStream(guest)
+	got, err := stream.Next(ctx)
+	require.NoError(t, err)
+	require.NoError(t, stream.Complete(ctx, hostlink.Response{GrantID: got.GrantID,
+		AttemptID: got.AttemptID, RequestID: got.RequestID, Payload: []byte(`{"late":true}`)}))
+
+	e := <-done
+	require.ErrorIs(t, e.err, ErrFunctionTimeout)
+	require.Empty(t, e.reply.Payload)
+}
+
+// A result the owner will not accept under the grant is never reported, and
+// the guest sees the grant ID the owner issued.
+func TestSession_ResultTheOwnerRefusesIsNotReported(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	refuse := func(context.Context, execution.Grant) error { return execution.ErrNotCurrent }
+	guest, done := startSession(t, ctx, testRequest(10_000), refuse)
+
+	stream := hostlink.NewTestStream(guest)
+	got, err := stream.Next(ctx)
+	require.NoError(t, err)
+	require.Equal(t, "g1", got.GrantID)
+	require.NoError(t, stream.Complete(ctx, hostlink.Response{GrantID: got.GrantID,
+		AttemptID: got.AttemptID, RequestID: got.RequestID, Payload: []byte(`{"ok":true}`)}))
+
+	e := <-done
+	require.ErrorIs(t, e.err, ErrResultNotAccepted)
+	require.ErrorIs(t, e.err, execution.ErrNotCurrent)
+	require.Empty(t, e.reply.Payload)
+	require.Equal(t, ErrResultNotAccepted.Error(), safeRunError(e.err))
+}
+
+// An acceptance the owner records only after the deadline is a timeout, and
+// Accept runs under the dispatch deadline.
+func TestSession_AcceptanceAfterTheDeadlineIsATimeout(t *testing.T) {
+	start := time.Now()
+	var late atomic.Bool
+	now = func() time.Time {
+		if late.Load() {
+			return start.Add(time.Hour)
+		}
+		return start
+	}
+	t.Cleanup(func() { now = time.Now })
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	var acceptDeadline time.Time
+	slowAccept := func(actx context.Context, _ execution.Grant) error {
+		acceptDeadline, _ = actx.Deadline()
+		late.Store(true)
+		return nil
+	}
+	guest, done := startSession(t, ctx, testRequest(10_000), slowAccept)
+
+	stream := hostlink.NewTestStream(guest)
+	got, err := stream.Next(ctx)
+	require.NoError(t, err)
+	require.NoError(t, stream.Complete(ctx, hostlink.Response{GrantID: got.GrantID,
+		AttemptID: got.AttemptID, RequestID: got.RequestID, Payload: []byte(`{"ok":true}`)}))
+
+	e := <-done
+	require.ErrorIs(t, e.err, ErrFunctionTimeout)
+	require.Empty(t, e.reply.Payload)
+	require.Equal(t, start.Add(10*time.Second), acceptDeadline, "Accept is bounded by the dispatch deadline")
+}
+
+// sessionDeadline is the fixed boottime deadline startSession passes; far
+// ahead unless a test moves it.
+var sessionDeadline = time.Duration(math.MaxInt64)
+
+// A session whose guest accept outlasts the fixed deadline sends no frame.
+func TestSession_ExpiredBeforeHostSendSendsNothing(t *testing.T) {
+	sessionDeadline = 0
+	t.Cleanup(func() { sessionDeadline = time.Duration(math.MaxInt64) })
+	req := testRequest(1000)
+	called := false
+	guest, done := startSession(t, context.Background(), req, func(context.Context, execution.Grant) error {
+		called = true
+		return nil
+	})
+	end := <-done
+	require.ErrorIs(t, end.err, ErrDeadlineExpired)
+	require.False(t, called)
+	_ = guest.SetReadDeadline(time.Now().Add(2 * time.Second))
+	n, _ := guest.Read(make([]byte, 1))
+	require.Zero(t, n, "no frame reached the guest")
+}

spinifex/runtime/compute/lambda/testlink/fetch_test.go

@@ -0,0 +1,30 @@
+//go:build lambdatestlink
+
+package testlink
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// A redirect whose Location cannot be parsed makes Go's client quote the
+// Location into its error; a credential carried there must not survive.
+func TestFetch_MalformedRedirectDropsCredential(t *testing.T) {
+	var served atomic.Int32
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		served.Add(1)
+		w.Header().Set("Location", "%zz?X-Amz-Credential=AKIASYNTHETICTESTKEY&X-Amz-Signature=deadbeef")
+		w.WriteHeader(http.StatusFound)
+	}))
+	t.Cleanup(srv.Close)
+
+	d := &Daemon{HTTP: srv.Client()}
+	_, err := d.fetch(t.Context(), Request{CodeURL: srv.URL + "/code", CodeSize: 1, CodeSHA256: "00"})
+	require.Error(t, err)
+	require.Equal(t, int32(1), served.Load(), "the redirect never reached the client")
+	require.Equal(t, "code download: request failed", err.Error())
+}

spinifex/runtime/compute/lambda/testlink/hardkill.go

@@ -0,0 +1,253 @@
+//go:build lambdatestlink
+
+package testlink
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"debug/elf"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"runtime"
+	"strconv"
+	"strings"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+// HardKillMargin is how far past the attempt's own bound the in-QEMU timer
+// fires; it backstops a daemon that stops enforcing the deadline.
+const HardKillMargin = 10 * time.Second
+
+// armWait bounds how long the daemon waits for the initializer's ack.
+const armWait = 5 * time.Second
+
+// ErrNotArmed means QEMU gave no valid proof that its hard-kill timer is armed.
+var ErrNotArmed = errors.New("hard-kill timer not armed")
+
+// ErrDeadlineExpired means the attempt's fixed hard-kill deadline has passed,
+// so nothing may be dispatched to the guest.
+var ErrDeadlineExpired = errors.New("hard-kill deadline expired")
+
+// bootNow reads CLOCK_BOOTTIME, which keeps counting through suspend; a var so
+// tests can move it.
+var bootNow = func() (time.Duration, error) {
+	var ts unix.Timespec
+	if err := unix.ClockGettime(unix.CLOCK_BOOTTIME, &ts); err != nil {
+		return 0, fmt.Errorf("read CLOCK_BOOTTIME: %w", err)
+	}
+	return time.Duration(ts.Nano()), nil
+}
+
+// expired reports whether the absolute boottime deadline has passed; an
+// unreadable clock counts as expired.
+func expired(deadline time.Duration) bool {
+	t, err := bootNow()
+	return err != nil || t >= deadline
+}
+
+// maxInitializer bounds the bytes read for the hard-kill initializer.
+const maxInitializer = 1 << 20
+
+// requiredSeals stop any later write, resize or seal change to the memfd.
+const requiredSeals = unix.F_SEAL_SEAL | unix.F_SEAL_SHRINK | unix.F_SEAL_GROW | unix.F_SEAL_WRITE
+
+// sealInitializer reads the initializer once, checks those bytes against the
+// pinned digest, and returns them in a sealed memfd that QEMU loads by fd. It
+// authenticates nothing about where the pinned digest came from.
+func (d *Daemon) sealInitializer() (*os.File, error) {
+	if d.Initializer == "" || d.InitializerSHA256 == "" {
+		return nil, errors.New("hard-kill initializer not configured")
+	}
+	f, err := os.Open(d.Initializer)
+	if err != nil {
+		return nil, errors.New("hard-kill initializer unreadable")
+	}
+	b, err := io.ReadAll(io.LimitReader(f, maxInitializer+1))
+	_ = f.Close()
+	if err != nil || len(b) == 0 || len(b) > maxInitializer {
+		return nil, errors.New("hard-kill initializer unreadable")
+	}
+	if !digestIs(b, d.InitializerSHA256) {
+		return nil, errors.New("hard-kill initializer digest mismatch")
+	}
+	if err := sameArch(b); err != nil {
+		return nil, err
+	}
+	m, err := sealedCopy(b)
+	if err != nil {
+		return nil, fmt.Errorf("hard-kill initializer seal: %w", err)
+	}
+	// The sealed object, not the buffer, is what QEMU maps: check it again.
+	got := make([]byte, len(b)+1)
+	n, err := m.ReadAt(got, 0)
+	if (err != nil && !errors.Is(err, io.EOF)) || !digestIs(got[:n], d.InitializerSHA256) {
+		_ = m.Close()
+		return nil, errors.New("hard-kill initializer seal: sealed copy differs")
+	}
+	return m, nil
+}
+
+// CheckInitializer proves the configured initializer matches the pin and this
+// daemon's architecture, so a mismatched pair is refused before serving.
+func (d *Daemon) CheckInitializer() error {
+	m, err := d.sealInitializer()
+	if err != nil {
+		return err
+	}
+	return m.Close()
+}
+
+// elfMachine maps GOARCH to the ELF machine an initializer must be built for.
+var elfMachine = map[string]elf.Machine{"amd64": elf.EM_X86_64, "arm64": elf.EM_AARCH64}
+
+// sameArch refuses an initializer whose ELF machine differs from the daemon's.
+func sameArch(b []byte) error {
+	want, ok := elfMachine[runtime.GOARCH]
+	f, err := elf.NewFile(bytes.NewReader(b))
+	if err != nil {
+		return errors.New("hard-kill initializer is not an ELF object")
+	}
+	if !ok || f.Machine != want || f.Class != elf.ELFCLASS64 {
+		return fmt.Errorf("hard-kill initializer built for %v, daemon is %s", f.Machine, runtime.GOARCH)
+	}
+	return nil
+}
+
+// memfdCreate is unix.MemfdCreate; tests replace it to reach kernel refusals.
+var memfdCreate = unix.MemfdCreate
+
+func digestIs(b []byte, want string) bool {
+	sum := sha256.Sum256(b)
+	return hex.EncodeToString(sum[:]) == want
+}
+
+// sealedCopy writes b to a new memfd and applies requiredSeals, failing
+// unless the kernel reports exactly those seals afterwards.
+func sealedCopy(b []byte) (*os.File, error) {
+	fd, err := memfdCreate("spx-hardkill", unix.MFD_CLOEXEC|unix.MFD_ALLOW_SEALING|unix.MFD_EXEC)
+	switch {
+	case errors.Is(err, unix.EINVAL):
+		// An old kernel or a security filter: no retry without MFD_EXEC.
+		return nil, fmt.Errorf("MFD_EXEC unavailable or refused: %w", err)
+	case errors.Is(err, unix.EACCES):
+		// Possibly vm.memfd_noexec, possibly another policy: never bypassed.
+		return nil, fmt.Errorf("executable memfd refused: %w", err)
+	case err != nil:
+		return nil, err
+	}
+	m := os.NewFile(uintptr(fd), "spx-hardkill")
+	if _, err := m.Write(b); err != nil {
+		_ = m.Close()
+		return nil, err
+	}
+	if _, err := unix.FcntlInt(m.Fd(), unix.F_ADD_SEALS, requiredSeals); err != nil {
+		_ = m.Close()
+		return nil, err
+	}
+	seals, err := unix.FcntlInt(m.Fd(), unix.F_GET_SEALS, 0)
+	if err != nil || seals&requiredSeals != requiredSeals {
+		_ = m.Close()
+		return nil, errors.New("seals not in place")
+	}
+	return m, nil
+}
+
+// hardKill is one attempt's arming request and the pipe its ack arrives on.
+type hardKill struct {
+	deadline time.Duration // absolute CLOCK_BOOTTIME
+	nonce    string
+	r, w     *os.File
+}
+
+// armCommand sets cmd's environment to the daemon's without any inherited
+// dynamic-loader or backstop setting, then preloads lib by its inherited fd
+// and passes the ack pipe and the absolute boottime deadline; the caller still owns lib.
+func armCommand(cmd *exec.Cmd, lib *os.File, deadline time.Duration) (*hardKill, error) {
+	if deadline <= 0 {
+		return nil, errors.New("hard-kill deadline not set")
+	}
+	nb := make([]byte, 16)
+	if _, err := rand.Read(nb); err != nil {
+		return nil, fmt.Errorf("hard-kill nonce: %w", err)
+	}
+	r, w, err := os.Pipe()
+	if err != nil {
+		return nil, fmt.Errorf("hard-kill pipe: %w", err)
+	}
+	hk := &hardKill{deadline: deadline, nonce: hex.EncodeToString(nb), r: r, w: w}
+	base := cmd.Env
+	if base == nil {
+		base = os.Environ()
+	}
+	env := make([]string, 0, len(base)+4)
+	for _, kv := range base {
+		if strings.HasPrefix(kv, "LD_") || strings.HasPrefix(kv, "SPX_HARDKILL_") {
+			continue
+		}
+		env = append(env, kv)
+	}
+	ack := 3 + len(cmd.ExtraFiles)
+	cmd.Env = append(env,
+		"LD_PRELOAD=/proc/self/fd/"+strconv.Itoa(ack+1),
+		"SPX_HARDKILL_LIB_FD="+strconv.Itoa(ack+1),
+		"SPX_HARDKILL_DEADLINE_NS="+strconv.FormatInt(int64(hk.deadline), 10),
+		"SPX_HARDKILL_ACK_FD="+strconv.Itoa(ack),
+		"SPX_HARDKILL_NONCE="+hk.nonce)
+	cmd.ExtraFiles = append(cmd.ExtraFiles, w, lib)
+	return hk, nil
+}
+
+// close drops both pipe ends; safe to call more than once.
+func (hk *hardKill) close() {
+	_ = hk.w.Close()
+	_ = hk.r.Close()
+}
+
+// await requires the exact ack for pid before anything is sent to the guest.
+// The daemon's copy of the write end is closed first, so a QEMU that exits or
+// never arms the timer ends the read instead of holding it open.
+func (hk *hardKill) await(ctx context.Context, pid int, wait time.Duration) error {
+	_ = hk.w.Close()
+	deadline := now().Add(wait)
+	if d, ok := ctx.Deadline(); ok && d.Before(deadline) {
+		deadline = d
+	}
+	if err := hk.r.SetReadDeadline(deadline); err != nil {
+		return fmt.Errorf("%w: %w", ErrNotArmed, err)
+	}
+	want := fmt.Sprintf("armed %d %d %s\n", pid, int64(hk.deadline), hk.nonce)
+	got := make([]byte, 0, len(want))
+	buf := make([]byte, len(want)+1)
+	for len(got) <= len(want) {
+		n, err := hk.r.Read(buf)
+		got = append(got, buf[:n]...)
+		if err != nil {
+			if errors.Is(err, io.EOF) && bytes.Equal(got, []byte(want)) {
+				if expired(hk.deadline) {
+					return ErrDeadlineExpired
+				}
+				return nil
+			}
+			break
+		}
+		if bytes.HasSuffix(got, []byte("\n")) {
+			break
+		}
+	}
+	if !bytes.Equal(got, []byte(want)) {
+		return ErrNotArmed
+	}
+	// An ack read at or after the deadline may have raced the due kill.
+	if expired(hk.deadline) {
+		return ErrDeadlineExpired
+	}
+	return nil
+}

spinifex/runtime/compute/lambda/testlink/hardkill/hardkill.c

@@ -0,0 +1,69 @@
+// Test-only backstop preloaded into QEMU: before main it arms a SIGKILL timer at an
+// absolute CLOCK_BOOTTIME deadline, then acks on an inherited fd; any failure kills it.
+// Build: cc -O2 -Wall -Wextra -shared -fPIC -o hardkill.so hardkill.c -lrt
+#define _POSIX_C_SOURCE 200809L
+#include <errno.h>
+#include <limits.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+
+static void die(void) {
+	kill(getpid(), SIGKILL);
+	_exit(127);
+}
+
+static long parse(const char *s, long min, long max) {
+	if (!s || !*s) die();
+	char *end;
+	errno = 0;
+	long v = strtol(s, &end, 10);
+	if (errno || *end || v < min || v > max) die();
+	return v;
+}
+
+__attribute__((constructor)) static void hardkill_arm(void) {
+	long dl = parse(getenv("SPX_HARDKILL_DEADLINE_NS"), 1, LONG_MAX);
+	int ack = (int)parse(getenv("SPX_HARDKILL_ACK_FD"), 3, INT_MAX);
+	int lib = (int)parse(getenv("SPX_HARDKILL_LIB_FD"), 3, INT_MAX);
+	if (lib == ack) die();
+	const char *nonce = getenv("SPX_HARDKILL_NONCE");
+	char tag[33];
+	if (!nonce || strlen(nonce) != 32 || strspn(nonce, "0123456789abcdef") != 32) die();
+	memcpy(tag, nonce, sizeof tag);
+	// Nothing QEMU starts later inherits the backstop's settings.
+	if (unsetenv("SPX_HARDKILL_DEADLINE_NS") || unsetenv("SPX_HARDKILL_ACK_FD") ||
+	    unsetenv("SPX_HARDKILL_NONCE") || unsetenv("SPX_HARDKILL_LIB_FD") ||
+	    unsetenv("LD_PRELOAD")) die();
+
+	struct sigevent sev;
+	memset(&sev, 0, sizeof sev);
+	sev.sigev_notify = SIGEV_SIGNAL;
+	sev.sigev_signo = SIGKILL;
+	timer_t t;
+	if (timer_create(CLOCK_BOOTTIME, &sev, &t) != 0) die();
+	struct itimerspec its;
+	memset(&its, 0, sizeof its);
+	its.it_value.tv_sec = dl / 1000000000L;
+	its.it_value.tv_nsec = dl % 1000000000L;
+	// A deadline already past makes the kill due at once.
+	if (timer_settime(t, TIMER_ABSTIME, &its, NULL) != 0) die();
+
+	// The loader has mapped the library; the ack also proves QEMU keeps no
+	// handle on its memfd.
+	if (close(lib) != 0) die();
+
+	char msg[128];
+	int n = snprintf(msg, sizeof msg, "armed %d %ld %s\n", getpid(), dl, tag);
+	if (n <= 0 || n >= (int)sizeof msg) die();
+	for (int off = 0; off < n;) {
+		ssize_t w = write(ack, msg + off, (size_t)(n - off));
+		if (w < 0 && errno == EINTR) continue;
+		if (w <= 0) die();
+		off += (int)w;
+	}
+	if (close(ack) != 0) die();
+}

spinifex/runtime/compute/lambda/testlink/hardkill_test.go

@@ -0,0 +1,378 @@
+//go:build lambdatestlink
+
+package testlink
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/execution"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/attempt"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
+)
+
+func TestArmCommand_DropsInheritedLoaderSettings(t *testing.T) {
+	t.Setenv("LD_PRELOAD", "/evil.so")
+	t.Setenv("LD_LIBRARY_PATH", "/evil")
+	t.Setenv("SPX_HARDKILL_DEADLINE_NS", "1")
+	t.Setenv("KEEP_ME", "yes")
+	cmd := exec.Command("/bin/true")
+	cmd.ExtraFiles = []*os.File{os.Stdin}
+	lib := emptyFile(t)
+	hk, err := armCommand(cmd, lib, 90*time.Second)
+	require.NoError(t, err)
+	defer hk.close()
+
+	var loader, backstop []string
+	for _, kv := range cmd.Env {
+		switch {
+		case strings.HasPrefix(kv, "LD_"):
+			loader = append(loader, kv)
+		case strings.HasPrefix(kv, "SPX_HARDKILL_"):
+			backstop = append(backstop, kv)
+		}
+	}
+	require.Equal(t, []string{"LD_PRELOAD=/proc/self/fd/5"}, loader)
+	require.ElementsMatch(t, []string{"SPX_HARDKILL_DEADLINE_NS=90000000000", "SPX_HARDKILL_ACK_FD=4",
+		"SPX_HARDKILL_LIB_FD=5", "SPX_HARDKILL_NONCE=" + hk.nonce}, backstop)
+	require.Contains(t, cmd.Env, "KEEP_ME=yes")
+	require.Equal(t, []*os.File{os.Stdin, hk.w, lib}, cmd.ExtraFiles)
+	require.Len(t, hk.nonce, 32)
+}
+
+// fromBoot is the absolute CLOCK_BOOTTIME deadline d from now.
+func fromBoot(t *testing.T, d time.Duration) time.Duration {
+	t.Helper()
+	b, err := bootNow()
+	require.NoError(t, err)
+	return b + d
+}
+
+func emptyFile(t *testing.T) *os.File {
+	t.Helper()
+	f, err := os.CreateTemp(t.TempDir(), "lib")
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = f.Close() })
+	return f
+}
+
+// runArmed runs script as the "VM" under attempt.Run with a backstop and
+// reports whether the step after the ack ran.
+func runArmed(t *testing.T, script string, wait time.Duration) (bool, attempt.Outcome, error) {
+	t.Helper()
+	cmd := exec.Command("/bin/sh", "-c", script)
+	hk, err := armCommand(cmd, emptyFile(t), fromBoot(t, time.Minute))
+	require.NoError(t, err)
+	defer hk.close()
+	cmd.Env = append(cmd.Env, "HK_WRONG_NONCE=0123456789abcdef0123456789abcdef")
+	sent := false
+	out, err := attempt.Run(context.Background(), cmd, 30*time.Second, func(ctx context.Context, _ int) error {
+		if err := hk.await(ctx, cmd.Process.Pid, wait); err != nil {
+			return err
+		}
+		sent = true
+		return nil
+	})
+	return sent, out, err
+}
+
+const ackLine = `printf 'armed %s %s %s\n' "$$" "$SPX_HARDKILL_DEADLINE_NS" `
+
+func TestHardKill_FrameWaitsForTheExactAck(t *testing.T) {
+	sent, out, err := runArmed(t, `eval "`+ackLine+`\"\$SPX_HARDKILL_NONCE\" >&$SPX_HARDKILL_ACK_FD"; exec sleep 60`, 2*time.Second)
+	require.NoError(t, err)
+	require.True(t, sent)
+	require.True(t, out.Stopped)
+}
+
+// Each failure kills the VM before anything is sent and proves the stop.
+func TestHardKill_MissingOrWrongAckSendsNothing(t *testing.T) {
+	cases := map[string]string{
+		"silent":         `exec sleep 60`,
+		"exits early":    `exit 0`,
+		"wrong nonce":    `eval "` + ackLine + `\"\$HK_WRONG_NONCE\" >&$SPX_HARDKILL_ACK_FD"; exec sleep 60`,
+		"wrong pid":      `eval "printf 'armed 1 %s %s\n' \"\$SPX_HARDKILL_DEADLINE_NS\" \"\$SPX_HARDKILL_NONCE\" >&$SPX_HARDKILL_ACK_FD"; exec sleep 60`,
+		"wrong deadline": `eval "printf 'armed %s 1 %s\n' \"\$\$\" \"\$SPX_HARDKILL_NONCE\" >&$SPX_HARDKILL_ACK_FD"; exec sleep 60`,
+		"trailing data":  `eval "` + ackLine + `\"\$SPX_HARDKILL_NONCE\" extra >&$SPX_HARDKILL_ACK_FD"; exec sleep 60`,
+	}
+	for name, script := range cases {
+		t.Run(name, func(t *testing.T) {
+			start := time.Now()
+			sent, out, err := runArmed(t, script, 500*time.Millisecond)
+			require.False(t, sent, "no frame without a valid ack")
+			require.True(t, out.Stopped, "the VM stop is proven")
+			require.Error(t, err)
+			if name != "exits early" {
+				require.ErrorIs(t, err, ErrNotArmed)
+			}
+			require.Less(t, time.Since(start), 10*time.Second)
+		})
+	}
+}
+
+// The real initializer, preloaded into a process that ignores its parent,
+// kills that process at the deadline after acknowledging the armed timer.
+func buildInitializer(t *testing.T) string {
+	t.Helper()
+	cc, err := exec.LookPath("cc")
+	if err != nil {
+		t.Skip("no C compiler to build the initializer")
+	}
+	lib := filepath.Join(t.TempDir(), "hardkill.so")
+	build := exec.Command(cc, "-std=c17", "-O2", "-Wall", "-Wextra", "-Werror", "-shared", "-fPIC", "-o", lib, "hardkill/hardkill.c", "-lrt")
+	if out, err := build.CombinedOutput(); err != nil {
+		t.Fatalf("build initializer: %v: %s", err, out)
+	}
+	return lib
+}
+
+func requireSIGKILL(t *testing.T, err error) {
+	t.Helper()
+	var exit *exec.ExitError
+	require.True(t, errors.As(err, &exit), "got %v", err)
+	ws := exit.Sys().(syscall.WaitStatus)
+	require.True(t, ws.Signaled(), "exited %d instead of being killed", ws.ExitStatus())
+	require.Equal(t, syscall.SIGKILL, ws.Signal())
+}
+
+// A loaded initializer that cannot arm or acknowledge kills its process
+// before main, so the target program never runs.
+func TestHardKill_InitializerFailureNeverReachesMain(t *testing.T) {
+	lib := buildInitializer(t)
+	nonce := "SPX_HARDKILL_NONCE=0123456789abcdef0123456789abcdef"
+	libFD := "SPX_HARDKILL_LIB_FD=4"
+	cases := map[string][]string{
+		"no settings":      nil,
+		"bad deadline":     {"SPX_HARDKILL_DEADLINE_NS=12x", "SPX_HARDKILL_ACK_FD=3", libFD, nonce},
+		"zero deadline":    {"SPX_HARDKILL_DEADLINE_NS=0", "SPX_HARDKILL_ACK_FD=3", libFD, nonce},
+		"low fd":           {"SPX_HARDKILL_DEADLINE_NS=9000000000000000000", "SPX_HARDKILL_ACK_FD=2", libFD, nonce},
+		"no lib fd":        {"SPX_HARDKILL_DEADLINE_NS=9000000000000000000", "SPX_HARDKILL_ACK_FD=3", nonce},
+		"lib fd is ack fd": {"SPX_HARDKILL_DEADLINE_NS=9000000000000000000", "SPX_HARDKILL_ACK_FD=3", "SPX_HARDKILL_LIB_FD=3", nonce},
+		"lib fd not open":  {"SPX_HARDKILL_DEADLINE_NS=9000000000000000000", "SPX_HARDKILL_ACK_FD=3", "SPX_HARDKILL_LIB_FD=98", nonce},
+		"short nonce":      {"SPX_HARDKILL_DEADLINE_NS=9000000000000000000", "SPX_HARDKILL_ACK_FD=3", libFD, "SPX_HARDKILL_NONCE=abc"},
+		"ack fd not open":  {"SPX_HARDKILL_DEADLINE_NS=9000000000000000000", "SPX_HARDKILL_ACK_FD=97", libFD, nonce},
+	}
+	for name, env := range cases {
+		t.Run(name, func(t *testing.T) {
+			marker := filepath.Join(t.TempDir(), "ran")
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			defer cancel()
+			cmd := exec.CommandContext(ctx, "/usr/bin/touch", marker)
+			cmd.Cancel = func() error { return cmd.Process.Signal(syscall.SIGTERM) }
+			cmd.Env = append([]string{"LD_PRELOAD=" + lib}, env...)
+			r, w, err := os.Pipe()
+			require.NoError(t, err)
+			defer r.Close()
+			cmd.ExtraFiles = []*os.File{w, emptyFile(t)}
+			runErr := cmd.Run()
+			_ = w.Close()
+			ack, _ := io.ReadAll(r)
+			requireSIGKILL(t, runErr)
+			require.NoFileExists(t, marker)
+			require.Empty(t, ack, "no ack from an initializer that failed")
+		})
+	}
+}
+
+// sealedBuilt builds the initializer and seals it the way the daemon does,
+// pinning the digest of the bytes just built as a trusted build would.
+func sealedBuilt(t *testing.T) *os.File {
+	t.Helper()
+	path := buildInitializer(t)
+	b, err := os.ReadFile(path)
+	require.NoError(t, err)
+	sum := sha256.Sum256(b)
+	d := &Daemon{Initializer: path, InitializerSHA256: hex.EncodeToString(sum[:])}
+	lib, err := d.sealInitializer()
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = lib.Close() })
+	return lib
+}
+
+// mdweWrapper builds a launcher that applies the kernel W^X restriction
+// (PR_SET_MDWE), moves MDWE_PRELOAD to LD_PRELOAD so only the exec'd
+// program loads the initializer, and execs its arguments with inherited fds.
+func mdweWrapper(t *testing.T) string {
+	t.Helper()
+	cc, err := exec.LookPath("cc")
+	if err != nil {
+		t.Skip("no C compiler")
+	}
+	dir := t.TempDir()
+	src := filepath.Join(dir, "mdwe.c")
+	require.NoError(t, os.WriteFile(src, []byte(`#include <sys/prctl.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <stdlib.h>
+int main(int c, char **v) {
+	(void)c;
+	if (prctl(65, 1, 0, 0, 0) != 0) { perror("PR_SET_MDWE"); return 125; }
+	if (prctl(66, 0, 0, 0, 0) != 1) { fputs("MDWE not in effect\n", stderr); return 124; }
+	const char *p = getenv("MDWE_PRELOAD");
+	if (!p || setenv("LD_PRELOAD", p, 1) || unsetenv("MDWE_PRELOAD")) return 123;
+	execv(v[1], v + 1);
+	return 126;
+}
+`), 0o600))
+	bin := filepath.Join(dir, "mdwe")
+	if out, err := exec.Command(cc, "-o", bin, src).CombinedOutput(); err != nil {
+		t.Fatalf("build wrapper: %v: %s", err, out)
+	}
+	return bin
+}
+
+// The sealed memfd loads through /proc/self/fd, arms the timer, leaves the
+// process no handle on it, and the timer kills the process at its deadline.
+func TestHardKill_SealedInitializerLoadsByFD(t *testing.T) {
+	launchers := map[string][]string{"plain": nil, "under PR_SET_MDWE": {mdweWrapper(t)}}
+	for name, prefix := range launchers {
+		t.Run(name, func(t *testing.T) {
+			lib := sealedBuilt(t)
+			argv := append(append([]string{}, prefix...), "/bin/sleep", "60")
+			cmd := exec.Command(argv[0], argv[1:]...)
+			hk, err := armCommand(cmd, lib, fromBoot(t, 400*time.Millisecond))
+			require.NoError(t, err)
+			defer hk.close()
+			if prefix != nil {
+				for i, kv := range cmd.Env {
+					if strings.HasPrefix(kv, "LD_PRELOAD=") {
+						cmd.Env[i] = "MDWE_" + kv[len("LD_"):]
+					}
+				}
+			}
+			require.NoError(t, cmd.Start())
+			require.NoError(t, hk.await(context.Background(), cmd.Process.Pid, 5*time.Second))
+
+			fds, err := os.ReadDir(fmt.Sprintf("/proc/%d/fd", cmd.Process.Pid))
+			require.NoError(t, err)
+			for _, fd := range fds {
+				target, _ := os.Readlink(fmt.Sprintf("/proc/%d/fd/%s", cmd.Process.Pid, fd.Name()))
+				require.NotContains(t, target, "spx-hardkill", "fd %s still open", fd.Name())
+			}
+			start := time.Now()
+			requireSIGKILL(t, cmd.Wait())
+			require.Less(t, time.Since(start), 5*time.Second)
+		})
+	}
+}
+
+func TestSealInitializer_SealedAgainstChange(t *testing.T) {
+	lib := sealedBuilt(t)
+	seals, err := unix.FcntlInt(lib.Fd(), unix.F_GET_SEALS, 0)
+	require.NoError(t, err)
+	require.Equal(t, requiredSeals, seals&requiredSeals)
+	_, err = lib.WriteAt([]byte("x"), 0)
+	require.ErrorIs(t, err, unix.EPERM)
+	require.ErrorIs(t, unix.Ftruncate(int(lib.Fd()), 1), unix.EPERM)
+	_, err = unix.FcntlInt(lib.Fd(), unix.F_ADD_SEALS, unix.F_SEAL_FUTURE_WRITE)
+	require.ErrorIs(t, err, unix.EPERM)
+
+	// Reopening through /proc reaches the same sealed object, not a copy.
+	again, err := os.OpenFile(fmt.Sprintf("/proc/self/fd/%d", lib.Fd()), os.O_RDWR, 0)
+	if err == nil {
+		_, err = again.WriteAt([]byte("x"), 0)
+		_ = again.Close()
+	}
+	require.Error(t, err)
+}
+
+// MFD_EXEC unavailable or an executable memfd refused fails closed, with no
+// retry that could create a memfd the node did not allow.
+func TestSealInitializer_MemfdExecRefusalsFailClosed(t *testing.T) {
+	cases := map[string]struct {
+		err  error
+		want string
+	}{
+		"MFD_EXEC unavailable":    {unix.EINVAL, "MFD_EXEC unavailable or refused"},
+		"executable memfd denied": {unix.EACCES, "executable memfd refused"},
+		"other failure":           {unix.EMFILE, "too many open files"},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			old := memfdCreate
+			t.Cleanup(func() { memfdCreate = old })
+			var flags []int
+			memfdCreate = func(string, int) (int, error) {
+				flags = append(flags, 0)
+				return -1, tc.err
+			}
+			m, err := sealedCopy([]byte("library"))
+			require.Nil(t, m)
+			require.ErrorIs(t, err, tc.err)
+			require.ErrorContains(t, err, tc.want)
+			require.Len(t, flags, 1, "no retry")
+		})
+	}
+}
+
+func TestDaemon_InitializerCheckRefusesBeforeTheGrantStarts(t *testing.T) {
+	code := []byte("zip bytes")
+	srv := codeServer(t, code)
+	lib := filepath.Join(t.TempDir(), "hardkill.so")
+	require.NoError(t, os.WriteFile(lib, []byte("library"), 0o600))
+	sum := sha256.Sum256([]byte("library"))
+	cases := map[string]func(d *Daemon){
+		"not configured": func(d *Daemon) { d.Initializer, d.InitializerSHA256 = "", "" },
+		"missing":        func(d *Daemon) { d.Initializer = lib + ".gone" },
+		"replaced":       func(d *Daemon) { d.InitializerSHA256 = strings.Repeat("0", 64) },
+		"empty file": func(d *Daemon) {
+			empty := filepath.Join(t.TempDir(), "empty.so")
+			require.NoError(t, os.WriteFile(empty, nil, 0o600))
+			d.Initializer = empty
+		},
+	}
+	for name, mutate := range cases {
+		t.Run(name, func(t *testing.T) {
+			g := grant(t, code)
+			ips := &fakeIPs{}
+			d := g.daemon(&Daemon{IPs: ips, Taps: &fakeTaps{}, HTTP: srv.Client(), WorkDir: t.TempDir()})
+			d.Initializer, d.InitializerSHA256 = lib, hex.EncodeToString(sum[:])
+			mutate(d)
+			rep := d.Handle(context.Background(), g.request(srv.URL, code))
+			require.Equal(t, OutcomeRefused, rep.Outcome)
+			require.Contains(t, rep.Error, "hard-kill initializer")
+			require.Empty(t, ips.allocated)
+			require.Equal(t, execution.StateGranted, g.state(t))
+		})
+	}
+}
+
+// An exact ack read at or after the fixed deadline may have raced the due
+// kill, so nothing is sent and the VM stop is still proven.
+func TestHardKill_AckAtOrAfterTheDeadlineSendsNothing(t *testing.T) {
+	for name, past := range map[string]time.Duration{"at": 0, "after": time.Millisecond} {
+		t.Run(name, func(t *testing.T) {
+			script := `eval "` + ackLine + `\"\$SPX_HARDKILL_NONCE\" >&$SPX_HARDKILL_ACK_FD"; exec sleep 60`
+			cmd := exec.Command("/bin/sh", "-c", script)
+			hk, err := armCommand(cmd, emptyFile(t), fromBoot(t, time.Minute))
+			require.NoError(t, err)
+			defer hk.close()
+			orig := bootNow
+			bootNow = func() (time.Duration, error) { return hk.deadline + past, nil }
+			t.Cleanup(func() { bootNow = orig })
+			sent := false
+			out, err := attempt.Run(context.Background(), cmd, 30*time.Second, func(ctx context.Context, _ int) error {
+				if err := hk.await(ctx, cmd.Process.Pid, 2*time.Second); err != nil {
+					return err
+				}
+				sent = true
+				return nil
+			})
+			require.ErrorIs(t, err, ErrDeadlineExpired)
+			require.False(t, sent)
+			require.True(t, out.Stopped)
+		})
+	}
+}

spinifex/runtime/compute/lambda/testlink/image.go

@@ -0,0 +1,253 @@
+//go:build lambdatestlink
+
+package testlink
+
+import (
+	"crypto/sha256"
+	"encoding/binary"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"hash"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+
+	"golang.org/x/sys/unix"
+)
+
+// The runtime image's identity constants; they are part of its digest and are
+// never read from disk.
+const (
+	ImageName    = "lambda-runtime"
+	ImageProfile = "lambda-testlink"
+	ImageArch    = "x86_64"
+)
+
+// The three boot files, in the name order the digest uses.
+const (
+	FileInitrd   = "initramfs.cpio.gz"
+	FileFirmware = "qboot.rom"
+	FileKernel   = "vmlinuz"
+)
+
+var imageFiles = []string{FileInitrd, FileFirmware, FileKernel}
+
+// maxImageFile caps each boot file, bounding the memory a prepared image holds.
+var maxImageFile = map[string]int64{FileFirmware: 4 << 20, FileKernel: 64 << 20, FileInitrd: 512 << 20}
+
+const imageDigestPrefix = "spinifex-lambda-image-content-v1\x00"
+
+var digestPattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
+
+// ErrImage means the runtime image could not be prepared or does not match its pin.
+var ErrImage = errors.New("lambda test link: runtime image refused")
+
+// FileSum is one boot file's SHA-256 and size.
+type FileSum struct {
+	SHA256 string
+	Size   int64
+}
+
+// ImagePaths names the three boot files inside dir; the digest tool and the
+// daemon both select files through it.
+func ImagePaths(dir string) map[string]string {
+	p := make(map[string]string, len(imageFiles))
+	for _, name := range imageFiles {
+		p[name] = filepath.Join(dir, name)
+	}
+	return p
+}
+
+// ImageDigest is the runtime image's identity: the constants, then each file's
+// name, SHA-256 and size in name order, each length-prefixed. It identifies the
+// bytes; it does not authenticate where they came from.
+func ImageDigest(sums map[string]FileSum) (string, error) {
+	h := sha256.New()
+	h.Write([]byte(imageDigestPrefix))
+	for _, s := range []string{ImageName, ImageProfile, ImageArch} {
+		field(h, s)
+	}
+	for _, name := range imageFiles {
+		s, ok := sums[name]
+		if !ok || !digestPattern.MatchString(s.SHA256) || s.Size <= 0 || s.Size > maxImageFile[name] {
+			return "", fmt.Errorf("%w: %s missing or out of range", ErrImage, name)
+		}
+		field(h, name)
+		field(h, s.SHA256)
+		field(h, strconv.FormatInt(s.Size, 10))
+	}
+	return hex.EncodeToString(h.Sum(nil)), nil
+}
+
+func field(h hash.Hash, s string) {
+	var n [8]byte
+	binary.BigEndian.PutUint64(n[:], uint64(len(s)))
+	h.Write(n[:])
+	h.Write([]byte(s))
+}
+
+// ValidDigest reports whether s is 64 lowercase hex digits.
+func ValidDigest(s string) bool { return digestPattern.MatchString(s) }
+
+// openRegular opens path without following a final symlink or blocking on a
+// FIFO, and refuses anything that is not a regular file.
+func openRegular(path string) (*os.File, int64, error) {
+	fd, err := unix.Open(path, unix.O_RDONLY|unix.O_NOFOLLOW|unix.O_NONBLOCK|unix.O_CLOEXEC, 0)
+	if err != nil {
+		return nil, 0, err
+	}
+	var st unix.Stat_t
+	if err := unix.Fstat(fd, &st); err != nil {
+		_ = unix.Close(fd)
+		return nil, 0, err
+	}
+	if st.Mode&unix.S_IFMT != unix.S_IFREG {
+		_ = unix.Close(fd)
+		return nil, 0, errors.New("not a regular file")
+	}
+	return os.NewFile(uintptr(fd), path), st.Size, nil
+}
+
+// SumFile hashes one boot file in place; the digest tool uses it, the daemon
+// hashes its sealed copies instead.
+func SumFile(name, path string) (FileSum, error) {
+	f, size, err := openRegular(path)
+	if err != nil {
+		return FileSum{}, err
+	}
+	defer func() { _ = f.Close() }()
+	if size <= 0 || size > maxImageFile[name] {
+		return FileSum{}, fmt.Errorf("size %d outside 1..%d", size, maxImageFile[name])
+	}
+	h := sha256.New()
+	if err := copyExact(h, f, size); err != nil {
+		return FileSum{}, err
+	}
+	return FileSum{SHA256: hex.EncodeToString(h.Sum(nil)), Size: size}, nil
+}
+
+// DirDigest is the digest of the three boot files in dir, hashed in place;
+// PrepareImage computes the same value from its sealed copies.
+func DirDigest(dir string) (string, error) {
+	sums := make(map[string]FileSum, len(imageFiles))
+	for name, path := range ImagePaths(dir) {
+		sum, err := SumFile(name, path)
+		if err != nil {
+			return "", fmt.Errorf("%w: %s: %w", ErrImage, name, err)
+		}
+		sums[name] = sum
+	}
+	return ImageDigest(sums)
+}
+
+// copyExact copies exactly n bytes and then requires a clean end of file, so a
+// file that grows or shrinks while it is read is refused.
+func copyExact(dst io.Writer, src io.Reader, n int64) error {
+	if got, err := io.CopyN(dst, src, n); err != nil || got != n {
+		return fmt.Errorf("copied %d of %d bytes: %w", got, n, err)
+	}
+	var b [1]byte
+	m, err := src.Read(b[:])
+	if m != 0 {
+		return errors.New("file longer than its size")
+	}
+	if !errors.Is(err, io.EOF) {
+		return fmt.Errorf("end of file not confirmed: %w", err)
+	}
+	return nil
+}
+
+// Image is the prepared runtime image: one sealed memfd per boot file, which
+// QEMU boots by fd. The daemon owns it for its whole life.
+type Image struct {
+	Digest                   string
+	Firmware, Kernel, Initrd *os.File
+}
+
+// Close releases the sealed copies; safe on a partial or nil image.
+func (im *Image) Close() {
+	if im == nil {
+		return
+	}
+	for _, f := range []*os.File{im.Firmware, im.Kernel, im.Initrd} {
+		if f != nil {
+			_ = f.Close()
+		}
+	}
+}
+
+// PrepareImage seals the three boot files in dir, hashes the sealed copies and
+// refuses unless their digest equals pin. On any failure nothing stays open.
+func PrepareImage(dir, pin string) (*Image, error) {
+	if !ValidDigest(pin) {
+		return nil, fmt.Errorf("%w: pin is not 64 lowercase hex digits", ErrImage)
+	}
+	paths := ImagePaths(dir)
+	im := &Image{}
+	sums := make(map[string]FileSum, len(imageFiles))
+	for _, f := range []struct {
+		name string
+		dst  **os.File
+	}{{FileFirmware, &im.Firmware}, {FileKernel, &im.Kernel}, {FileInitrd, &im.Initrd}} {
+		sealed, sum, err := sealImageFile(f.name, paths[f.name])
+		if err != nil {
+			im.Close()
+			return nil, fmt.Errorf("%w: %s: %w", ErrImage, f.name, err)
+		}
+		*f.dst = sealed
+		sums[f.name] = sum
+	}
+	got, err := ImageDigest(sums)
+	if err != nil {
+		im.Close()
+		return nil, err
+	}
+	if got != pin {
+		im.Close()
+		return nil, fmt.Errorf("%w: digest %s does not match the pin", ErrImage, got)
+	}
+	im.Digest = got
+	return im, nil
+}
+
+// sealImageFile copies one boot file into a sealed memfd created with
+// MFD_NOEXEC_SEAL and hashes the sealed copy, which is what QEMU will read.
+func sealImageFile(name, path string) (*os.File, FileSum, error) {
+	src, size, err := openRegular(path)
+	if err != nil {
+		return nil, FileSum{}, err
+	}
+	defer func() { _ = src.Close() }()
+	if size <= 0 || size > maxImageFile[name] {
+		return nil, FileSum{}, fmt.Errorf("size %d outside 1..%d", size, maxImageFile[name])
+	}
+	fd, err := memfdCreate("spx-image", unix.MFD_CLOEXEC|unix.MFD_ALLOW_SEALING|unix.MFD_NOEXEC_SEAL)
+	if err != nil {
+		// Kernels before 6.3 lack MFD_NOEXEC_SEAL; no retry without it.
+		return nil, FileSum{}, fmt.Errorf("sealed memfd unavailable: %w", err)
+	}
+	m := os.NewFile(uintptr(fd), "spx-image")
+	fail := func(err error) (*os.File, FileSum, error) { _ = m.Close(); return nil, FileSum{}, err }
+	if err := copyExact(m, src, size); err != nil {
+		return fail(err)
+	}
+	if _, err := unix.FcntlInt(m.Fd(), unix.F_ADD_SEALS, requiredSeals); err != nil {
+		return fail(fmt.Errorf("seal memfd: %w", err))
+	}
+	seals, err := unix.FcntlInt(m.Fd(), unix.F_GET_SEALS, 0)
+	if err != nil || seals&requiredSeals != requiredSeals {
+		return fail(errors.New("memfd seals not confirmed"))
+	}
+	var st unix.Stat_t
+	if err := unix.Fstat(int(m.Fd()), &st); err != nil || st.Size != size {
+		return fail(errors.New("sealed size differs from the source"))
+	}
+	h := sha256.New()
+	if n, err := io.Copy(h, io.NewSectionReader(m, 0, size)); err != nil || n != size {
+		return fail(errors.New("sealed copy unreadable"))
+	}
+	return m, FileSum{SHA256: hex.EncodeToString(h.Sum(nil)), Size: size}, nil
+}

spinifex/runtime/compute/lambda/testlink/image_test.go

@@ -0,0 +1,400 @@
+//go:build lambdatestlink
+
+package testlink
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/execution"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
+)
+
+var testImageBytes = map[string][]byte{FileKernel: []byte("kernel"), FileInitrd: []byte("initramfs"), FileFirmware: []byte("firmware")}
+
+// testImageDigest was computed independently of this package for testImageBytes.
+const testImageDigest = "e4784145d0ab779740de46394eb95823159428bfdb57d3b3ee5728b387692bc1"
+
+func writeImage(t *testing.T) string {
+	t.Helper()
+	dir := t.TempDir()
+	for name, b := range testImageBytes {
+		require.NoError(t, os.WriteFile(filepath.Join(dir, name), b, 0o600))
+	}
+	return dir
+}
+
+// preparedImage is a prepared copy of testImageBytes, closed at test end.
+func preparedImage(t *testing.T) *Image {
+	t.Helper()
+	im, err := PrepareImage(writeImage(t), testImageDigest)
+	require.NoError(t, err)
+	t.Cleanup(im.Close)
+	return im
+}
+
+func openFDs(t *testing.T) int {
+	t.Helper()
+	ents, err := os.ReadDir("/proc/self/fd")
+	require.NoError(t, err)
+	return len(ents)
+}
+
+func sealedBytes(t *testing.T, f *os.File) []byte {
+	t.Helper()
+	b, err := io.ReadAll(io.NewSectionReader(f, 0, 1<<20))
+	require.NoError(t, err)
+	return b
+}
+
+func TestImageDigest_FixedVector(t *testing.T) {
+	sums := map[string]FileSum{}
+	dir := writeImage(t)
+	for name, path := range ImagePaths(dir) {
+		s, err := SumFile(name, path)
+		require.NoError(t, err)
+		sums[name] = s
+	}
+	got, err := ImageDigest(sums)
+	require.NoError(t, err)
+	require.Equal(t, testImageDigest, got)
+}
+
+func TestPrepareImage_SealsExactBytesUnderThePin(t *testing.T) {
+	dir := writeImage(t)
+	im, err := PrepareImage(dir, testImageDigest)
+	require.NoError(t, err)
+	defer im.Close()
+	require.Equal(t, testImageDigest, im.Digest)
+	// Every disk file replaced after preparation leaves the sealed copies alone.
+	for name := range testImageBytes {
+		require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte("replaced"), 0o600))
+	}
+	require.Equal(t, testImageBytes[FileFirmware], sealedBytes(t, im.Firmware))
+	require.Equal(t, testImageBytes[FileKernel], sealedBytes(t, im.Kernel))
+	require.Equal(t, testImageBytes[FileInitrd], sealedBytes(t, im.Initrd))
+	for _, f := range []*os.File{im.Firmware, im.Kernel, im.Initrd} {
+		_, err := f.WriteAt([]byte("x"), 0)
+		require.Error(t, err, "a sealed copy must refuse writes")
+	}
+}
+
+func TestPrepareImage_RefusesAndLeavesNothingOpen(t *testing.T) {
+	cases := map[string]func(t *testing.T, dir string) string{
+		"wrong pin":     func(_ *testing.T, _ string) string { return strings.Repeat("0", 64) },
+		"malformed pin": func(_ *testing.T, _ string) string { return "ABC" },
+		"changed kernel byte": func(t *testing.T, dir string) string {
+			require.NoError(t, os.WriteFile(filepath.Join(dir, FileKernel), []byte("kernex"), 0o600))
+			return testImageDigest
+		},
+		"changed initramfs byte": func(t *testing.T, dir string) string {
+			require.NoError(t, os.WriteFile(filepath.Join(dir, FileInitrd), []byte("initramfx"), 0o600))
+			return testImageDigest
+		},
+		"changed firmware byte": func(t *testing.T, dir string) string {
+			require.NoError(t, os.WriteFile(filepath.Join(dir, FileFirmware), []byte("firmwarx"), 0o600))
+			return testImageDigest
+		},
+		"missing firmware": func(t *testing.T, dir string) string {
+			require.NoError(t, os.Remove(filepath.Join(dir, FileFirmware)))
+			return testImageDigest
+		},
+		"symlinked kernel": func(t *testing.T, dir string) string {
+			p := filepath.Join(dir, FileKernel)
+			require.NoError(t, os.Rename(p, p+".real"))
+			require.NoError(t, os.Symlink(p+".real", p))
+			return testImageDigest
+		},
+		"directory as initramfs": func(t *testing.T, dir string) string {
+			p := filepath.Join(dir, FileInitrd)
+			require.NoError(t, os.Remove(p))
+			require.NoError(t, os.Mkdir(p, 0o700))
+			return testImageDigest
+		},
+		"FIFO as initramfs": func(t *testing.T, dir string) string {
+			p := filepath.Join(dir, FileInitrd)
+			require.NoError(t, os.Remove(p))
+			require.NoError(t, unix.Mkfifo(p, 0o600))
+			return testImageDigest
+		},
+		"firmware over its cap": func(t *testing.T, dir string) string {
+			require.NoError(t, os.Truncate(filepath.Join(dir, FileFirmware), maxImageFile[FileFirmware]+1))
+			return testImageDigest
+		},
+	}
+	for name, setup := range cases {
+		t.Run(name, func(t *testing.T) {
+			dir := writeImage(t)
+			pin := setup(t, dir)
+			before := openFDs(t)
+			done := make(chan error, 1)
+			go func() {
+				im, err := PrepareImage(dir, pin)
+				im.Close()
+				done <- err
+			}()
+			select {
+			case err := <-done:
+				require.ErrorIs(t, err, ErrImage)
+			case <-time.After(10 * time.Second):
+				t.Fatal("preparation blocked")
+			}
+			require.Equal(t, before, openFDs(t), "a refused preparation leaves a descriptor open")
+		})
+	}
+}
+
+// readResult is a reader whose single read returns n bytes and err.
+type readResult struct {
+	n   int
+	err error
+}
+
+func (r readResult) Read(p []byte) (int, error) { return r.n, r.err }
+
+// copyExact passes only on exactly n bytes followed by a clean end of file.
+func TestCopyExact_OnlyExactLengthAndCleanEndPass(t *testing.T) {
+	require.NoError(t, copyExact(io.Discard, strings.NewReader("abc"), 3))
+	require.Error(t, copyExact(io.Discard, strings.NewReader("abcd"), 3), "file grew")
+	require.Error(t, copyExact(io.Discard, strings.NewReader("ab"), 3), "file shrank")
+	require.Error(t, copyExact(io.Discard, io.MultiReader(strings.NewReader("abc"), readResult{1, io.EOF}), 3))
+	require.Error(t, copyExact(io.Discard, io.MultiReader(strings.NewReader("abc"), readResult{0, errors.New("eio")}), 3))
+	require.Error(t, copyExact(io.Discard, io.MultiReader(strings.NewReader("abc"), readResult{0, nil}), 3))
+}
+
+// The digest tool and the daemon select the same three files in one directory,
+// so firmware outside it cannot change what boots.
+func TestImagePaths_PairDigestToolAndDaemon(t *testing.T) {
+	dir := writeImage(t)
+	sums := map[string]FileSum{}
+	for name, path := range ImagePaths(dir) {
+		require.Equal(t, dir, filepath.Dir(path))
+		s, err := SumFile(name, path)
+		require.NoError(t, err)
+		sums[name] = s
+	}
+	pin, err := ImageDigest(sums)
+	require.NoError(t, err)
+	im, err := PrepareImage(dir, pin)
+	require.NoError(t, err)
+	im.Close()
+	require.NoError(t, os.WriteFile(filepath.Join(filepath.Dir(dir), FileFirmware), []byte("shared"), 0o600))
+	im, err = PrepareImage(dir, pin)
+	require.NoError(t, err, "a firmware file beside the directory is not part of the image")
+	im.Close()
+}
+
+// The image fds are 3 to 5 and the hard-kill ack and initializer 6 and 7; an
+// inherited file that would shift them is refused, and every launch reuses the
+// daemon's still-open sealed copies.
+func TestArmLaunch_DescriptorOrder(t *testing.T) {
+	im := preparedImage(t)
+	lib, err := os.CreateTemp(t.TempDir(), "lib")
+	require.NoError(t, err)
+	defer lib.Close()
+	for range 2 {
+		cmd := exec.Command("true")
+		hk, err := armLaunch(cmd, im, lib, time.Hour)
+		require.NoError(t, err)
+		require.Equal(t, []*os.File{im.Firmware, im.Kernel, im.Initrd, hk.w, lib}, cmd.ExtraFiles)
+		require.Contains(t, cmd.Env, "LD_PRELOAD=/proc/self/fd/7")
+		require.Contains(t, cmd.Env, "SPX_HARDKILL_ACK_FD=6")
+		require.NoError(t, cmd.Run())
+		hk.close()
+		for _, f := range []*os.File{im.Firmware, im.Kernel, im.Initrd} {
+			var st unix.Stat_t
+			require.NoError(t, unix.Fstat(int(f.Fd()), &st), "the daemon's sealed copy was closed by a launch")
+		}
+	}
+	cmd := exec.Command("true")
+	cmd.ExtraFiles = []*os.File{lib}
+	_, err = armLaunch(cmd, im, lib, time.Hour)
+	require.Error(t, err)
+	_, err = armLaunch(exec.Command("true"), nil, lib, time.Hour)
+	require.Error(t, err)
+}
+
+// The VM command's firmware, kernel and initramfs arguments name only the
+// sealed-copy fds; whether QEMU boots them is shown only by a KVM run.
+func TestPrepare_ImageArgumentsNameOnlyTheSealedFDs(t *testing.T) {
+	req := baseRequest("https://example.invalid/code", []byte("zip"))
+	req.Grant = execution.Grant{GrantID: "g-1", AttemptID: req.AttemptID, EnvironmentID: "env-x", Generation: 1, Node: testNode}
+	cfg, _, ln, err := (&Daemon{HostIP: "127.0.0.1"}).prepare(req, []byte("zip"), filepath.Join(t.TempDir(), "a"), "tap0", "02:00:00:00:00:01", "10.0.0.2")
+	require.NoError(t, err)
+	defer ln.Close()
+	cmd, err := cfg.Execute()
+	require.NoError(t, err)
+	args := strings.Join(cmd.Args, " ")
+	for _, want := range []string{"-bios /proc/self/fd/3", "-kernel /proc/self/fd/4", "-initrd /proc/self/fd/5"} {
+		require.Contains(t, args, want)
+	}
+}
+
+// Without a prepared image a request is refused before the slot or the grant
+// is touched.
+func TestHandle_NoPreparedImageFailsClosed(t *testing.T) {
+	code := []byte("zip bytes")
+	srv := codeServer(t, code)
+	g := grant(t, code)
+	ips := &fakeIPs{}
+	d := g.daemon(&Daemon{IPs: ips, Taps: &fakeTaps{}, HTTP: srv.Client(), WorkDir: t.TempDir()})
+	d.Image = nil
+	rep := d.Handle(context.Background(), g.request(srv.URL, code))
+	require.Equal(t, OutcomeRefused, rep.Outcome)
+	require.Contains(t, rep.Error, "runtime image not prepared")
+	require.False(t, d.busy.Load())
+	require.Empty(t, ips.allocated)
+	require.Equal(t, g.att.State, g.state(t))
+}
+
+// With the boot files removed after preparation and hardkill.so kept beside
+// them, Handle still checks the initializer and builds the VM command; the
+// fixed deadline stops it before QEMU would start.
+func TestHandle_BootFilesRemovedAfterPreparation(t *testing.T) {
+	code := []byte("zip bytes")
+	srv := codeServer(t, code)
+	g := grant(t, code)
+	dir := writeImage(t)
+	lib, err := os.ReadFile(g.lib)
+	require.NoError(t, err)
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "hardkill.so"), lib, 0o600))
+	im, err := PrepareImage(dir, testImageDigest)
+	require.NoError(t, err)
+	t.Cleanup(im.Close)
+	for _, path := range ImagePaths(dir) {
+		require.NoError(t, os.Remove(path))
+	}
+	ips, taps := &fakeIPs{}, &fakeTaps{}
+	d := g.daemon(&Daemon{IPs: ips, Taps: taps, HTTP: srv.Client(), WorkDir: t.TempDir(), HostIP: "127.0.0.1"})
+	d.Image, d.Initializer = im, filepath.Join(dir, "hardkill.so")
+	req := g.request(srv.URL, code)
+	var parsed Request
+	require.NoError(t, json.Unmarshal(req, &parsed))
+	stepBoot(t, time.Hour, hardDeadlineFrom(time.Hour, parsed))
+
+	rep := d.Handle(context.Background(), req)
+	require.Equal(t, OutcomeRefused, rep.Outcome)
+	require.Equal(t, ErrDeadlineExpired.Error(), rep.Error, "refused for a reason other than the deadline")
+	require.Len(t, taps.removed, 1)
+	require.Equal(t, execution.StateEnded, g.state(t))
+}
+
+// An oversized boot file is refused from its fstat size before any memfd is
+// created, so a huge or sparse file is never copied into memory.
+func TestPrepare_OversizedFileRefusedBeforeAnyMemfd(t *testing.T) {
+	dir := writeImage(t)
+	require.NoError(t, os.Truncate(ImagePaths(dir)[FileFirmware], maxImageFile[FileFirmware]+1))
+	calls := 0
+	old := memfdCreate
+	t.Cleanup(func() { memfdCreate = old })
+	memfdCreate = func(name string, flags int) (int, error) {
+		calls++
+		return old(name, flags)
+	}
+	_, err := PrepareImage(dir, testImageDigest)
+	require.ErrorIs(t, err, ErrImage)
+	require.Contains(t, err.Error(), "outside 1..")
+	require.Zero(t, calls, "memfd created before the size cap refused the file")
+}
+
+// The digest tool and the daemon select and hash the same three files from
+// one directory, so the printed digest is the pin the daemon accepts.
+func TestDirDigest_PairsWithPrepare(t *testing.T) {
+	dir := writeImage(t)
+	got, err := DirDigest(dir)
+	require.NoError(t, err)
+	require.Equal(t, testImageDigest, got)
+	im, err := PrepareImage(dir, got)
+	require.NoError(t, err)
+	im.Close()
+
+	require.NoError(t, os.Remove(ImagePaths(dir)[FileFirmware]))
+	_, err = DirDigest(dir)
+	require.ErrorIs(t, err, ErrImage)
+	_, err = PrepareImage(dir, got)
+	require.ErrorIs(t, err, ErrImage)
+}
+
+// Firmware comes only from the image directory: a firmware file beside it,
+// where the shared microvm qboot.rom lives, changes nothing.
+func TestDirDigest_IgnoresFirmwareOutsideTheDirectory(t *testing.T) {
+	parent := t.TempDir()
+	dir := filepath.Join(parent, "lambda-testlink")
+	require.NoError(t, os.Mkdir(dir, 0o700))
+	for name, b := range testImageBytes {
+		require.NoError(t, os.WriteFile(filepath.Join(dir, name), b, 0o600))
+	}
+	require.NoError(t, os.WriteFile(filepath.Join(parent, FileFirmware), []byte("other firmware"), 0o600))
+	got, err := DirDigest(dir)
+	require.NoError(t, err)
+	require.Equal(t, testImageDigest, got)
+	im, err := PrepareImage(dir, testImageDigest)
+	require.NoError(t, err)
+	im.Close()
+}
+
+// Each loader step that can fail on a real kernel fails the kernel file, the
+// second one sealed, so the firmware copy already made must be closed too.
+func TestPrepareImage_LoaderStepFailuresLeaveNothingOpen(t *testing.T) {
+	orig := memfdCreate
+	cases := map[string]struct {
+		create func(t *testing.T, dir, name string, flags int) (int, error)
+		want   string
+	}{
+		"file grows after fstat": {func(t *testing.T, dir, name string, flags int) (int, error) {
+			f, err := os.OpenFile(filepath.Join(dir, FileKernel), os.O_APPEND|os.O_WRONLY, 0)
+			require.NoError(t, err)
+			_, err = f.Write([]byte("x"))
+			require.NoError(t, errors.Join(err, f.Close()))
+			return orig(name, flags)
+		}, "file longer than its size"},
+		"file shrinks after fstat": {func(t *testing.T, dir, name string, flags int) (int, error) {
+			require.NoError(t, os.Truncate(filepath.Join(dir, FileKernel), 1))
+			return orig(name, flags)
+		}, "copied"},
+		"copy into the memfd fails": {func(t *testing.T, _, _ string, _ int) (int, error) {
+			return unix.Open("/dev/null", unix.O_RDONLY|unix.O_CLOEXEC, 0)
+		}, "copied"},
+		"sealing refused": {func(t *testing.T, _, name string, _ int) (int, error) {
+			return orig(name, unix.MFD_CLOEXEC)
+		}, "seal memfd"},
+		"sealed size differs": {func(t *testing.T, _, name string, flags int) (int, error) {
+			fd, err := orig(name, flags)
+			require.NoError(t, err)
+			_, err = unix.Write(fd, []byte("x"))
+			require.NoError(t, err)
+			return fd, nil
+		}, "sealed size differs"},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			dir := writeImage(t)
+			calls := 0
+			t.Cleanup(func() { memfdCreate = orig })
+			memfdCreate = func(n string, flags int) (int, error) {
+				calls++
+				if calls == 2 {
+					return tc.create(t, dir, n, flags)
+				}
+				return orig(n, flags)
+			}
+			before := openFDs(t)
+			im, err := PrepareImage(dir, testImageDigest)
+			require.ErrorIs(t, err, ErrImage)
+			require.Nil(t, im)
+			require.Contains(t, err.Error(), FileKernel)
+			require.Contains(t, err.Error(), tc.want)
+			require.Equal(t, 2, calls, "failure did not come from the kernel file")
+			require.Equal(t, before, openFDs(t), "a failed step leaves a descriptor open")
+		})
+	}
+}

spinifex/runtime/compute/lambda/testlink/main_test.go

@@ -0,0 +1,35 @@
+//go:build lambdatestlink
+
+package testlink
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// fakeQEMUExit is the stub's exit status, so a test that execs it sees a failed VM, never a launch.
+const fakeQEMUExit = 97
+
+// TestMain puts a stub qemu-system-x86_64 at the front of PATH for the whole run, so the VM command
+// builds on hosts without QEMU; hostos caches the resolved path, so the stub outlives every test.
+func TestMain(m *testing.M) {
+	dir, err := os.MkdirTemp("", "testlink-fake-qemu-")
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "fake qemu dir:", err)
+		os.Exit(1)
+	}
+	stub := fmt.Sprintf("#!/bin/sh\necho 'fake qemu-system-x86_64: not a VM' >&2\nexit %d\n", fakeQEMUExit)
+	if err := os.WriteFile(filepath.Join(dir, "qemu-system-x86_64"), []byte(stub), 0o755); err != nil {
+		fmt.Fprintln(os.Stderr, "fake qemu stub:", err)
+		os.Exit(1)
+	}
+	if err := os.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")); err != nil {
+		fmt.Fprintln(os.Stderr, "fake qemu PATH:", err)
+		os.Exit(1)
+	}
+	code := m.Run()
+	_ = os.RemoveAll(dir)
+	os.Exit(code)
+}

spinifex/runtime/compute/lambda/testlink/recover.go

@@ -0,0 +1,282 @@
+//go:build lambdatestlink
+
+package testlink
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"golang.org/x/sys/unix"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/execution"
+	"github.com/mulgadc/spinifex/spinifex/domains/network/host"
+)
+
+// pidfsMagic is PID_FS_MAGIC, the filesystem a pidfd lives on when the kernel
+// gives each process its own pidfd inode.
+const pidfsMagic = 0x50494446
+
+// recoverKillWait bounds how long recovery waits for a killed VM to exit.
+const recoverKillWait = 10 * time.Second
+
+var (
+	// ErrNoPidfs: pidfds here are not on pidfs, so their inodes identify nothing.
+	ErrNoPidfs = errors.New("lambda test link: pidfds are not on pidfs")
+	// ErrLaunchNotRecorded: the owner did not confirm the launch identity.
+	ErrLaunchNotRecorded = errors.New("lambda test link: launch identity not recorded")
+	// ErrVMGone: the VM exited before its launch identity was taken.
+	ErrVMGone = errors.New("lambda test link: VM exited before its launch was recorded")
+)
+
+// pidfsInode returns the pidfs inode of pidfd. Older anon-inode pidfds all
+// share one inode, so a pidfd on any other filesystem is refused.
+func pidfsInode(pidfd int) (uint64, error) {
+	var fs unix.Statfs_t
+	if err := unix.Fstatfs(pidfd, &fs); err != nil {
+		return 0, fmt.Errorf("fstatfs pidfd: %w", err)
+	}
+	if fs.Type != pidfsMagic {
+		return 0, ErrNoPidfs
+	}
+	var st unix.Stat_t
+	if err := unix.Fstat(pidfd, &st); err != nil {
+		return 0, fmt.Errorf("fstat pidfd: %w", err)
+	}
+	if st.Ino == 0 {
+		return 0, ErrNoPidfs
+	}
+	return st.Ino, nil
+}
+
+// selfInode returns this process's pidfs inode.
+func selfInode() (uint64, error) {
+	fd, err := unix.PidfdOpen(os.Getpid(), 0)
+	if err != nil {
+		return 0, fmt.Errorf("pidfd for this process: %w", err)
+	}
+	defer func() { _ = unix.Close(fd) }()
+	return pidfsInode(fd)
+}
+
+// hostIDs is what scopes a recorded PID and pidfs inode.
+type hostIDs struct {
+	machine, boot string
+	pidns         uint64
+}
+
+func readHost() (hostIDs, error) {
+	m, err := os.ReadFile("/etc/machine-id")
+	if err != nil {
+		return hostIDs{}, fmt.Errorf("read machine ID: %w", err)
+	}
+	b, err := os.ReadFile("/proc/sys/kernel/random/boot_id")
+	if err != nil {
+		return hostIDs{}, fmt.Errorf("read boot ID: %w", err)
+	}
+	var st unix.Stat_t
+	if err := unix.Stat("/proc/self/ns/pid", &st); err != nil {
+		return hostIDs{}, fmt.Errorf("read PID namespace: %w", err)
+	}
+	h := hostIDs{machine: strings.TrimSpace(string(m)), boot: strings.TrimSpace(string(b)), pidns: st.Ino}
+	if h.machine == "" || h.boot == "" || h.pidns == 0 {
+		return hostIDs{}, errors.New("host identity incomplete")
+	}
+	return h, nil
+}
+
+// launchIdentity identifies the VM by the exact pidfd taken when it was cloned,
+// so a reaped and reused PID can never stand in for it.
+func launchIdentity(pidfd, pid int, tap host.OwnedTap) (execution.Launch, error) {
+	ino, err := pidfsInode(pidfd)
+	if err != nil {
+		return execution.Launch{}, err
+	}
+	if exited, err := pidfdExited(pidfd, 0); err != nil {
+		return execution.Launch{}, err
+	} else if exited {
+		return execution.Launch{}, ErrVMGone
+	}
+	self, err := selfInode()
+	if err != nil {
+		return execution.Launch{}, err
+	}
+	h, err := readHost()
+	if err != nil {
+		return execution.Launch{}, err
+	}
+	return execution.Launch{MachineID: h.machine, BootID: h.boot, PIDNamespace: h.pidns,
+		LauncherPID: os.Getpid(), LauncherInode: self, PID: pid, Inode: ino,
+		TapName: tap.Name, TapIndex: tap.IfIndex, TapOwner: tap.Owner}, nil
+}
+
+// pidfdExited reports whether pidfd's process has exited, waiting up to wait.
+func pidfdExited(pidfd int, wait time.Duration) (bool, error) {
+	fds := []unix.PollFd{{Fd: int32(pidfd), Events: unix.POLLIN}}
+	for {
+		n, err := unix.Poll(fds, int(wait.Milliseconds()))
+		if errors.Is(err, unix.EINTR) {
+			continue
+		}
+		if err != nil {
+			return false, fmt.Errorf("poll pidfd: %w", err)
+		}
+		if n == 0 {
+			return false, nil
+		}
+		// Only readability is the kernel's exit report; anything else proves nothing.
+		ev := fds[0].Revents
+		if ev&(unix.POLLNVAL|unix.POLLERR) != 0 || ev&unix.POLLIN == 0 {
+			return false, fmt.Errorf("pidfd poll returned unexpected events %#x", ev)
+		}
+		return true, nil
+	}
+}
+
+// processGone reports whether the process recorded as pid with pidfs inode ino
+// no longer exists. With kill set, a live match is killed through its own pidfd
+// and reported gone only once that pidfd shows the exit.
+func processGone(pid int, ino uint64, kill bool) (bool, error) {
+	fd, err := unix.PidfdOpen(pid, 0)
+	if errors.Is(err, unix.ESRCH) {
+		return true, nil
+	}
+	if err != nil {
+		return false, fmt.Errorf("pidfd for %d: %w", pid, err)
+	}
+	defer func() { _ = unix.Close(fd) }()
+	got, err := pidfsInode(fd)
+	if err != nil {
+		return false, err
+	}
+	if got != ino {
+		// The PID names another live process, so the recorded one is gone.
+		return true, nil
+	}
+	if !kill {
+		return false, nil
+	}
+	if err := unix.PidfdSendSignal(fd, unix.SIGKILL, nil, 0); err != nil && !errors.Is(err, unix.ESRCH) {
+		return false, fmt.Errorf("kill recorded VM: %w", err)
+	}
+	return pidfdExited(fd, recoverKillWait)
+}
+
+// proveStopped proves the recorded VM is gone. A recorded PID and inode mean
+// something only on the same boot and PID namespace with no launcher left, so
+// anything else, a reboot included, is held.
+func proveStopped(l execution.Launch) error {
+	h, err := readHost()
+	if err != nil {
+		return err
+	}
+	if h.boot != l.BootID || h.machine != l.MachineID {
+		return errors.New("launch recorded on another boot or host")
+	}
+	if h.pidns != l.PIDNamespace {
+		return errors.New("launch recorded in another PID namespace")
+	}
+	if gone, err := processGone(l.LauncherPID, l.LauncherInode, false); err != nil {
+		return fmt.Errorf("launcher: %w", err)
+	} else if !gone {
+		return errors.New("recorded launcher still runs")
+	}
+	gone, err := processGone(l.PID, l.Inode, true)
+	if err != nil {
+		return fmt.Errorf("VM: %w", err)
+	}
+	if !gone {
+		return errors.New("recorded VM did not exit after kill")
+	}
+	return nil
+}
+
+// RecoverFirst is the first retry interval; it doubles up to RecoverMax.
+const (
+	RecoverFirst = 5 * time.Second
+	RecoverMax   = 5 * time.Minute
+)
+
+// recoverWait sleeps between rounds; it reports false once ctx ends.
+var recoverWait = func(ctx context.Context, d time.Duration) bool {
+	t := time.NewTimer(d)
+	defer t.Stop()
+	select {
+	case <-ctx.Done():
+		return false
+	case <-t.C:
+		return true
+	}
+}
+
+// RecoverLoop retries Recover on ctx until it settles, waiting first and then
+// doubling up to ceiling. Each round takes the node's slot, so it never runs
+// beside Handle. It covers what a previous daemon left, not a later Handle.
+func (d *Daemon) RecoverLoop(ctx context.Context, first, ceiling time.Duration) {
+	wait := first
+	for {
+		if d.busy.CompareAndSwap(false, true) {
+			settled := d.Recover(ctx)
+			d.busy.Store(false)
+			if settled {
+				return
+			}
+		}
+		if !recoverWait(ctx, wait) {
+			return
+		}
+		wait = min(wait*2, ceiling)
+	}
+}
+
+// Recover proves the stop of an attempt an earlier daemon on this boot left
+// unended, then releases what it held. Anything it cannot prove stays held,
+// and the node with it. It reports whether nothing is left to recover.
+func (d *Daemon) Recover(ctx context.Context) bool {
+	cur, err := d.Grants.Current(ctx, d.Node)
+	if errors.Is(err, execution.ErrNotCurrent) {
+		return true
+	}
+	if err != nil {
+		slog.Error("Lambda test link: recovery cannot read the node record; node held", "err", err)
+		return false
+	}
+	id := cur.Grant.AttemptID
+	if !attemptIDPattern.MatchString(id) {
+		slog.Error("Lambda test link: recovery found an unrecognised attempt ID; node held")
+		return false
+	}
+	marker := filepath.Join(d.WorkDir, id+markerSuffix)
+	switch cur.State {
+	case execution.StateRunning, execution.StateAccepted, execution.StateAmbiguous:
+	case execution.StateReleased:
+		if !d.complete(ctx, cur.Grant, marker) {
+			return false
+		}
+		slog.Info("Lambda test link: recovered attempt stopped", "attempt", id)
+		return true
+	default:
+		return true
+	}
+	if cur.Launch == nil {
+		slog.Error("Lambda test link: attempt has no launch identity; held", "attempt", id)
+		return false
+	}
+	l := *cur.Launch
+	if err := proveStopped(l); err != nil {
+		slog.Error("Lambda test link: VM stop not proven at recovery; held", "attempt", id, "err", err)
+		return false
+	}
+	a := &held{id: "lambda-" + id, dir: filepath.Join(d.WorkDir, id), marker: marker, ip: true,
+		tap: host.OwnedTap{Name: l.TapName, IfIndex: l.TapIndex, Owner: l.TapOwner}}
+	if !d.release(a) || !d.finish(ctx, cur.Grant, true, marker) {
+		return false
+	}
+	slog.Info("Lambda test link: recovered attempt stopped", "attempt", id)
+	return true
+}

spinifex/runtime/compute/lambda/testlink/recover_test.go

@@ -0,0 +1,380 @@
+//go:build lambdatestlink
+
+package testlink
+
+import (
+	"context"
+	"errors"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/execution"
+	"github.com/mulgadc/spinifex/spinifex/domains/network/host"
+)
+
+const recoverAttempt = "0123456789ab"
+
+// recoverGrants answers only what recovery asks of the owner.
+type recoverGrants struct {
+	Grants
+	cur                 execution.Attempt
+	curErr              error
+	releaseErr, stopErr error
+	released, stopped   int
+}
+
+func (g *recoverGrants) Released(context.Context, execution.Grant) error {
+	g.released++
+	if g.releaseErr == nil {
+		g.cur.State = execution.StateReleased
+	}
+	return g.releaseErr
+}
+
+func (g *recoverGrants) Current(context.Context, string) (execution.Attempt, error) {
+	return g.cur, g.curErr
+}
+
+func (g *recoverGrants) Stopped(context.Context, execution.Grant) error {
+	g.stopped++
+	if g.stopErr == nil {
+		g.cur.State = execution.StateEnded
+	}
+	return g.stopErr
+}
+
+// requirePidfs skips a real-process case only when this kernel's pidfds are
+// not on pidfs; any other probe failure fails the test.
+func requirePidfs(t *testing.T) {
+	t.Helper()
+	if _, err := selfInode(); errors.Is(err, ErrNoPidfs) {
+		t.Skip("pidfds are not on pidfs on this kernel")
+	} else if err != nil {
+		t.Fatalf("probe pidfs: %v", err)
+	}
+}
+
+// child starts a long sleep and returns it with its pidfs inode.
+func child(t *testing.T) (*exec.Cmd, uint64) {
+	t.Helper()
+	requirePidfs(t)
+	fd := -1
+	cmd := exec.Command("sleep", "300")
+	cmd.SysProcAttr = &syscall.SysProcAttr{PidFD: &fd}
+	require.NoError(t, cmd.Start())
+	t.Cleanup(func() {
+		if cmd.ProcessState == nil {
+			_ = unix.PidfdSendSignal(fd, unix.SIGKILL, nil, 0)
+			_ = cmd.Wait()
+		}
+		_ = unix.Close(fd)
+	})
+	ino, err := pidfsInode(fd)
+	require.NoError(t, err)
+	return cmd, ino
+}
+
+// alive reports whether cmd's process is still running, without reaping it.
+func alive(cmd *exec.Cmd) bool {
+	var info unix.Siginfo
+	err := unix.Waitid(unix.P_PID, cmd.Process.Pid, &info, unix.WEXITED|unix.WNOHANG|unix.WNOWAIT, nil)
+	return err == nil && info.Signo == 0
+}
+
+// goneLauncher returns the PID and inode of a process that has exited.
+func goneLauncher(t *testing.T) (int, uint64) {
+	t.Helper()
+	cmd, ino := child(t)
+	require.NoError(t, cmd.Process.Kill())
+	_ = cmd.Wait()
+	return cmd.Process.Pid, ino
+}
+
+func recordedLaunch(t *testing.T, vm *exec.Cmd, ino uint64) execution.Launch {
+	t.Helper()
+	h, err := readHost()
+	require.NoError(t, err)
+	lpid, lino := goneLauncher(t)
+	return execution.Launch{MachineID: h.machine, BootID: h.boot, PIDNamespace: h.pidns,
+		LauncherPID: lpid, LauncherInode: lino, PID: vm.Process.Pid, Inode: ino,
+		TapName: "lm" + recoverAttempt, TapIndex: 7, TapOwner: "lambda-" + recoverAttempt}
+}
+
+type recoverRig struct {
+	d      *Daemon
+	grants *recoverGrants
+	ips    *fakeIPs
+	taps   *fakeTaps
+	marker string
+}
+
+func newRecoverRig(t *testing.T, state string, l *execution.Launch) *recoverRig {
+	t.Helper()
+	work := t.TempDir()
+	marker := filepath.Join(work, recoverAttempt+markerSuffix)
+	require.NoError(t, os.WriteFile(marker, nil, 0o600))
+	require.NoError(t, os.MkdirAll(filepath.Join(work, recoverAttempt), 0o700))
+	g := &recoverGrants{cur: execution.Attempt{State: state, Launch: l,
+		Grant: execution.Grant{GrantID: "g", AttemptID: recoverAttempt, Node: testNode}}}
+	ips, taps := &fakeIPs{}, &fakeTaps{}
+	return &recoverRig{d: &Daemon{Node: testNode, WorkDir: work, Grants: g, IPs: ips, Taps: taps},
+		grants: g, ips: ips, taps: taps, marker: marker}
+}
+
+func (r *recoverRig) assertHeld(t *testing.T) {
+	t.Helper()
+	assert.FileExists(t, r.marker)
+	assert.DirExists(t, filepath.Join(r.d.WorkDir, recoverAttempt))
+	assert.Empty(t, r.ips.released)
+	assert.Empty(t, r.taps.removed)
+	assert.Zero(t, r.grants.released)
+	assert.Zero(t, r.grants.stopped)
+}
+
+func (r *recoverRig) assertRecovered(t *testing.T) {
+	t.Helper()
+	assert.NoFileExists(t, r.marker)
+	assert.NoDirExists(t, filepath.Join(r.d.WorkDir, recoverAttempt))
+	assert.Equal(t, []string{"lambda-" + recoverAttempt}, r.ips.released)
+	assert.Equal(t, []host.OwnedTap{{Name: "lm" + recoverAttempt, IfIndex: 7, Owner: "lambda-" + recoverAttempt}}, r.taps.removed)
+	assert.Equal(t, 1, r.grants.released)
+	assert.Equal(t, 1, r.grants.stopped)
+	assert.Equal(t, execution.StateEnded, r.grants.cur.State)
+}
+
+// Recovery proves nothing it cannot scope to this boot and PID namespace with
+// no launcher left, and then it signals nothing and releases nothing.
+func TestRecover_HoldsWhatItCannotProve(t *testing.T) {
+	cases := map[string]func(l *execution.Launch, vm *exec.Cmd){
+		"no launch identity": nil,
+		"another boot":       func(l *execution.Launch, _ *exec.Cmd) { l.BootID = "other-boot" },
+		"another host":       func(l *execution.Launch, _ *exec.Cmd) { l.MachineID = "other-host" },
+		"another namespace":  func(l *execution.Launch, _ *exec.Cmd) { l.PIDNamespace++ },
+		"launcher still runs": func(l *execution.Launch, _ *exec.Cmd) {
+			l.LauncherPID = os.Getpid()
+			ino, err := selfInode()
+			if err == nil {
+				l.LauncherInode = ino
+			}
+		},
+	}
+	for name, change := range cases {
+		t.Run(name, func(t *testing.T) {
+			vm, ino := child(t)
+			var l *execution.Launch
+			if change != nil {
+				rec := recordedLaunch(t, vm, ino)
+				change(&rec, vm)
+				l = &rec
+			}
+			r := newRecoverRig(t, execution.StateRunning, l)
+			r.d.Recover(context.Background())
+			r.assertHeld(t)
+			assert.True(t, alive(vm), "recovery signalled the process")
+		})
+	}
+}
+
+// A recorded VM whose PID has no process, or names a process with another
+// pidfs inode, is gone: nothing is signalled and the attempt is released.
+func TestRecover_ProvesAGoneVMWithoutSignalling(t *testing.T) {
+	t.Run("no process", func(t *testing.T) {
+		vm, ino := child(t)
+		l := recordedLaunch(t, vm, ino)
+		require.NoError(t, vm.Process.Kill())
+		_ = vm.Wait()
+		r := newRecoverRig(t, execution.StateAccepted, &l)
+		r.d.Recover(context.Background())
+		r.assertRecovered(t)
+	})
+	t.Run("PID now another process", func(t *testing.T) {
+		vm, ino := child(t)
+		l := recordedLaunch(t, vm, ino)
+		l.Inode = ino + 1
+		r := newRecoverRig(t, execution.StateRunning, &l)
+		r.d.Recover(context.Background())
+		r.assertRecovered(t)
+		assert.True(t, alive(vm), "recovery signalled a process that was not the VM")
+	})
+}
+
+// The recorded VM still running is killed through its own pidfd, and the
+// attempt released only once that pidfd reports the exit.
+func TestRecover_KillsTheRecordedVMBeforeReleasing(t *testing.T) {
+	vm, ino := child(t)
+	l := recordedLaunch(t, vm, ino)
+	r := newRecoverRig(t, execution.StateAmbiguous, &l)
+	r.d.Recover(context.Background())
+	r.assertRecovered(t)
+	assert.False(t, alive(vm))
+	err := vm.Wait()
+	var exit *exec.ExitError
+	require.ErrorAs(t, err, &exit)
+	assert.Equal(t, syscall.SIGKILL, exit.Sys().(syscall.WaitStatus).Signal())
+}
+
+// Each step waits on the one before, and a later recovery finishes from
+// whichever side of a failed step the attempt was left on.
+func TestRecover_PartialProgressIsRetried(t *testing.T) {
+	vm, ino := child(t)
+	l := recordedLaunch(t, vm, ino)
+	require.NoError(t, vm.Process.Kill())
+	_ = vm.Wait()
+
+	r := newRecoverRig(t, execution.StateRunning, &l)
+	r.ips.releaseErr = errors.New("kv unavailable")
+	r.d.Recover(context.Background())
+	assert.FileExists(t, r.marker)
+	assert.Zero(t, r.grants.released, "release recorded with the address unreleased")
+
+	r.ips.releaseErr = nil
+	r.grants.releaseErr = execution.ErrOutcomeUnknown
+	r.d.Recover(context.Background())
+	assert.FileExists(t, r.marker, "marker removed without a confirmed release")
+	assert.Zero(t, r.grants.stopped)
+
+	r.grants.releaseErr = nil
+	r.grants.stopErr = execution.ErrOutcomeUnknown
+	r.d.Recover(context.Background())
+	assert.NoFileExists(t, r.marker)
+	assert.Equal(t, execution.StateReleased, r.grants.cur.State, "node freed without a confirmed stop")
+
+	// Released with the marker already gone: recovery only frees the node.
+	r.grants.stopErr = nil
+	released := r.grants.released
+	r.d.Recover(context.Background())
+	assert.Equal(t, execution.StateEnded, r.grants.cur.State)
+	assert.Equal(t, released, r.grants.released)
+}
+
+// The launch identity comes from the exact clone-time pidfd, and an exited VM
+// is refused before anything is recorded.
+func TestLaunchIdentity(t *testing.T) {
+	requirePidfs(t)
+	fd := -1
+	cmd := exec.Command("sleep", "300")
+	cmd.SysProcAttr = &syscall.SysProcAttr{PidFD: &fd}
+	require.NoError(t, cmd.Start())
+	defer func() { _ = unix.Close(fd) }()
+
+	l, err := launchIdentity(fd, cmd.Process.Pid, host.OwnedTap{Name: "lmx", IfIndex: 3, Owner: "lambda-x"})
+	require.NoError(t, err)
+	want, err := pidfsInode(fd)
+	require.NoError(t, err)
+	assert.Equal(t, want, l.Inode)
+	assert.Equal(t, cmd.Process.Pid, l.PID)
+	assert.Equal(t, os.Getpid(), l.LauncherPID)
+	assert.NotEqual(t, l.Inode, l.LauncherInode)
+	assert.Equal(t, "lmx", l.TapName)
+
+	require.NoError(t, unix.PidfdSendSignal(fd, unix.SIGKILL, nil, 0))
+	exited, err := pidfdExited(fd, 5*time.Second)
+	require.NoError(t, err)
+	require.True(t, exited)
+	_, err = launchIdentity(fd, cmd.Process.Pid, host.OwnedTap{})
+	assert.ErrorIs(t, err, ErrVMGone)
+	_ = cmd.Wait()
+}
+
+// A descriptor that is not on pidfs has no usable identity, so it is refused
+// rather than compared.
+func TestPidfsInode_RefusesANonPidfsDescriptor(t *testing.T) {
+	f, err := os.Open(os.DevNull)
+	require.NoError(t, err)
+	defer func() { _ = f.Close() }()
+	_, err = pidfsInode(int(f.Fd()))
+	assert.ErrorIs(t, err, ErrNoPidfs)
+}
+
+// An invalid descriptor is never read as an exit.
+func TestPidfdExited_InvalidDescriptorIsNotExit(t *testing.T) {
+	exited, err := pidfdExited(1<<20, 0)
+	assert.Error(t, err)
+	assert.False(t, exited)
+}
+
+// A transient failure is retried until recovery converges.
+func TestRecoverLoop_RetriesATransientFailure(t *testing.T) {
+	vm, ino := child(t)
+	l := recordedLaunch(t, vm, ino)
+	require.NoError(t, vm.Process.Kill())
+	_ = vm.Wait()
+	r := newRecoverRig(t, execution.StateRunning, &l)
+	// Far more failed rounds than any fixed bound, as in a long KV outage.
+	r.ips.releaseFails = 500
+	var waits []time.Duration
+	prev := recoverWait
+	recoverWait = func(_ context.Context, d time.Duration) bool { waits = append(waits, d); return true }
+	t.Cleanup(func() { recoverWait = prev })
+
+	r.d.RecoverLoop(context.Background(), time.Second, 8*time.Second)
+	assert.Equal(t, execution.StateEnded, r.grants.cur.State)
+	assert.NoFileExists(t, r.marker)
+	assert.Len(t, r.ips.released, 501)
+	require.Len(t, waits, 500)
+	assert.Equal(t, []time.Duration{time.Second, 2 * time.Second, 4 * time.Second, 8 * time.Second, 8 * time.Second}, waits[:5])
+	assert.Equal(t, 8*time.Second, waits[499], "the interval is capped")
+	assert.False(t, r.d.busy.Load(), "recovery left the node slot taken")
+}
+
+// The loop ends when its context does, leaving the attempt held.
+func TestRecoverLoop_StopsWithItsContext(t *testing.T) {
+	vm, ino := child(t)
+	l := recordedLaunch(t, vm, ino)
+	require.NoError(t, vm.Process.Kill())
+	_ = vm.Wait()
+	r := newRecoverRig(t, execution.StateRunning, &l)
+	r.ips.releaseFails = 1 << 30
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	done := make(chan struct{})
+	go func() { r.d.RecoverLoop(ctx, time.Hour, time.Hour); close(done) }()
+	select {
+	case <-done:
+	case <-time.After(5 * time.Second):
+		t.Fatal("RecoverLoop did not stop with its context")
+	}
+	assert.NotEqual(t, execution.StateEnded, r.grants.cur.State)
+	assert.FileExists(t, r.marker)
+}
+
+// A round never runs while a request holds the node slot.
+func TestRecoverLoop_NeverRunsBesideARequest(t *testing.T) {
+	vm, ino := child(t)
+	l := recordedLaunch(t, vm, ino)
+	require.NoError(t, vm.Process.Kill())
+	_ = vm.Wait()
+	r := newRecoverRig(t, execution.StateRunning, &l)
+	r.d.busy.Store(true)
+	rounds := 0
+	prev := recoverWait
+	recoverWait = func(context.Context, time.Duration) bool { rounds++; return rounds < 3 }
+	t.Cleanup(func() { recoverWait = prev })
+
+	r.d.RecoverLoop(context.Background(), time.Second, time.Second)
+	r.assertHeld(t)
+}
+
+// Recovery uses only the owner record, taps and addresses: a held attempt is
+// released on a daemon whose missing pin and initializer refuse serving.
+func TestRecover_DoesNotNeedServingDependencies(t *testing.T) {
+	vm, ino := child(t)
+	l := recordedLaunch(t, vm, ino)
+	require.NoError(t, vm.Process.Kill())
+	_ = vm.Wait()
+	r := newRecoverRig(t, execution.StateRunning, &l)
+	require.Error(t, r.d.CheckInitializer(), "serving check should refuse without a pin")
+	require.Empty(t, r.d.HostIP)
+	require.Nil(t, r.d.HTTP)
+	r.d.Recover(context.Background())
+	r.assertRecovered(t)
+}

spinifex/runtime/compute/lambda/testlink/seam_guest.go

@@ -0,0 +1,80 @@
+//go:build lambdatestlink && lambdatestlinkseam
+
+package testlink
+
+import (
+	"context"
+	"encoding/json"
+	"math"
+	"net"
+	"os"
+	"path/filepath"
+	"time"
+
+	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/guestconn"
+)
+
+// HandleWithGuest answers one encoded request through Handle's grant, fetch,
+// session and finish steps, serving the session to guest on loopback instead
+// of a launched VM. It exists only for composed tests.
+func (d *Daemon) HandleWithGuest(ctx context.Context, data []byte, guest func(context.Context, net.Conn)) Reply {
+	var req Request
+	if err := json.Unmarshal(data, &req); err != nil {
+		return refused("unreadable request")
+	}
+	if err := CheckRequest(req); err != nil {
+		return refused(err.Error())
+	}
+	rep, _ := d.attempt(ctx, req, func([]byte) (Reply, bool, string) {
+		r, err := d.loopbackSession(ctx, req, guest)
+		return sessionReply(r, err), true, ""
+	})
+	return rep
+}
+
+func (d *Daemon) loopbackSession(ctx context.Context, req Request, guest func(context.Context, net.Conn)) (Reply, error) {
+	gen, err := guestconn.NewGeneration(lambdav1.Scope(scope(req)), time.Hour)
+	if err != nil {
+		return Reply{}, err
+	}
+	ln, err := net.ListenTCP("tcp", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1)})
+	if err != nil {
+		return Reply{}, err
+	}
+	defer func() { _ = ln.Close() }()
+	dir, err := os.MkdirTemp("", "lambda-seam-")
+	if err != nil {
+		return Reply{}, err
+	}
+	defer func() { _ = os.RemoveAll(dir) }()
+	guestDir := filepath.Join(dir, "guest")
+	for name, b := range gen.GuestEntries(ln.Addr().String()) {
+		if err := os.MkdirAll(filepath.Join(guestDir, name), 0o700); err != nil {
+			return Reply{}, err
+		}
+		if err := os.WriteFile(filepath.Join(guestDir, name, "raw"), b, 0o600); err != nil {
+			return Reply{}, err
+		}
+	}
+	id, err := hostlink.LoadIdentity(guestDir)
+	if err != nil {
+		return Reply{}, err
+	}
+	workDir := filepath.Join(dir, "work")
+	if err := os.Mkdir(workDir, 0o700); err != nil {
+		return Reply{}, err
+	}
+	gctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	go func() {
+		conn, err := hostlink.Dial(gctx, id)
+		if err != nil {
+			return
+		}
+		defer func() { _ = conn.Close() }()
+		guest(gctx, conn)
+	}()
+	return d.session(ctx, req, gen, ln, workDir, time.Duration(math.MaxInt64), d.Grants.Accept)
+}

spinifex/runtime/compute/lambda/testlink/testlink_test.go

@@ -0,0 +1,449 @@
+//go:build lambdatestlink
+
+package testlink
+
+import (
+	"context"
+	"crypto/sha256"
+	"debug/elf"
+	"encoding/binary"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/execution"
+	"github.com/mulgadc/spinifex/spinifex/domains/network/host"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/stretchr/testify/require"
+)
+
+type fakeIPs struct {
+	allocErr, releaseErr error
+	releaseFails         int
+	allocated, released  []string
+}
+
+func (f *fakeIPs) Allocate(id string) (string, error) {
+	f.allocated = append(f.allocated, id)
+	return "10.15.8.250", f.allocErr
+}
+func (f *fakeIPs) ReleaseChecked(id string) error {
+	f.released = append(f.released, id)
+	if f.releaseFails > 0 {
+		f.releaseFails--
+		return errors.New("kv unavailable")
+	}
+	return f.releaseErr
+}
+
+type fakeTaps struct {
+	createErr, removeErr error
+	created              host.OwnedTap
+	removed              []host.OwnedTap
+}
+
+func (f *fakeTaps) CreateOwnedTap(spec vm.TapSpec, owner string) (host.OwnedTap, error) {
+	if f.createErr != nil {
+		return host.OwnedTap{}, f.createErr
+	}
+	f.created = host.OwnedTap{Name: spec.Name, IfIndex: 41, Owner: owner}
+	return f.created, nil
+}
+
+func (f *fakeTaps) RemoveOwnedTap(tap host.OwnedTap) error {
+	f.removed = append(f.removed, tap)
+	return f.removeErr
+}
+
+func codeServer(t *testing.T, body []byte) *httptest.Server {
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write(body) }))
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+const testNode = "node1"
+
+// granted is a real owner holding one issued grant for the test request.
+type granted struct {
+	store       *execution.Store
+	att         execution.Attempt
+	lib, libSHA string
+	image       *Image
+}
+
+func grant(t *testing.T, code []byte) *granted {
+	t.Helper()
+	_, nc, _ := testutil.StartTestJetStream(t)
+	store, err := execution.Open(context.Background(), testutil.NewJetStream(t, nc))
+	require.NoError(t, err)
+	att, err := store.Issue(context.Background(), execution.Attempt{
+		Grant:    execution.Grant{AttemptID: "0123456789ab", EnvironmentID: "env-0123456789ab", Generation: 1, Node: testNode},
+		Identity: baseRequest("", code).Identity(),
+		LaunchBy: time.Now().Add(time.Minute),
+	})
+	require.NoError(t, err)
+	lib := filepath.Join(t.TempDir(), "hardkill.so")
+	elfLib := elfHeader(elfMachine[runtime.GOARCH])
+	require.NoError(t, os.WriteFile(lib, elfLib, 0o600))
+	sum := sha256.Sum256(elfLib)
+	return &granted{store: store, att: att, lib: lib, libSHA: hex.EncodeToString(sum[:]), image: preparedImage(t)}
+}
+
+func (g *granted) state(t *testing.T) string {
+	t.Helper()
+	a, err := g.store.Current(context.Background(), testNode)
+	require.NoError(t, err)
+	return a.State
+}
+
+// daemon wires the owner and a valid pinned initializer into d.
+func (g *granted) daemon(d *Daemon) *Daemon {
+	d.Node, d.Grants = testNode, g.store
+	d.Initializer, d.InitializerSHA256 = g.lib, g.libSHA
+	d.Image = g.image
+	return d
+}
+
+func baseRequest(url string, code []byte) Request {
+	sum := sha256.Sum256(code)
+	return Request{
+		AttemptID: "0123456789ab", RequestID: "r", Region: "ap-southeast-2", AccountID: "123456789012",
+		FunctionID: "fn1", RevisionID: "rev1", Handler: "bootstrap.handler", TimeoutMs: 3000, CodeURL: url, CodeSize: int64(len(code)),
+		CodeSHA256: hex.EncodeToString(sum[:]),
+	}
+}
+
+func (g *granted) request(url string, code []byte) []byte {
+	req := baseRequest(url, code)
+	req.Grant = g.att.Grant
+	b, _ := json.Marshal(req)
+	return b
+}
+
+func TestDaemon_RefusesBeforeAllocating(t *testing.T) {
+	code := []byte("zip bytes")
+	good := codeServer(t, code)
+	wrong := codeServer(t, []byte("other one"))
+	short := codeServer(t, code[:4])
+	cases := map[string]struct {
+		data  func(g *granted) []byte
+		state string
+	}{
+		// Refused before Start: the grant stays unstarted for the gateway to withdraw.
+		"plain HTTP":     {func(g *granted) []byte { return g.request(strings.Replace(good.URL, "https", "http", 1), code) }, execution.StateGranted},
+		"bad attempt ID": {func(*granted) []byte { return []byte(`{"attempt_id":"../x"}`) }, execution.StateGranted},
+		"no grant": {func(*granted) []byte {
+			b, _ := json.Marshal(baseRequest(good.URL, code))
+			return b
+		}, execution.StateGranted},
+		// Refused after Start with nothing launched: the daemon records the end.
+		"digest mismatch": {func(g *granted) []byte { return g.request(wrong.URL, code) }, execution.StateEnded},
+		"size mismatch":   {func(g *granted) []byte { return g.request(short.URL, code) }, execution.StateEnded},
+		"untrusted TLS":   {func(g *granted) []byte { return g.request(good.URL, code) }, execution.StateEnded},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			g := grant(t, code)
+			ips, taps := &fakeIPs{}, &fakeTaps{}
+			client := good.Client()
+			if name == "untrusted TLS" {
+				client = &http.Client{}
+			}
+			rep := g.daemon(&Daemon{IPs: ips, Taps: taps, HTTP: client, WorkDir: t.TempDir()}).Handle(context.Background(), tc.data(g))
+			require.Equal(t, OutcomeRefused, rep.Outcome, rep.Error)
+			require.Empty(t, ips.allocated)
+			require.NotContains(t, rep.Error, good.URL)
+			require.Equal(t, tc.state, g.state(t))
+		})
+	}
+}
+
+// The owner refuses a stale, replayed or mismatched grant before the daemon
+// fetches or allocates anything, and the grant is left as it was.
+func TestDaemon_OwnerRefusalComesBeforeAnyAllocation(t *testing.T) {
+	code := []byte("zip bytes")
+	cases := map[string]func(req *Request){
+		"stale grant ID":    func(req *Request) { req.Grant.GrantID = "superseded" },
+		"older sequence":    func(req *Request) { req.Grant.Seq = 0 },
+		"other revision":    func(req *Request) { req.RevisionID = "rev2" },
+		"other code digest": func(req *Request) { req.CodeSHA256 = strings.Repeat("0", 64) },
+		"other request":     func(req *Request) { req.RequestID = "r2" },
+		"other node":        func(req *Request) { req.Grant.Node = "node2" },
+	}
+	for name, mutate := range cases {
+		t.Run(name, func(t *testing.T) {
+			fetched := false
+			srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+				fetched = true
+				_, _ = w.Write(code)
+			}))
+			t.Cleanup(srv.Close)
+			g := grant(t, code)
+			req := baseRequest(srv.URL, code)
+			req.Grant = g.att.Grant
+			mutate(&req)
+			data, _ := json.Marshal(req)
+			ips := &fakeIPs{}
+			rep := g.daemon(&Daemon{IPs: ips, Taps: &fakeTaps{}, HTTP: srv.Client(), WorkDir: t.TempDir()}).Handle(context.Background(), data)
+			require.Equal(t, OutcomeRefused, rep.Outcome)
+			require.False(t, fetched)
+			require.Empty(t, ips.allocated)
+			require.Equal(t, execution.StateGranted, g.state(t))
+		})
+	}
+}
+
+func TestDaemon_ReplayedGrantIsRefused(t *testing.T) {
+	code := []byte("zip bytes")
+	srv := codeServer(t, code)
+	g := grant(t, code)
+	ips := &fakeIPs{}
+	taps := &fakeTaps{createErr: fmt.Errorf("%w: %w", host.ErrTapNotCreated, host.ErrTapExists)}
+	d := g.daemon(&Daemon{IPs: ips, Taps: taps, HTTP: srv.Client(), WorkDir: t.TempDir()})
+	require.Equal(t, OutcomeRefused, d.Handle(context.Background(), g.request(srv.URL, code)).Outcome)
+	require.Equal(t, execution.StateEnded, g.state(t))
+
+	ips.allocated = nil
+	rep := d.Handle(context.Background(), g.request(srv.URL, code))
+	require.Equal(t, OutcomeRefused, rep.Outcome)
+	require.Empty(t, ips.allocated, "an ended grant never starts again")
+}
+
+func TestDaemon_TapNotCreatedRemovesNothingAndResolves(t *testing.T) {
+	code := []byte("zip bytes")
+	srv := codeServer(t, code)
+	g := grant(t, code)
+	ips, taps := &fakeIPs{}, &fakeTaps{createErr: fmt.Errorf("%w: %w", host.ErrTapNotCreated, host.ErrTapExists)}
+	d := g.daemon(&Daemon{IPs: ips, Taps: taps, HTTP: srv.Client(), WorkDir: t.TempDir()})
+	rep := d.Handle(context.Background(), g.request(srv.URL, code))
+	require.Equal(t, OutcomeRefused, rep.Outcome)
+	require.Empty(t, taps.removed, "a tap this attempt did not create is never removed")
+	require.Equal(t, []string{"lambda-0123456789ab"}, ips.released)
+	id, err := d.unresolved()
+	require.NoError(t, err)
+	require.Empty(t, id)
+	require.False(t, d.busy.Load())
+	require.Equal(t, execution.StateEnded, g.state(t))
+}
+
+// A tap that will not go keeps the address, the marker, the slot and the
+// grant, and a fresh daemon on the same work directory still refuses.
+func TestDaemon_FailedCleanupHoldsTheSlotAcrossRestart(t *testing.T) {
+	code := []byte("zip bytes")
+	srv := codeServer(t, code)
+	g := grant(t, code)
+	work := t.TempDir()
+	ips, taps := &fakeIPs{}, &fakeTaps{removeErr: errors.New("busy")}
+	d := g.daemon(&Daemon{IPs: ips, Taps: taps, HTTP: srv.Client(), WorkDir: work, HostIP: "127.0.0.1"})
+	// A file where the attempt directory goes fails preparation after the tap exists.
+	require.NoError(t, os.WriteFile(filepath.Join(work, "0123456789ab"), nil, 0o600))
+	rep := d.Handle(context.Background(), g.request(srv.URL, code))
+	require.Equal(t, OutcomeRefused, rep.Outcome)
+	require.Len(t, taps.removed, 1)
+	require.Empty(t, ips.released)
+	require.True(t, d.busy.Load())
+	require.FileExists(t, filepath.Join(work, "0123456789ab"+markerSuffix))
+	require.Equal(t, execution.StateAmbiguous, g.state(t))
+
+	fresh := g.daemon(&Daemon{IPs: &fakeIPs{}, Taps: &fakeTaps{}, HTTP: srv.Client(), WorkDir: work})
+	rep = fresh.Handle(context.Background(), g.request(srv.URL, code))
+	require.Equal(t, OutcomeRefused, rep.Outcome)
+	require.Contains(t, rep.Error, "unresolved")
+	require.Equal(t, execution.StateAmbiguous, g.state(t))
+}
+
+// Neither failure proves nothing was left behind, so both keep the marker,
+// the address, the slot and the grant.
+func TestDaemon_UnprovenOwnershipHoldsTheAttempt(t *testing.T) {
+	cases := map[string]struct {
+		ips  *fakeIPs
+		taps *fakeTaps
+	}{
+		"allocation outcome unknown": {ips: &fakeIPs{allocErr: errors.New("CAS timeout")}, taps: &fakeTaps{}},
+		"tap may exist":              {ips: &fakeIPs{}, taps: &fakeTaps{createErr: errors.New("ifindex unreadable")}},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			code := []byte("zip bytes")
+			srv := codeServer(t, code)
+			g := grant(t, code)
+			work := t.TempDir()
+			d := g.daemon(&Daemon{IPs: tc.ips, Taps: tc.taps, HTTP: srv.Client(), WorkDir: work})
+			rep := d.Handle(context.Background(), g.request(srv.URL, code))
+			require.Equal(t, OutcomeRefused, rep.Outcome)
+			require.Empty(t, tc.ips.released)
+			require.Empty(t, tc.taps.removed)
+			require.True(t, d.busy.Load())
+			require.FileExists(t, filepath.Join(work, "0123456789ab"+markerSuffix))
+			require.Equal(t, execution.StateAmbiguous, g.state(t))
+		})
+	}
+}
+
+func TestDaemon_MarkerBlocksBeforeAnyAllocation(t *testing.T) {
+	code := []byte("zip bytes")
+	srv := codeServer(t, code)
+	g := grant(t, code)
+	work := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(work, "aaaaaaaaaaaa"+markerSuffix), nil, 0o600))
+	ips := &fakeIPs{}
+	d := g.daemon(&Daemon{IPs: ips, Taps: &fakeTaps{}, HTTP: srv.Client(), WorkDir: work})
+	rep := d.Handle(context.Background(), g.request(srv.URL, code))
+	require.Equal(t, OutcomeRefused, rep.Outcome)
+	require.Empty(t, ips.allocated)
+	require.False(t, d.busy.Load(), "a refused request takes no slot of its own")
+	require.Equal(t, execution.StateGranted, g.state(t))
+}
+
+func TestDaemon_OneAttemptAtATime(t *testing.T) {
+	code := []byte("x")
+	g := grant(t, code)
+	d := g.daemon(&Daemon{})
+	d.busy.Store(true)
+	rep := d.Handle(context.Background(), g.request("https://h/x", code))
+	require.Equal(t, OutcomeRefused, rep.Outcome)
+	require.Contains(t, rep.Error, "another")
+	require.Equal(t, execution.StateGranted, g.state(t))
+}
+
+func TestCheckRequest_RefusesGrantWithoutEnvironmentGeneration(t *testing.T) {
+	cases := map[string]func(*execution.Grant){
+		"no environment":       func(g *execution.Grant) { g.EnvironmentID = "" },
+		"generation zero":      func(g *execution.Grant) { g.Generation = 0 },
+		"slash in environment": func(g *execution.Grant) { g.EnvironmentID = "env/1" },
+		"newline in grant":     func(g *execution.Grant) { g.GrantID = "g\n1" },
+	}
+	for name, mutate := range cases {
+		t.Run(name, func(t *testing.T) {
+			req := baseRequest("https://example.invalid/code", []byte("zip"))
+			req.Grant = execution.Grant{GrantID: "g1", AttemptID: req.AttemptID, EnvironmentID: "env-1", Generation: 3, Node: testNode}
+			require.NoError(t, CheckRequest(req))
+			mutate(&req.Grant)
+			require.ErrorContains(t, CheckRequest(req), "grant")
+		})
+	}
+}
+
+// The guest's certificates name the granted environment generation, and the
+// grant entry it binds its run to names the same grant.
+func TestPrepare_ScopeAndGrantEntryComeFromTheGrant(t *testing.T) {
+	req := baseRequest("https://example.invalid/code", []byte("zip"))
+	req.Grant = execution.Grant{GrantID: "g-7", AttemptID: req.AttemptID, EnvironmentID: "env-x", Generation: 3, Node: testNode}
+	d := &Daemon{HostIP: "127.0.0.1"}
+	dir := filepath.Join(t.TempDir(), "attempt")
+	_, gen, ln, err := d.prepare(req, []byte("zip"), dir, "tap0", "02:00:00:00:00:01", "10.0.0.2")
+	require.NoError(t, err)
+	defer ln.Close()
+
+	entry, err := os.ReadFile(filepath.Join(dir, "fwcfg-test-grant"))
+	require.NoError(t, err)
+	require.Equal(t, "g-7\n0123456789ab\nenv-x\n3\n", string(entry))
+	handler, err := os.ReadFile(filepath.Join(dir, "fwcfg-test-handler"))
+	require.NoError(t, err)
+	require.Equal(t, req.Handler, string(handler))
+	expect := gen.GuestEntries(ln.Addr().String())[lambdav1.EntryScope]
+	require.Equal(t, "ap-southeast-2/123456789012/fn1/env-x/3", strings.TrimSpace(string(expect)))
+}
+
+// elfHeader is a bare 64-bit little-endian shared-object header for machine.
+func elfHeader(machine elf.Machine) []byte {
+	h := make([]byte, 64)
+	copy(h, elf.ELFMAG)
+	h[elf.EI_CLASS], h[elf.EI_DATA], h[elf.EI_VERSION] = byte(elf.ELFCLASS64), byte(elf.ELFDATA2LSB), byte(elf.EV_CURRENT)
+	binary.LittleEndian.PutUint16(h[16:], uint16(elf.ET_DYN))
+	binary.LittleEndian.PutUint16(h[18:], uint16(machine))
+	binary.LittleEndian.PutUint32(h[20:], uint32(elf.EV_CURRENT))
+	binary.LittleEndian.PutUint16(h[52:], 64)
+	return h
+}
+
+func TestSameArch_RefusesOtherMachinesAndNonELF(t *testing.T) {
+	require.NoError(t, sameArch(elfHeader(elfMachine[runtime.GOARCH])))
+	other := elf.EM_AARCH64
+	if runtime.GOARCH == "arm64" {
+		other = elf.EM_X86_64
+	}
+	require.ErrorContains(t, sameArch(elfHeader(other)), "built for")
+	require.ErrorContains(t, sameArch([]byte("library")), "not an ELF object")
+}
+
+// stepBoot makes the first CLOCK_BOOTTIME read return base and every later
+// read return later, so the fixed deadline is set before the jump.
+func stepBoot(t *testing.T, base, later time.Duration) {
+	t.Helper()
+	orig := bootNow
+	var calls atomic.Int32
+	bootNow = func() (time.Duration, error) {
+		if calls.Add(1) == 1 {
+			return base, nil
+		}
+		return later, nil
+	}
+	t.Cleanup(func() { bootNow = orig })
+}
+
+// A launch delayed past the deadline fixed before the grant started is refused
+// before QEMU starts, and everything it took is released.
+func TestDaemon_DelayedLaunchPastTheFixedDeadlineIsRefused(t *testing.T) {
+	code := []byte("zip bytes")
+	srv := codeServer(t, code)
+	g := grant(t, code)
+	ips, taps := &fakeIPs{}, &fakeTaps{}
+	d := g.daemon(&Daemon{IPs: ips, Taps: taps, HTTP: srv.Client(), WorkDir: t.TempDir()})
+	req := g.request(srv.URL, code)
+	var parsed Request
+	require.NoError(t, json.Unmarshal(req, &parsed))
+	stepBoot(t, time.Hour, hardDeadlineFrom(time.Hour, parsed))
+
+	rep := d.Handle(context.Background(), req)
+	require.Equal(t, OutcomeRefused, rep.Outcome)
+	require.Equal(t, ErrDeadlineExpired.Error(), rep.Error)
+	require.Equal(t, []string{"lambda-0123456789ab"}, ips.released)
+	require.Len(t, taps.removed, 1)
+	require.False(t, d.busy.Load())
+	require.Equal(t, execution.StateEnded, g.state(t))
+}
+
+// A code fetch that uses its whole allowance still leaves the function its
+// full configured timeout, plus the kill margin, before the fixed backstop.
+func TestHardDeadline_SlowFetchKeepsTheFullFunctionTimeout(t *testing.T) {
+	req := Request{TimeoutMs: 3000}
+	base := time.Hour
+	afterFetch := base + fetchTimeout
+	stepBoot(t, base, afterFetch)
+
+	boot, err := bootNow()
+	require.NoError(t, err)
+	deadline := hardDeadlineFrom(boot, req)
+
+	require.False(t, expired(deadline), "a full-allowance fetch must not expire the attempt")
+	require.Equal(t, timeoutFor(req)+HardKillMargin, deadline-afterFetch)
+}
+
+// The handler becomes the guest's _HANDLER, so it must be one AWS-shaped token.
+func TestCheckRequest_RefusesUnusableHandler(t *testing.T) {
+	for name, h := range map[string]string{
+		"empty": "", "space": "a b", "newline": "a\nb", "nul": "a\x00b", "129 characters": strings.Repeat("h", 129),
+	} {
+		t.Run(name, func(t *testing.T) {
+			req := baseRequest("https://example.invalid/code", []byte("zip"))
+			req.Grant = execution.Grant{GrantID: "g1", AttemptID: req.AttemptID, EnvironmentID: "env-1", Generation: 3, Node: testNode}
+			req.Handler = strings.Repeat("h", 128)
+			require.NoError(t, CheckRequest(req))
+			req.Handler = h
+			require.ErrorContains(t, CheckRequest(req), "handler")
+		})
+	}
+}

spinifex/runtime/daemon/daemon.go

@@ -2518,6 +2518,7 @@ func (d *Daemon) startCluster() error {
 	if err := d.subscribeAll(); err != nil {
 		return fmt.Errorf("failed to subscribe to NATS topics: %w", err)
 	}
+	d.startLambdaTestLink()
 
 	// Ochre vector store platform appliance + VectorService, backgrounded:
 	// launching the appliance can block on a cold RDS VM boot for minutes, and

spinifex/runtime/daemon/daemon_lambda_testlink.go

@@ -0,0 +1,152 @@
+//go:build lambdatestlink
+
+package daemon
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/execution"
+	"github.com/mulgadc/spinifex/spinifex/domains/network/host"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/testlink"
+	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// lambdaHardKillSHA256 pins the hard-kill initializer; tagged test builds set
+// it with -ldflags -X, and an unset pin means the test link does not serve.
+var lambdaHardKillSHA256 string
+
+// lambdaImageDigest pins the runtime image the same way; it identifies the
+// image bytes and does not authenticate where they came from.
+var lambdaImageDigest string
+
+// startLambdaTestLink serves controlled-test Lambda invocations for this node.
+// Recovery needs only the owner store, taps and addresses, so it starts even
+// when the bridge address, initializer, pin or download trust would not serve.
+func (d *Daemon) startLambdaTestLink() {
+	if d.mgmtIPAllocator == nil || d.networkPlumber == nil {
+		slog.Error("Lambda test link: address or tap cleanup unavailable; not serving or recovering")
+		return
+	}
+	taps, ok := d.networkPlumber.(testlink.Taps)
+	if !ok {
+		slog.Error("Lambda test link: plumber cannot create owned taps; not serving or recovering")
+		return
+	}
+	workDir := filepath.Join(hostprocess.RuntimeDir(), "lambda-testlink")
+	if err := os.MkdirAll(workDir, 0o700); err != nil {
+		slog.Error("Lambda test link: work directory", "err", err)
+		return
+	}
+	js, err := jetstream.New(d.natsConn)
+	if err != nil {
+		slog.Error("Lambda test link: JetStream", "err", err)
+		return
+	}
+	grants, err := execution.Open(d.ctx, js)
+	if err != nil {
+		slog.Error("Lambda test link: execution owner unavailable; not serving or recovering", "err", err)
+		return
+	}
+	httpClient, trustErr := d.lambdaTestLinkHTTP()
+	imageDir := filepath.Join(microvmDir, "lambda-testlink")
+	tl := &testlink.Daemon{
+		Node:    d.node,
+		Grants:  grants,
+		Taps:    taps,
+		IPs:     d.mgmtIPAllocator,
+		Bridge:  d.mgmtBridgeName(),
+		HostIP:  d.mgmtBridgeIP,
+		WorkDir: workDir,
+
+		Initializer:       filepath.Join(imageDir, "hardkill.so"),
+		InitializerSHA256: lambdaHardKillSHA256,
+		HTTP:              httpClient,
+	}
+	err = runLambdaTestLink(
+		func() { go tl.RecoverLoop(d.ctx, testlink.RecoverFirst, testlink.RecoverMax) },
+		func() error { return lambdaTestLinkReady(tl, imageDir, lambdaImageDigest, trustErr) },
+		func() error { _, err := tl.Serve(d.ctx, d.natsConn, d.node); return err },
+		func() { releaseLambdaTestLinkImage(tl) },
+	)
+	if err != nil {
+		slog.Error("Lambda test link: not serving new invocations; recovery continues", "err", err)
+		return
+	}
+	slog.Warn("Lambda test link serving: test-only, not for production; the guest does not verify owner grant authority",
+		"node", d.node, "image_digest", tl.Image.Digest)
+}
+
+// lambdaTestLinkReady runs the serving checks and, only once they pass,
+// prepares the pinned image into tl; a refusal leaves tl.Image nil.
+func lambdaTestLinkReady(tl *testlink.Daemon, imageDir, imagePin string, trustErr error) error {
+	if tl.HostIP == "" {
+		return errors.New("management bridge address unavailable")
+	}
+	if tl.InitializerSHA256 == "" {
+		return errors.New("hard-kill initializer digest not pinned")
+	}
+	if trustErr != nil {
+		return trustErr
+	}
+	if !testlink.ValidDigest(imagePin) {
+		return errors.New("runtime image digest not pinned")
+	}
+	if err := tl.CheckInitializer(); err != nil {
+		return err
+	}
+	im, err := testlink.PrepareImage(imageDir, imagePin)
+	if err != nil {
+		return err
+	}
+	tl.Image = im
+	return nil
+}
+
+// releaseLambdaTestLinkImage closes whatever ready prepared.
+func releaseLambdaTestLinkImage(tl *testlink.Daemon) {
+	tl.Image.Close()
+	tl.Image = nil
+}
+
+// runLambdaTestLink starts recovery before any serving check, so a failed
+// check refuses new invocations without stopping cleanup of earlier ones.
+// release drops whatever ready prepared when ready or serve fails.
+func runLambdaTestLink(startRecovery func(), ready, serve func() error, release func()) error {
+	startRecovery()
+	if err := ready(); err != nil {
+		release()
+		return err
+	}
+	if err := serve(); err != nil {
+		release()
+		return err
+	}
+	return nil
+}
+
+// lambdaTestLinkHTTP builds the code-download client trusting the cluster CA.
+func (d *Daemon) lambdaTestLinkHTTP() (*http.Client, error) {
+	roots, err := x509.SystemCertPool()
+	if err != nil {
+		return nil, fmt.Errorf("system trust store: %w", err)
+	}
+	if d.config.NATS.CACert != "" {
+		ca, err := os.ReadFile(d.config.NATS.CACert)
+		if err != nil || !roots.AppendCertsFromPEM(ca) {
+			return nil, errors.New("cluster CA unreadable")
+		}
+	}
+	return &http.Client{Transport: &http.Transport{
+		TLSClientConfig: &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12},
+	}}, nil
+}
+
+var _ testlink.Taps = (*host.OVSPlumber)(nil)

spinifex/runtime/daemon/daemon_lambda_testlink_off.go

@@ -0,0 +1,6 @@
+//go:build !lambdatestlink
+
+package daemon
+
+// Without the test link the daemon runs no Lambda code and subscribes nothing.
+func (d *Daemon) startLambdaTestLink() {}

spinifex/runtime/daemon/daemon_lambda_testlink_pin_test.go

@@ -0,0 +1,203 @@
+//go:build lambdatestlink
+
+package daemon
+
+import (
+	"crypto/sha256"
+	"debug/elf"
+	"encoding/binary"
+	"encoding/hex"
+	"errors"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/testlink"
+)
+
+// TestLambdaHardKillPin runs from make build-lambda-testlink with the same pin
+// as the built daemon: the paired initializer passes and a changed one fails.
+func TestLambdaHardKillPin(t *testing.T) {
+	lib := os.Getenv("LAMBDA_TESTLINK_INITIALIZER")
+	if lambdaHardKillSHA256 == "" || lib == "" {
+		t.Skip("run by make build-lambda-testlink with a pinned initializer")
+	}
+	if err := (&testlink.Daemon{Initializer: lib, InitializerSHA256: lambdaHardKillSHA256}).CheckInitializer(); err != nil {
+		t.Fatalf("paired initializer refused: %v", err)
+	}
+	b, err := os.ReadFile(lib)
+	if err != nil {
+		t.Fatal(err)
+	}
+	b[len(b)-1] ^= 0xff
+	changed := filepath.Join(t.TempDir(), "hardkill.so")
+	if err := os.WriteFile(changed, b, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	err = (&testlink.Daemon{Initializer: changed, InitializerSHA256: lambdaHardKillSHA256}).CheckInitializer()
+	if err == nil || !strings.Contains(err.Error(), "digest mismatch") {
+		t.Fatalf("changed initializer not refused by the pin: %v", err)
+	}
+}
+
+// TestLambdaImageDigestPin runs from make build-lambda-testlink with the same
+// ldflags as the built daemon: the pinned image digest arrives exactly as given.
+func TestLambdaImageDigestPin(t *testing.T) {
+	want := os.Getenv("LAMBDA_IMAGE_DIGEST")
+	if want == "" {
+		t.Skip("run by make build-lambda-testlink with a pinned image digest")
+	}
+	if lambdaImageDigest != want || !testlink.ValidDigest(lambdaImageDigest) {
+		t.Fatalf("pinned image digest %q, want %q", lambdaImageDigest, want)
+	}
+}
+
+// A failed serving check refuses new invocations but recovery has already
+// started, so an attempt launched before a restart is still cleaned up.
+func TestLambdaTestLinkRecoveryStartsDespiteFailedServingCheck(t *testing.T) {
+	var recovered, served, released bool
+	refused := errors.New("runtime image refused")
+	err := runLambdaTestLink(
+		func() { recovered = true },
+		func() error {
+			if !recovered {
+				t.Fatal("serving check ran before recovery started")
+			}
+			return refused
+		},
+		func() error { served = true; return nil },
+		func() { released = true },
+	)
+	if !errors.Is(err, refused) || !recovered || served || !released {
+		t.Fatalf("err=%v recovered=%v served=%v released=%v", err, recovered, served, released)
+	}
+}
+
+// What ready prepared is released when serving fails, and kept once it serves.
+func TestLambdaTestLinkReleasesPreparedImageOnlyWhenNotServing(t *testing.T) {
+	for _, tc := range []struct {
+		name     string
+		serveErr error
+	}{{"serve fails", errors.New("subscribe failed")}, {"serving", nil}} {
+		t.Run(tc.name, func(t *testing.T) {
+			released := false
+			err := runLambdaTestLink(func() {}, func() error { return nil },
+				func() error { return tc.serveErr }, func() { released = true })
+			if !errors.Is(err, tc.serveErr) || released != (tc.serveErr != nil) {
+				t.Fatalf("err=%v released=%v", err, released)
+			}
+		})
+	}
+}
+
+// readyFixture is a test-link daemon whose initializer and image both pass.
+func readyFixture(t *testing.T) (*testlink.Daemon, string, string) {
+	t.Helper()
+	dir := t.TempDir()
+	hdr := make([]byte, 64)
+	copy(hdr, "\x7fELF\x02\x01\x01")
+	binary.LittleEndian.PutUint16(hdr[16:], uint16(elf.ET_DYN))
+	binary.LittleEndian.PutUint16(hdr[18:], uint16(elf.EM_X86_64))
+	binary.LittleEndian.PutUint32(hdr[20:], 1)
+	binary.LittleEndian.PutUint16(hdr[52:], 64)
+	lib := filepath.Join(dir, "hardkill.so")
+	if err := os.WriteFile(lib, hdr, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	sum := sha256.Sum256(hdr)
+	for name, b := range map[string]string{"qboot.rom": "firmware", "vmlinuz": "kernel", "initramfs.cpio.gz": "initramfs"} {
+		if err := os.WriteFile(filepath.Join(dir, name), []byte(b), 0o600); err != nil {
+			t.Fatal(err)
+		}
+	}
+	pin, err := testlink.DirDigest(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return &testlink.Daemon{HostIP: "10.15.8.1", Initializer: lib, InitializerSHA256: hex.EncodeToString(sum[:])}, dir, pin
+}
+
+func openFDs(t *testing.T) int {
+	t.Helper()
+	ents, err := os.ReadDir("/proc/self/fd")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return len(ents)
+}
+
+// Each serving check refuses before the image is prepared, and a refused
+// preparation leaves no descriptor open.
+func TestLambdaTestLinkReadyRefusesWithNothingPrepared(t *testing.T) {
+	if runtime.GOARCH != "amd64" {
+		t.Skip("the test link supports only amd64")
+	}
+	cases := map[string]func(tl *testlink.Daemon, dir string, pin *string, trust *error){
+		"no bridge address": func(tl *testlink.Daemon, _ string, _ *string, _ *error) { tl.HostIP = "" },
+		"initializer unset": func(tl *testlink.Daemon, _ string, _ *string, _ *error) { tl.InitializerSHA256 = "" },
+		"download trust": func(_ *testlink.Daemon, _ string, _ *string, trust *error) {
+			*trust = errors.New("cluster CA unreadable")
+		},
+		"image pin missing":   func(_ *testlink.Daemon, _ string, pin *string, _ *error) { *pin = "" },
+		"image pin malformed": func(_ *testlink.Daemon, _ string, pin *string, _ *error) { *pin = strings.ToUpper(*pin) },
+		"initializer changed": func(tl *testlink.Daemon, _ string, _ *string, _ *error) {
+			tl.InitializerSHA256 = strings.Repeat("0", 64)
+		},
+		"image pin differs": func(_ *testlink.Daemon, _ string, pin *string, _ *error) { *pin = strings.Repeat("0", 64) },
+		"image file missing": func(_ *testlink.Daemon, dir string, _ *string, _ *error) {
+			if err := os.Remove(filepath.Join(dir, "vmlinuz")); err != nil {
+				panic(err)
+			}
+		},
+	}
+	for name, mutate := range cases {
+		t.Run(name, func(t *testing.T) {
+			tl, dir, pin := readyFixture(t)
+			var trust error
+			mutate(tl, dir, &pin, &trust)
+			before := openFDs(t)
+			if err := lambdaTestLinkReady(tl, dir, pin, trust); err == nil {
+				t.Fatal("ready passed")
+			}
+			if tl.Image != nil || openFDs(t) != before {
+				t.Fatalf("refusal left image=%v, fds %d -> %d", tl.Image != nil, before, openFDs(t))
+			}
+		})
+	}
+}
+
+// A prepared image is kept while serving and fully closed when the
+// subscription fails after readiness.
+func TestLambdaTestLinkServeFailureClosesThePreparedImage(t *testing.T) {
+	if runtime.GOARCH != "amd64" {
+		t.Skip("the test link supports only amd64")
+	}
+	for _, tc := range []struct {
+		name     string
+		serveErr error
+	}{{"serve fails", errors.New("subscribe failed")}, {"serving", nil}} {
+		t.Run(tc.name, func(t *testing.T) {
+			tl, dir, pin := readyFixture(t)
+			before := openFDs(t)
+			prepared := 0
+			err := runLambdaTestLink(func() {},
+				func() error { return lambdaTestLinkReady(tl, dir, pin, nil) },
+				func() error { prepared = openFDs(t) - before; return tc.serveErr },
+				func() { releaseLambdaTestLinkImage(tl) })
+			if !errors.Is(err, tc.serveErr) || prepared != 3 {
+				t.Fatalf("err=%v, prepared %d descriptors, want 3", err, prepared)
+			}
+			if tc.serveErr != nil && (tl.Image != nil || openFDs(t) != before) {
+				t.Fatalf("serve failure left image=%v, fds %d -> %d", tl.Image != nil, before, openFDs(t))
+			}
+			if tc.serveErr == nil {
+				if tl.Image == nil || openFDs(t) != before+3 {
+					t.Fatalf("serving dropped the image: fds %d -> %d", before, openFDs(t))
+				}
+				releaseLambdaTestLinkImage(tl)
+			}
+		})
+	}
+}

spinifex/runtime/daemon/daemon_mgmt_ip.go

@@ -238,18 +238,56 @@ func nextFreeIP(baseIP net.IP, rangeMin, rangeMax byte, allocated []MgmtIPEntry)
 // returned — callers (LaunchSystemInstance rollback, CleanupMgmtNetwork)
 // cannot act on a Release error, matching the pre-existing signature.
 func (a *MgmtIPAllocator) Release(instanceID string) {
+	ip, foreign, node, err := a.release(instanceID, false)
+	if err != nil && !errors.Is(err, errMgmtIPNoKV) {
+		slog.Warn("Failed to release mgmt IP in cluster KV; address remains reserved until reconciled",
+			"instance", instanceID, "err", err)
+	}
+	if foreign != "" {
+		slog.Info("Left a management IP reserved: it belongs to the node running this instance now",
+			"instance", instanceID, "ip", ip, "holder", foreign, "node", node)
+	}
+}
+
+// ReleaseChecked returns nil only once the cluster record no longer holds the address for
+// this node; any failure, another node's entry or one naming no node leaves both entries
+// in place. Callers must serialize Allocate and ReleaseChecked for the same instance ID.
+func (a *MgmtIPAllocator) ReleaseChecked(instanceID string) error {
+	ip, foreign, _, err := a.release(instanceID, true)
+	if err != nil {
+		return err
+	}
+	if foreign != "" {
+		return fmt.Errorf("mgmt IP for %s is held by node %s", instanceID, foreign)
+	}
+	a.mu.Lock()
+	if cur, ok := a.allocated[instanceID]; ok && cur == ip {
+		delete(a.allocated, instanceID)
+	}
+	a.mu.Unlock()
+	return nil
+}
+
+var errMgmtIPNoKV = errors.New("mgmt IP release unconfirmed: no cluster KV bound")
+
+var errMgmtIPOwnerless = errors.New("mgmt IP entry names no node")
+
+func (a *MgmtIPAllocator) release(instanceID string, requireOwner bool) (ip, foreign, node string, err error) {
 	a.mu.Lock()
 	ip, hadLocal := a.allocated[instanceID]
-	delete(a.allocated, instanceID)
-	jsManager, node, subnet := a.jsManager, a.node, a.subnet
+	if !requireOwner {
+		delete(a.allocated, instanceID)
+	}
+	jsManager, subnet := a.jsManager, a.subnet
+	node = a.node
 	a.mu.Unlock()
 
-	if hadLocal {
+	if hadLocal && !requireOwner {
 		slog.Debug("Management IP released locally", "instance", instanceID, "ip", ip)
 	}
 
 	if jsManager == nil {
-		return
+		return ip, "", node, errMgmtIPNoKV
 	}
 
 	// createIfAbsent=false: releasing an instance this node never allocated
@@ -262,13 +300,17 @@ func (a *MgmtIPAllocator) Release(instanceID string) {
 	// it up for the next instance anywhere — and this path runs on exactly the
 	// node that should not: the one dropping a guest another node now runs,
 	// whether it was fenced or found superseded on its return.
-	foreign := ""
-	_, err := updateMgmtIPAMWithRetry(jsManager, subnet, func(r *MgmtIPRecord) {
-		foreign = ""
+	ownerless := false
+	_, err = updateMgmtIPAMWithRetry(jsManager, subnet, func(r *MgmtIPRecord) {
+		foreign, ownerless = "", false
 		for i, e := range r.Allocated {
 			if e.InstanceID != instanceID {
 				continue
 			}
+			if requireOwner && (e.Node == "" || node == "") {
+				ownerless = true
+				return
+			}
 			// An entry written before the node field existed names nobody, and
 			// refusing those would leak every pre-upgrade address forever.
 			if e.Node != "" && e.Node != node {
@@ -279,14 +321,16 @@ func (a *MgmtIPAllocator) Release(instanceID string) {
 			return
 		}
 	}, false)
-	if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
-		slog.Warn("Failed to release mgmt IP in cluster KV; address remains reserved until reconciled",
-			"instance", instanceID, "err", err)
+	if errors.Is(err, jetstream.ErrKeyNotFound) {
+		err = nil
 	}
-	if foreign != "" {
-		slog.Info("Left a management IP reserved: it belongs to the node running this instance now",
-			"instance", instanceID, "ip", ip, "holder", foreign, "node", node)
+	if err != nil {
+		return ip, foreign, node, fmt.Errorf("release mgmt IP for %s: %w", instanceID, err)
 	}
+	if ownerless {
+		return ip, "", node, fmt.Errorf("release mgmt IP for %s: %w", instanceID, errMgmtIPOwnerless)
+	}
+	return ip, foreign, node, nil
 }
 
 // Claim records this node as the holder of an instance's management IP, for an

spinifex/runtime/daemon/daemon_mgmt_ip_test.go

@@ -2,6 +2,7 @@ package daemon
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strings"
 	"sync"
@@ -960,3 +961,134 @@ func TestMgmtIPAllocator_RebuildTakesOverAnEntryFromAnotherNode(t *testing.T) {
 		t.Errorf("holder after rebuild = %q, want node-2, or nothing can ever release this address", entry.Node)
 	}
 }
+
+// A checked release proves this node held the address. An entry naming no node
+// is unknown ownership, so it stays reserved and the caller keeps the obligation.
+func TestMgmtIPAllocator_ReleaseCheckedRefusesAnEntryThatNamesNoNode(t *testing.T) {
+	jsm := newTestMgmtJSM(t)
+
+	a, err := NewMgmtIPAllocator("10.35.8.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	a.BindKV(jsm, "node-a")
+	cleanupMgmtIPAM(t, jsm, a)
+
+	if _, err := updateMgmtIPAMWithRetry(jsm, a.subnet, func(r *MgmtIPRecord) {
+		r.Subnet = a.subnet
+		r.Allocated = append(r.Allocated, MgmtIPEntry{IP: "10.35.8.10", InstanceID: "i-ownerless"})
+	}, true); err != nil {
+		t.Fatal(err)
+	}
+	a.mu.Lock()
+	a.allocated["i-ownerless"] = "10.35.8.10"
+	a.mu.Unlock()
+
+	err = a.ReleaseChecked("i-ownerless")
+	if !errors.Is(err, errMgmtIPOwnerless) {
+		t.Fatalf("ReleaseChecked on an ownerless entry = %v, want errMgmtIPOwnerless", err)
+	}
+	entry, present := mgmtIPEntryFor(t, jsm, a, "i-ownerless")
+	if !present || entry.IP != "10.35.8.10" || entry.Node != "" {
+		t.Errorf("ownerless entry after refusal = %+v (present %v), want it unchanged", entry, present)
+	}
+	a.mu.Lock()
+	local := a.allocated["i-ownerless"]
+	a.mu.Unlock()
+	if local != "10.35.8.10" {
+		t.Errorf("local record after refusal = %q, want it kept", local)
+	}
+}
+
+func TestMgmtIPAllocator_ReleaseCheckedFreesOwnAndRefusesForeign(t *testing.T) {
+	jsm := newTestMgmtJSM(t)
+
+	node1, err := NewMgmtIPAllocator("10.36.8.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	node1.BindKV(jsm, "node-1")
+	cleanupMgmtIPAM(t, jsm, node1)
+	node2, err := NewMgmtIPAllocator("10.36.8.2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	node2.BindKV(jsm, "node-2")
+
+	if _, err := node1.Allocate("i-own"); err != nil {
+		t.Fatal(err)
+	}
+	if err := node1.ReleaseChecked("i-own"); err != nil {
+		t.Fatalf("ReleaseChecked on this node's entry = %v", err)
+	}
+	if entry, present := mgmtIPEntryFor(t, jsm, node1, "i-own"); present {
+		t.Errorf("own entry still reserved: %+v", entry)
+	}
+
+	ip, err := node2.Allocate("i-theirs")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := node1.ReleaseChecked("i-theirs"); err == nil {
+		t.Fatal("ReleaseChecked freed another node's address without error")
+	}
+	if entry, present := mgmtIPEntryFor(t, jsm, node1, "i-theirs"); !present || entry.Node != "node-2" || entry.IP != ip {
+		t.Errorf("foreign entry after refusal = %+v (present %v), want node-2 holding %s", entry, present, ip)
+	}
+}
+
+// Release, unlike ReleaseChecked, always drops the local entry: a foreign holder
+// keeps its cluster entry and a failed KV write is only logged, matching what its
+// callers relied on before ReleaseChecked existed.
+func TestMgmtIPAllocator_ReleaseDropsLocalOnForeignHolderAndKVFailure(t *testing.T) {
+	jsm := newTestMgmtJSM(t)
+	node1, err := NewMgmtIPAllocator("10.33.8.1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	node1.BindKV(jsm, "node-1")
+	cleanupMgmtIPAM(t, jsm, node1)
+	node2, err := NewMgmtIPAllocator("10.33.8.2")
+	if err != nil {
+		t.Fatal(err)
+	}
+	node2.BindKV(jsm, "node-2")
+
+	ip, err := node1.Allocate("i-moved")
+	if err != nil {
+		t.Fatal(err)
+	}
+	node2.Claim("i-moved", ip)
+	node1.Release("i-moved")
+	if node1.AllocatedCount() != 0 {
+		t.Errorf("local entries after release with a foreign holder = %d, want 0", node1.AllocatedCount())
+	}
+	if entry, present := mgmtIPEntryFor(t, jsm, node1, "i-moved"); !present || entry.Node != "node-2" {
+		t.Errorf("foreign entry after release = %+v (present %v), want node-2 kept", entry, present)
+	}
+
+	nc, err := nats.Connect(sharedJSNATSURL)
+	if err != nil {
+		t.Fatalf("connect NATS: %v", err)
+	}
+	broken, err := NewJetStreamManager(nc)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := broken.InitClusterStateBucket(); err != nil {
+		t.Fatal(err)
+	}
+	node3, err := NewMgmtIPAllocator("10.33.8.3")
+	if err != nil {
+		t.Fatal(err)
+	}
+	node3.BindKV(broken, "node-3")
+	if _, err := node3.Allocate("i-lost-kv"); err != nil {
+		t.Fatal(err)
+	}
+	nc.Close()
+	node3.Release("i-lost-kv")
+	if node3.AllocatedCount() != 0 {
+		t.Errorf("local entries after release with KV down = %d, want 0", node3.AllocatedCount())
+	}
+}

spinifex/runtime/roles/awsgw/awsgw.go

@@ -390,8 +390,9 @@ func launchService(config *config.ClusterConfig) error {
 	if err != nil {
 		return fmt.Errorf("lambda environment key: %w", err)
 	}
-	// Built but not routed: Lambda stays out of the public router until its runtime image exists.
-	_ = lambdaawsapi.NewRegistration(lambdaawsapi.Deps{
+	// Built but routed only in test-link builds: Lambda stays out of the public
+	// router until its runtime image exists.
+	lambdaControlPlane := lambdaawsapi.NewRegistration(lambdaawsapi.Deps{
 		Meta: lambdafunction.NewNATSClient(natsConn),
 		Codes: func(ctx context.Context) (lambdaawsapi.CodeStore, error) {
 			store, err := lambdaCodeStore.Store(ctx)
@@ -403,6 +404,7 @@ func launchService(config *config.ClusterConfig) error {
 		Roles:  iamdomain.NewRoleResolver(iamService),
 		Trusts: lambdaRoleTrusts(iamService),
 		AEAD:   lambdaKey.AEAD,
+		Exec:   lambdaTestExecutor(natsConn, config.Node, lambdaCodeStore),
 	})
 
 	// ECR auth bridge: load (or first-run create) the ES256 signing key from the
@@ -746,7 +748,7 @@ func launchService(config *config.ClusterConfig) error {
 		Admission:    admission,
 		PublicRoutes: publicRoutes,
 		PreAuth:      []dispatch.PreAuth{stsawsapi.NewPreAuth(stsDeps)},
-	}, append([]dispatch.Registration{ecrControlPlane, acmControlPlane, route53ControlPlane, taggingControlPlane, elbv2ControlPlane, rdsControlPlane, iamControlPlane, stsControlPlane, operatorControlPlane, ecsControlPlane, eksControlPlane, bedrockControlPlane, sqsWorkerControlPlane, ec2ControlPlane}, sqsControlPlane...)...)
+	}, append([]dispatch.Registration{ecrControlPlane, acmControlPlane, route53ControlPlane, taggingControlPlane, elbv2ControlPlane, rdsControlPlane, iamControlPlane, stsControlPlane, operatorControlPlane, ecsControlPlane, eksControlPlane, bedrockControlPlane, sqsWorkerControlPlane, ec2ControlPlane}, append(sqsControlPlane, lambdaTestRoutes(lambdaControlPlane)...)...)...)
 	if err != nil {
 		return fmt.Errorf("awsgw: %w", err)
 	}

spinifex/runtime/roles/awsgw/lambda_gateway_test.go

@@ -0,0 +1,75 @@
+package awsgw
+
+import (
+	"bytes"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/go-chi/chi/v5"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/internal/testkit/gwfix"
+	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
+	lambdaawsapi "github.com/mulgadc/spinifex/spinifex/domains/lambda/awsapi"
+	stsawsapi "github.com/mulgadc/spinifex/spinifex/domains/sts/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
+	"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/accountadmission"
+	"github.com/mulgadc/spinifex/spinifex/runtime/roles/awsgw/credverify"
+	"github.com/stretchr/testify/require"
+)
+
+// defaultGateway composes the gateway as launchService does, with signing and an
+// allow-all policy, and the given Lambda registrations appended to it.
+func defaultGateway(t *testing.T, regs ...dispatch.Registration) http.Handler {
+	t.Helper()
+	secret, err := iamdomain.EncryptSecret(gwfix.SecretKey, gwfix.MasterKey)
+	require.NoError(t, err)
+	_, nc := testutil.StartTestNATS(t)
+	iam := &gwfix.PolicyIAM{
+		IAM: gwfix.IAM{
+			MasterKey: gwfix.MasterKey,
+			AccessKeys: map[string]*iamdomain.AccessKey{gwfix.AccessKey: {
+				AccessKeyID: gwfix.AccessKey, SecretAccessKey: secret, UserName: "alice",
+				AccountID: "123456789012", Status: "Active",
+			}},
+		},
+		GetUserPoliciesFn: func(_, _ string) ([]iamdomain.PolicyDocument, error) {
+			return []iamdomain.PolicyDocument{{Version: "2012-10-17", Statement: []iamdomain.Statement{
+				{Effect: "Allow", Action: iamdomain.StringOrArr{"*"}, Resource: iamdomain.StringOrArr{"*"}},
+			}}}, nil
+		},
+	}
+	gw, err := composeGateway(gatewayInputs{
+		Region:       gwfix.Region,
+		NATS:         nc,
+		IAM:          iam,
+		Credentials:  credverify.New(iam, nil),
+		Admission:    accountadmission.New(iam),
+		PublicRoutes: PublicRoutes(nil, nil),
+		PreAuth:      []dispatch.PreAuth{stsawsapi.NewPreAuth(stsawsapi.Deps{})},
+	}, regs...)
+	require.NoError(t, err)
+	gw.DisableLogging = true
+	r := chi.NewRouter()
+	r.Use(gw.SigV4AuthMiddleware())
+	r.HandleFunc("/*", gw.Request)
+	return r
+}
+
+func signedLambdaInvoke(t *testing.T, app http.Handler) *http.Response {
+	t.Helper()
+	return signedLambdaInvokeAt(t, app, "/2015-03-31/functions/f/invocations")
+}
+
+// signedLambdaInvokeAt sends a signed Invoke to target, a path with any query.
+func signedLambdaInvokeAt(t *testing.T, app http.Handler, target string) *http.Response {
+	t.Helper()
+	body := []byte(`{}`)
+	req := httptest.NewRequest(http.MethodPost, target, bytes.NewReader(body))
+	req.Host = "localhost:9999"
+	gwfix.SignRequestAs(t, req, body, gwfix.AccessKey, gwfix.SecretKey, lambdaawsapi.ServiceName, time.Now().UTC())
+	w := httptest.NewRecorder()
+	app.ServeHTTP(w, req)
+	return w.Result()
+}

spinifex/runtime/roles/awsgw/lambda_testlink.go

@@ -0,0 +1,41 @@
+//go:build lambdatestlink
+
+package awsgw
+
+import (
+	"context"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/codestore"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/execution"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// lambdaTestExecutor sends each Invoke to this node's test-link daemon.
+func lambdaTestExecutor(nc *nats.Conn, node string, codes *codestore.Manager) awsapi.Executor {
+	return &testLinkExecutor{
+		NATS: nc,
+		Node: node,
+		Codes: func(ctx context.Context) (codePresigner, error) {
+			store, err := codes.Store(ctx)
+			if err != nil {
+				return nil, err
+			}
+			return store, nil
+		},
+		Grants: func(ctx context.Context) (grantIssuer, error) {
+			js, err := jetstream.New(nc)
+			if err != nil {
+				return nil, err
+			}
+			return execution.Open(ctx, js)
+		},
+	}
+}
+
+// lambdaTestRoutes routes Lambda only in test-link builds.
+func lambdaTestRoutes(lambda dispatch.Registration) []dispatch.Registration {
+	return []dispatch.Registration{lambda}
+}

spinifex/runtime/roles/awsgw/lambda_testlink_executor.go

@@ -0,0 +1,172 @@
+//go:build lambdatestlink
+
+package awsgw
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/codestore"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/execution"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/testlink"
+	"github.com/nats-io/nats.go"
+)
+
+// codePresigner makes the download link for one stored code copy.
+type codePresigner interface {
+	PresignGet(ctx context.Context, ref codestore.Ref, sha256Hex string, ttl time.Duration) (string, error)
+}
+
+// natsRequester sends one NATS request; *nats.Conn satisfies it.
+type natsRequester interface {
+	RequestWithContext(ctx context.Context, subj string, data []byte) (*nats.Msg, error)
+	MaxPayload() int64
+}
+
+var _ natsRequester = (*nats.Conn)(nil)
+
+// grantIssuer records an attempt's grant and withdraws one never sent.
+type grantIssuer interface {
+	Issue(ctx context.Context, a execution.Attempt) (execution.Attempt, error)
+	Withdraw(ctx context.Context, g execution.Grant) error
+}
+
+var _ grantIssuer = (*execution.Store)(nil)
+
+// testLinkExecutor sends each invocation to the test-link daemon on Node.
+type testLinkExecutor struct {
+	NATS   natsRequester
+	Node   string
+	Codes  func(context.Context) (codePresigner, error)
+	Grants func(context.Context) (grantIssuer, error)
+	now    func() time.Time
+}
+
+var _ awsapi.Executor = (*testLinkExecutor)(nil)
+
+// InvokeSync records the attempt's grant before sending and refuses anything
+// it cannot send whole. Once the request is on the wire, every failure is an
+// unknown outcome, never a refusal.
+func (e *testLinkExecutor) InvokeSync(ctx context.Context, in awsapi.SyncInvocation) (awsapi.SyncResult, error) {
+	rec := in.Function
+	// The test link cannot deliver environment variables, so it refuses a
+	// function that has any before presigning, granting or sending.
+	if rec.EnvCipher != "" {
+		return awsapi.SyncResult{}, &awsapi.RefusalError{Err: awserrors.Errorf(awserrors.ErrorServiceException,
+			"the test execution path does not support function environment variables")}
+	}
+	if rec.Code.SHA256 == "" || rec.Code.Size <= 0 || rec.Code.Size > testlink.MaxCodeBytes {
+		return awsapi.SyncResult{}, &awsapi.RefusalError{Err: fmt.Errorf("code copy of %d bytes is outside the test bound", rec.Code.Size)}
+	}
+	store, err := e.Codes(ctx)
+	if err != nil {
+		return awsapi.SyncResult{}, &awsapi.RefusalError{Err: fmt.Errorf("code store: %w", err)}
+	}
+	ref := codestore.Ref{AccountID: rec.AccountID, FunctionID: rec.FunctionID, RevisionID: rec.Code.RevisionID}
+	url, err := store.PresignGet(ctx, ref, rec.Code.SHA256, testlink.CodeURLTTL)
+	if err != nil {
+		return awsapi.SyncResult{}, &awsapi.RefusalError{Err: errors.New("code download link unavailable")}
+	}
+	attemptID, err := testLinkID()
+	if err != nil {
+		return awsapi.SyncResult{}, &awsapi.RefusalError{Err: err}
+	}
+	requestID, err := testLinkID()
+	if err != nil {
+		return awsapi.SyncResult{}, &awsapi.RefusalError{Err: err}
+	}
+	grants, err := e.Grants(ctx)
+	if err != nil {
+		return awsapi.SyncResult{}, &awsapi.RefusalError{Err: fmt.Errorf("execution owner: %w", err)}
+	}
+	now := time.Now
+	if e.now != nil {
+		now = e.now
+	}
+	// Nothing has been sent yet, so even a grant write of unknown outcome is
+	// a refusal; a landed grant expires unstarted at its launch deadline.
+	att, err := grants.Issue(ctx, execution.Attempt{
+		Grant: execution.Grant{AttemptID: attemptID, EnvironmentID: "env-" + attemptID, Generation: 1, Node: e.Node},
+		Identity: execution.Identity{RequestID: requestID, AccountID: rec.AccountID, Region: rec.Region,
+			FunctionID: rec.FunctionID, RevisionID: rec.Code.RevisionID, CodeSHA256: rec.Code.SHA256},
+		LaunchBy: now().Add(testlink.LaunchWindow),
+	})
+	if err != nil {
+		return awsapi.SyncResult{}, &awsapi.RefusalError{Err: fmt.Errorf("attempt grant: %w", err)}
+	}
+	refuse := func(err error) (awsapi.SyncResult, error) {
+		if werr := grants.Withdraw(ctx, att.Grant); werr != nil {
+			err = fmt.Errorf("%w; grant not withdrawn: %w", err, werr)
+		}
+		return awsapi.SyncResult{}, &awsapi.RefusalError{Err: err}
+	}
+	body, err := json.Marshal(testlink.Request{
+		Grant:       att.Grant,
+		AttemptID:   attemptID,
+		RequestID:   requestID,
+		FunctionARN: invokedARN(rec, in.Qualifier),
+		Handler:     rec.Handler,
+		Region:      rec.Region,
+		AccountID:   rec.AccountID,
+		FunctionID:  rec.FunctionID,
+		RevisionID:  rec.Code.RevisionID,
+		TimeoutMs:   rec.Timeout * 1000,
+		MemoryMiB:   rec.MemorySize,
+		CodeURL:     url,
+		CodeSize:    rec.Code.Size,
+		CodeSHA256:  rec.Code.SHA256,
+		Payload:     in.Payload,
+	})
+	if err != nil {
+		return refuse(errors.New("encode test-link request"))
+	}
+	// The encoded request, base64 payload included, must fit one NATS message.
+	if limit := e.NATS.MaxPayload(); int64(len(body)) > limit {
+		return refuse(fmt.Errorf("encoded request of %d bytes exceeds the %d-byte message limit", len(body), limit))
+	}
+	msg, err := e.NATS.RequestWithContext(ctx, testlink.Subject(e.Node), body)
+	if err != nil {
+		return awsapi.SyncResult{}, fmt.Errorf("test-link attempt %s: no reply: %w", attemptID, err)
+	}
+	var rep testlink.Reply
+	if err := json.Unmarshal(msg.Data, &rep); err != nil {
+		return awsapi.SyncResult{}, fmt.Errorf("test-link attempt %s: unreadable reply", attemptID)
+	}
+	switch rep.Outcome {
+	case testlink.OutcomeDone:
+		return awsapi.SyncResult{Payload: rep.Payload, FunctionError: rep.FunctionError, ExecutedVersion: "$LATEST"}, nil
+	case testlink.OutcomeRefused:
+		// The daemon launched nothing. A grant it started it has ended itself,
+		// so withdrawing is refused then and changes nothing.
+		_ = grants.Withdraw(ctx, att.Grant)
+		return awsapi.SyncResult{}, &awsapi.RefusalError{Err: fmt.Errorf("test-link daemon refused: %s", rep.Error)}
+	default:
+		return awsapi.SyncResult{}, fmt.Errorf("test-link attempt %s: outcome %q: %s", attemptID, rep.Outcome, rep.Error)
+	}
+}
+
+func testLinkID() (string, error) {
+	b := make([]byte, 6)
+	if _, err := rand.Read(b); err != nil {
+		return "", fmt.Errorf("attempt ID: %w", err)
+	}
+	return hex.EncodeToString(b), nil
+}
+
+// invokedARN is the function ARN the caller invoked, with its qualifier.
+func invokedARN(rec function.Record, qualifier string) string {
+	a := arn.FormatLambdaFunction(rec.Region, rec.AccountID, rec.FunctionName)
+	if qualifier != "" {
+		a += ":" + qualifier
+	}
+	return a
+}

spinifex/runtime/roles/awsgw/lambda_testlink_executor_test.go

@@ -0,0 +1,183 @@
+//go:build lambdatestlink
+
+package awsgw
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/codestore"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/execution"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/testlink"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/require"
+)
+
+const secretURL = "https://store.example/code?X-Amz-Signature=secret-signature"
+
+type fakePresigner struct{ err error }
+
+func (f fakePresigner) PresignGet(context.Context, codestore.Ref, string, time.Duration) (string, error) {
+	return secretURL, f.err
+}
+
+type fakeNATS struct {
+	max   int64
+	sent  [][]byte
+	reply *testlink.Reply
+	err   error
+}
+
+func (f *fakeNATS) MaxPayload() int64 { return f.max }
+
+func (f *fakeNATS) RequestWithContext(_ context.Context, _ string, data []byte) (*nats.Msg, error) {
+	f.sent = append(f.sent, data)
+	if f.err != nil {
+		return nil, f.err
+	}
+	body, _ := json.Marshal(f.reply)
+	return &nats.Msg{Data: body}, nil
+}
+
+func invocation(payload []byte) awsapi.SyncInvocation {
+	return awsapi.SyncInvocation{Payload: payload, Function: function.Record{
+		FunctionID: "fn1", AccountID: "123456789012", Region: "ap-southeast-2", FunctionName: "f",
+		Handler: "bootstrap.h", Timeout: 3, MemorySize: 128,
+		Code: function.Code{RevisionID: "r1", SHA256: strings.Repeat("ab", 32), Size: 100},
+	}}
+}
+
+func grantStore(t *testing.T) *execution.Store {
+	t.Helper()
+	_, nc, _ := testutil.StartTestJetStream(t)
+	store, err := execution.Open(context.Background(), testutil.NewJetStream(t, nc))
+	require.NoError(t, err)
+	return store
+}
+
+func executor(n *fakeNATS, presignErr error, store *execution.Store) *testLinkExecutor {
+	return &testLinkExecutor{NATS: n, Node: "node1",
+		Codes:  func(context.Context) (codePresigner, error) { return fakePresigner{err: presignErr}, nil },
+		Grants: func(context.Context) (grantIssuer, error) { return store, nil },
+	}
+}
+
+// grantState is the node's current attempt state, or "" with no grant.
+func grantState(t *testing.T, store *execution.Store) string {
+	t.Helper()
+	a, err := store.Current(context.Background(), "node1")
+	if errors.Is(err, execution.ErrNotCurrent) {
+		return ""
+	}
+	require.NoError(t, err)
+	return a.State
+}
+
+func TestLambdaTestLinkExecutor_RefusesWithoutSending(t *testing.T) {
+	cases := map[string]struct {
+		in         awsapi.SyncInvocation
+		presignErr error
+		max        int64
+		state      string
+	}{
+		"no code digest":    {in: func() awsapi.SyncInvocation { i := invocation(nil); i.Function.Code.SHA256 = ""; return i }(), max: 1 << 20},
+		"presign fails":     {in: invocation(nil), presignErr: errors.New("boom"), max: 1 << 20},
+		"encoded too large": {in: invocation(make([]byte, 3000)), max: 4000, state: execution.StateWithdrawn},
+		"code beyond bound": {in: func() awsapi.SyncInvocation {
+			i := invocation(nil)
+			i.Function.Code.Size = testlink.MaxCodeBytes + 1
+			return i
+		}(), max: 1 << 20},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			n, store := &fakeNATS{max: tc.max}, grantStore(t)
+			_, err := executor(n, tc.presignErr, store).InvokeSync(context.Background(), tc.in)
+			var refusal *awsapi.RefusalError
+			require.ErrorAs(t, err, &refusal)
+			require.Empty(t, n.sent, "a refusal must send nothing")
+			require.NotContains(t, err.Error(), "secret-signature")
+			require.Equal(t, tc.state, grantState(t, store), "no grant, or one withdrawn unsent")
+		})
+	}
+}
+
+func TestLambdaTestLinkExecutor_BusyNodeSendsNothing(t *testing.T) {
+	store := grantStore(t)
+	held := &fakeNATS{max: 1 << 20, err: nats.ErrTimeout}
+	_, err := executor(held, nil, store).InvokeSync(context.Background(), invocation(nil))
+	require.Error(t, err)
+
+	n := &fakeNATS{max: 1 << 20, reply: &testlink.Reply{Outcome: testlink.OutcomeDone}}
+	_, err = executor(n, nil, store).InvokeSync(context.Background(), invocation(nil))
+	var refusal *awsapi.RefusalError
+	require.ErrorAs(t, err, &refusal)
+	require.ErrorIs(t, err, execution.ErrNodeBusy)
+	require.Empty(t, n.sent, "no second grant while an attempt of unknown outcome holds the node")
+}
+
+// 3000 payload bytes are 4000 as base64: the bound counts the encoding.
+func TestLambdaTestLinkExecutor_BoundCountsEncodedPayload(t *testing.T) {
+	n := &fakeNATS{max: 3500, reply: &testlink.Reply{Outcome: testlink.OutcomeDone}}
+	_, err := executor(n, nil, grantStore(t)).InvokeSync(context.Background(), invocation(make([]byte, 3000)))
+	var refusal *awsapi.RefusalError
+	require.ErrorAs(t, err, &refusal)
+	require.Empty(t, n.sent)
+}
+
+func TestLambdaTestLinkExecutor_PostDispatchFailureIsNeverARefusal(t *testing.T) {
+	cases := map[string]*fakeNATS{
+		"no reply":      {max: 1 << 20, err: nats.ErrTimeout},
+		"no responders": {max: 1 << 20, err: nats.ErrNoResponders},
+		"unknown":       {max: 1 << 20, reply: &testlink.Reply{Outcome: testlink.OutcomeUnknown, Error: "VM stop not proven"}},
+		"unrecognised":  {max: 1 << 20, reply: &testlink.Reply{Outcome: "later"}},
+	}
+	for name, n := range cases {
+		t.Run(name, func(t *testing.T) {
+			store := grantStore(t)
+			_, err := executor(n, nil, store).InvokeSync(context.Background(), invocation([]byte(`{}`)))
+			require.Error(t, err)
+			var refusal *awsapi.RefusalError
+			require.False(t, errors.As(err, &refusal), "a sent request has an unknown outcome")
+			require.Len(t, n.sent, 1)
+			require.NotContains(t, err.Error(), "secret-signature")
+			require.Equal(t, execution.StateGranted, grantState(t, store), "a sent grant is never withdrawn")
+		})
+	}
+}
+
+func TestLambdaTestLinkExecutor_DaemonRefusalAndResult(t *testing.T) {
+	store := grantStore(t)
+	n := &fakeNATS{max: 1 << 20, reply: &testlink.Reply{Outcome: testlink.OutcomeRefused, Error: "busy"}}
+	_, err := executor(n, nil, store).InvokeSync(context.Background(), invocation(nil))
+	var refusal *awsapi.RefusalError
+	require.ErrorAs(t, err, &refusal)
+	require.Equal(t, execution.StateWithdrawn, grantState(t, store), "an unstarted grant the daemon refused is withdrawn")
+
+	n = &fakeNATS{max: 1 << 20, reply: &testlink.Reply{Outcome: testlink.OutcomeDone, Payload: []byte(`{"sum":42}`), FunctionError: "Unhandled"}}
+	out, err := executor(n, nil, store).InvokeSync(context.Background(), invocation([]byte(`{"a":1}`)))
+	require.NoError(t, err)
+	require.Equal(t, awsapi.SyncResult{Payload: []byte(`{"sum":42}`), FunctionError: "Unhandled", ExecutedVersion: "$LATEST"}, out)
+
+	var sent testlink.Request
+	require.NoError(t, json.Unmarshal(n.sent[0], &sent))
+	require.Equal(t, secretURL, sent.CodeURL)
+	require.Equal(t, int64(3000), sent.TimeoutMs)
+	require.Equal(t, "bootstrap.h", sent.Handler)
+	require.Regexp(t, `^[0-9a-f]{12}$`, sent.AttemptID)
+	require.NoError(t, testlink.CheckRequest(sent))
+
+	// The request carries exactly the grant and identity the owner recorded.
+	cur, err := store.Current(context.Background(), "node1")
+	require.NoError(t, err)
+	require.Equal(t, cur.Grant, sent.Grant)
+	require.Equal(t, cur.Identity, sent.Identity())
+	require.Equal(t, "r1", cur.Identity.RevisionID)
+}

spinifex/runtime/roles/awsgw/lambda_testlink_off.go

@@ -0,0 +1,16 @@
+//go:build !lambdatestlink
+
+package awsgw
+
+import (
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/codestore"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
+	"github.com/nats-io/nats.go"
+)
+
+// Without the test link there is no executor, so Invoke refuses before it
+// presigns, sends or allocates anything, and Lambda stays unrouted.
+func lambdaTestExecutor(*nats.Conn, string, *codestore.Manager) awsapi.Executor { return nil }
+
+func lambdaTestRoutes(dispatch.Registration) []dispatch.Registration { return nil }

spinifex/runtime/roles/awsgw/lambda_testlink_off_test.go

@@ -0,0 +1,70 @@
+//go:build !lambdatestlink
+
+package awsgw
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"sync/atomic"
+	"testing"
+
+	lambdaawsapi "github.com/mulgadc/spinifex/spinifex/domains/lambda/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	"github.com/stretchr/testify/require"
+)
+
+// spyFunctions counts every lookup and serves one Active function with stored code.
+type spyFunctions struct{ gets atomic.Int32 }
+
+var _ function.Client = (*spyFunctions)(nil)
+
+func (s *spyFunctions) Get(_ context.Context, account, region, name string) (function.Record, uint64, error) {
+	s.gets.Add(1)
+	return function.Record{
+		FunctionID: "f1", AccountID: account, Region: region, FunctionName: name, Runtime: "provided.al2023",
+		Timeout: 3, MemorySize: 128, State: function.StateActive,
+		Code: function.Code{RevisionID: "r1", State: function.CodeStored},
+	}, 1, nil
+}
+
+func (*spyFunctions) Create(context.Context, string, string, function.Record) (uint64, error) {
+	return 0, errors.New("unused")
+}
+
+func (*spyFunctions) CodeStored(context.Context, string, string, string, uint64, string, int64) (function.Record, uint64, error) {
+	return function.Record{}, 0, errors.New("unused")
+}
+
+func (*spyFunctions) RollbackCreate(context.Context, string, string, string, uint64) error {
+	return errors.New("unused")
+}
+
+// spyExec counts invocations that would start an attempt.
+type spyExec struct{ calls atomic.Int32 }
+
+func (e *spyExec) InvokeSync(context.Context, lambdaawsapi.SyncInvocation) (lambdaawsapi.SyncResult, error) {
+	e.calls.Add(1)
+	return lambdaawsapi.SyncResult{Payload: []byte(`{}`), ExecutedVersion: "$LATEST"}, nil
+}
+
+// A default build composes the gateway with Lambda unrouted and no executor: a
+// signed Invoke is refused before any Lambda handler runs and starts no attempt.
+func TestDefaultBuild_InvokeNeverReachesLambda(t *testing.T) {
+	require.Nil(t, lambdaTestExecutor(nil, "node1", nil), "a default build has no Lambda executor")
+
+	meta, exec := &spyFunctions{}, &spyExec{}
+	lambda := lambdaawsapi.NewRegistration(lambdaawsapi.Deps{Meta: meta, Exec: exec})
+
+	resp := signedLambdaInvoke(t, defaultGateway(t, lambdaTestRoutes(lambda)...))
+	require.NotEqual(t, http.StatusOK, resp.StatusCode, "a default build answered Invoke")
+	require.Zero(t, meta.gets.Load(), "a default build ran a Lambda handler")
+	require.Zero(t, exec.calls.Load(), "a default build started an attempt")
+
+	// The same signed request reaches both once Lambda is routed, so the
+	// refusal above comes from the missing route.
+	resp = signedLambdaInvoke(t, defaultGateway(t, lambda))
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+	require.Equal(t, int32(1), meta.gets.Load())
+	require.Equal(t, int32(1), exec.calls.Load())
+}

spinifex/runtime/roles/awsgw/lambda_testlink_reply_test.go

@@ -0,0 +1,95 @@
+//go:build lambdatestlink
+
+package awsgw
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/internal/testkit/gwtest"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/pipeline"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/reqctx"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/testlink"
+	"github.com/stretchr/testify/require"
+)
+
+// storedFunction serves one Active function "f" whose code is stored.
+type storedFunction struct{}
+
+var _ function.Client = storedFunction{}
+
+func (storedFunction) Get(_ context.Context, account, region, name string) (function.Record, uint64, error) {
+	if name != "f" {
+		return function.Record{}, 0, function.ErrNotFound
+	}
+	rec := invocation(nil).Function
+	rec.AccountID, rec.Region, rec.State = account, region, function.StateActive
+	rec.Code.State = function.CodeStored
+	return rec, 1, nil
+}
+
+func (storedFunction) Create(context.Context, string, string, function.Record) (uint64, error) {
+	return 0, errors.New("unused")
+}
+
+func (storedFunction) CodeStored(context.Context, string, string, string, uint64, string, int64) (function.Record, uint64, error) {
+	return function.Record{}, 0, errors.New("unused")
+}
+
+func (storedFunction) RollbackCreate(context.Context, string, string, string, uint64) error {
+	return errors.New("unused")
+}
+
+// invokeThrough sends one Invoke through the Lambda handler to the real
+// test-link executor; fakeNATS stands in for the daemon and answers with rep.
+func invokeThrough(t *testing.T, rep testlink.Reply) (*httptest.ResponseRecorder, error) {
+	t.Helper()
+	n := &fakeNATS{max: 1 << 20, reply: &rep}
+	reg := awsapi.NewRegistration(awsapi.Deps{Meta: storedFunction{}, Exec: executor(n, nil, grantStore(t))})
+	rec := httptest.NewRecorder()
+	req := httptest.NewRequest(http.MethodPost, "/2015-03-31/functions/f/invocations", strings.NewReader(`{}`))
+	err := gwtest.AllowAll().Dispatch(reg, awsapi.ServiceName, rec, req)
+	require.Len(t, n.sent, 1, "the invocation reached the daemon")
+	return rec, err
+}
+
+// The session's reply for an oversized function response reaches the caller
+// as 200 with an Unhandled function error.
+func TestLambdaTestLinkReply_OversizedResponseIsUnhandled(t *testing.T) {
+	payload := `{"errorType":"Function.ResponseSizeTooLarge","errorMessage":"too large"}`
+	rec, err := invokeThrough(t, testlink.Reply{Outcome: testlink.OutcomeDone, FunctionError: "Unhandled", Payload: []byte(payload)})
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, rec.Code)
+	require.Equal(t, "Unhandled", rec.Header().Get("X-Amz-Function-Error"))
+	require.JSONEq(t, payload, rec.Body.String())
+}
+
+// An unproven timeout claim is an unknown outcome, never a 200: the gateway's
+// REST-JSON error handler writes it as a 500 ServiceException.
+func TestLambdaTestLinkReply_UnprovenTimeoutIsUnknown(t *testing.T) {
+	_, err := invokeThrough(t, testlink.Reply{Outcome: testlink.OutcomeUnknown, Error: testlink.ErrTimeoutUnproven.Error()})
+	require.Error(t, err)
+
+	b := dispatch.NewBuilder()
+	require.NoError(t, b.Register(awsapi.NewRegistration(awsapi.Deps{})))
+	gw := &pipeline.Config{Services: b.Build()}
+	req := httptest.NewRequest(http.MethodPost, "/2015-03-31/functions/f/invocations", nil)
+	req = req.WithContext(reqctx.WithService(req.Context(), awsapi.ServiceName))
+	rec := httptest.NewRecorder()
+	gw.ErrorHandler(rec, req, err)
+
+	require.Equal(t, http.StatusInternalServerError, rec.Code)
+	require.Equal(t, "ServiceException", rec.Header().Get("X-Amzn-Errortype"))
+	var body map[string]string
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &body))
+	require.NotEmpty(t, body["message"])
+	require.NotContains(t, rec.Body.String(), "Sandbox.Timedout")
+}

spinifex/runtime/roles/awsgw/lambda_testlink_seam_test.go

@@ -0,0 +1,376 @@
+//go:build lambdatestlink && lambdatestlinkseam
+
+package awsgw
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"io"
+	"log/slog"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/agent"
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
+	"github.com/mulgadc/spinifex/spinifex/agents/lambda/runtimeapi"
+	lambdaawsapi "github.com/mulgadc/spinifex/spinifex/domains/lambda/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/codestore"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/execution"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/testlink"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/require"
+)
+
+// seamCredential stands in for a presigned credential; it is not a real key.
+const seamCredential = "AKIASYNTHETICSEAMKEY"
+
+var seamCode = []byte("synthetic function code")
+
+// seamHandler is the configured handler of the seam's function.
+const seamHandler = "bootstrap.seam"
+
+// seamFunction serves one Active function "f" whose stored code is seamCode.
+type seamFunction struct {
+	timeout   int64
+	envCipher string
+}
+
+var _ function.Client = seamFunction{}
+
+func (s seamFunction) Get(_ context.Context, account, region, name string) (function.Record, uint64, error) {
+	if name != "f" {
+		return function.Record{}, 0, function.ErrNotFound
+	}
+	sum := sha256.Sum256(seamCode)
+	return function.Record{FunctionID: "fn1", AccountID: account, Region: region, FunctionName: name,
+		Runtime: "provided.al2023", Handler: seamHandler, EnvCipher: s.envCipher, Timeout: s.timeout, MemorySize: 128, State: function.StateActive,
+		Code: function.Code{RevisionID: "r1", State: function.CodeStored, SHA256: hex.EncodeToString(sum[:]), Size: int64(len(seamCode))},
+	}, 1, nil
+}
+
+func (seamFunction) Create(context.Context, string, string, function.Record) (uint64, error) {
+	return 0, errors.New("unused")
+}
+
+func (seamFunction) CodeStored(context.Context, string, string, string, uint64, string, int64) (function.Record, uint64, error) {
+	return function.Record{}, 0, errors.New("unused")
+}
+
+func (seamFunction) RollbackCreate(context.Context, string, string, string, uint64) error {
+	return errors.New("unused")
+}
+
+type urlPresigner struct {
+	url   string
+	calls *atomic.Int32
+}
+
+func (p urlPresigner) PresignGet(context.Context, codestore.Ref, string, time.Duration) (string, error) {
+	if p.calls != nil {
+		p.calls.Add(1)
+	}
+	return p.url, nil
+}
+
+// countedGrants counts the grants the executor issues.
+type countedGrants struct {
+	grantIssuer
+	issued atomic.Int32
+}
+
+func (c *countedGrants) Issue(ctx context.Context, a execution.Attempt) (execution.Attempt, error) {
+	c.issued.Add(1)
+	return c.grantIssuer.Issue(ctx, a)
+}
+
+// seamNATS is the transport only: it hands the executor's encoded request to
+// the daemon's real attempt steps and returns the daemon's encoded reply.
+type seamNATS struct {
+	t        *testing.T
+	d        *testlink.Daemon
+	guest    func(context.Context, net.Conn)
+	mu       sync.Mutex
+	requests [][]byte
+	replies  [][]byte
+}
+
+func (*seamNATS) MaxPayload() int64 { return 1 << 20 }
+
+func (n *seamNATS) RequestWithContext(ctx context.Context, subj string, data []byte) (*nats.Msg, error) {
+	require.Equal(n.t, testlink.Subject("node1"), subj)
+	body, err := json.Marshal(n.d.HandleWithGuest(ctx, data, n.guest))
+	n.mu.Lock()
+	n.requests = append(n.requests, data)
+	n.replies = append(n.replies, body)
+	n.mu.Unlock()
+	return &nats.Msg{Data: body}, err
+}
+
+// skewAhead moves every deadline the agent sees earlier, as a guest clock
+// running that far ahead of the host's would.
+type skewAhead struct {
+	hostlink.Link
+	ahead time.Duration
+}
+
+func (l skewAhead) Next(ctx context.Context) (hostlink.Request, error) {
+	r, err := l.Link.Next(ctx)
+	r.DeadlineMs -= l.ahead.Milliseconds()
+	return r, err
+}
+
+// agentGuest runs the real guest agent and Runtime API server; bootstrap is
+// an HTTP client standing in for the function runtime.
+func agentGuest(ahead time.Duration, bootstrap func(base string)) func(context.Context, net.Conn) {
+	return func(ctx context.Context, conn net.Conn) {
+		srv := runtimeapi.New()
+		ts := httptest.NewServer(srv.Handler())
+		defer ts.Close()
+		if bootstrap != nil {
+			go bootstrap(ts.URL + "/2018-06-01/runtime/")
+		}
+		var link hostlink.Link = hostlink.NewTestStream(conn)
+		if ahead > 0 {
+			link = skewAhead{link, ahead}
+		}
+		_ = agent.Run(ctx, link, srv)
+	}
+}
+
+func nextInvocationID(base string) (string, bool) {
+	resp, err := http.Get(base + "invocation/next")
+	if err != nil {
+		return "", false
+	}
+	_, _ = io.Copy(io.Discard, resp.Body)
+	_ = resp.Body.Close()
+	return resp.Header.Get("Lambda-Runtime-Aws-Request-Id"), true
+}
+
+// lockedBuffer collects log output written from several goroutines.
+type lockedBuffer struct {
+	mu sync.Mutex
+	b  bytes.Buffer
+}
+
+func (l *lockedBuffer) Write(p []byte) (int, error) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return l.b.Write(p)
+}
+
+func (l *lockedBuffer) String() string {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return l.b.String()
+}
+
+type seamRun struct {
+	resp     *http.Response
+	body     []byte
+	requests [][]byte
+	replies  [][]byte
+	log      string
+	presigns int32
+	grants   int32
+}
+
+// invokeComposed sends one signed Invoke through the composed gateway, the
+// Lambda handler and the real test-link executor into the daemon's attempt.
+func invokeComposed(t *testing.T, timeout int64, codeURL string, client *http.Client, guest func(context.Context, net.Conn)) seamRun {
+	t.Helper()
+	run := invokeComposedAt(t, "/2015-03-31/functions/f/invocations", seamFunction{timeout: timeout}, codeURL, client, guest)
+	require.Len(t, run.replies, 1, "the request reached the daemon once")
+	return run
+}
+
+// invokeComposedAt is invokeComposed for one target and function configuration.
+func invokeComposedAt(t *testing.T, target string, fn seamFunction, codeURL string, client *http.Client, guest func(context.Context, net.Conn)) seamRun {
+	t.Helper()
+	logs := &lockedBuffer{}
+	prev := slog.Default()
+	slog.SetDefault(slog.New(slog.NewTextHandler(logs, &slog.HandlerOptions{Level: slog.LevelDebug})))
+	t.Cleanup(func() { slog.SetDefault(prev) })
+
+	store := grantStore(t)
+	grants := &countedGrants{grantIssuer: store}
+	var presigns atomic.Int32
+	n := &seamNATS{t: t, d: &testlink.Daemon{Node: "node1", Grants: store, HTTP: client}, guest: guest}
+	exec := &testLinkExecutor{NATS: n, Node: "node1",
+		Codes:  func(context.Context) (codePresigner, error) { return urlPresigner{codeURL, &presigns}, nil },
+		Grants: func(context.Context) (grantIssuer, error) { return grants, nil },
+	}
+	reg := lambdaawsapi.NewRegistration(lambdaawsapi.Deps{Meta: fn, Exec: exec})
+	resp := signedLambdaInvokeAt(t, defaultGateway(t, lambdaTestRoutes(reg)...), target)
+	body, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	return seamRun{resp: resp, body: body, requests: n.requests, replies: n.replies, log: logs.String(),
+		presigns: presigns.Load(), grants: grants.issued.Load()}
+}
+
+func codeServer(t *testing.T) *httptest.Server {
+	t.Helper()
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write(seamCode)
+	}))
+	t.Cleanup(srv.Close)
+	return srv
+}
+
+func daemonReply(t *testing.T, raw []byte) testlink.Reply {
+	t.Helper()
+	var rep testlink.Reply
+	require.NoError(t, json.Unmarshal(raw, &rep))
+	return rep
+}
+
+// An oversized function response reaches the caller as 200 with an Unhandled
+// function error and the runtime's error payload.
+func TestLambdaTestLinkSeam_OversizedResponseIsUnhandled(t *testing.T) {
+	srv := codeServer(t)
+	run := invokeComposed(t, 10, srv.URL+"/code", srv.Client(), agentGuest(0, func(base string) {
+		if id, ok := nextInvocationID(base); ok {
+			if resp, err := http.Post(base+"invocation/"+id+"/response", "application/json",
+				bytes.NewReader(make([]byte, runtimeapi.MaxResponseBytes+1))); err == nil {
+				_ = resp.Body.Close()
+			}
+		}
+	}))
+	require.Equal(t, testlink.OutcomeDone, daemonReply(t, run.replies[0]).Outcome)
+	require.Equal(t, http.StatusOK, run.resp.StatusCode)
+	require.Equal(t, "Unhandled", run.resp.Header.Get("X-Amz-Function-Error"))
+	var payload struct {
+		ErrorType string `json:"errorType"`
+	}
+	require.NoError(t, json.Unmarshal(run.body, &payload))
+	require.Equal(t, "Function.ResponseSizeTooLarge", payload.ErrorType)
+}
+
+// Neither an early guest timeout claim nor the host's deadline is a function
+// result: both reach the caller as a 500 ServiceException.
+func TestLambdaTestLinkSeam_TimeoutsAreUnknown(t *testing.T) {
+	cases := map[string]struct {
+		timeout int64
+		guest   func(context.Context, net.Conn)
+		reason  string
+	}{
+		"agent timeout before the host deadline": {2, agentGuest(2*time.Second-200*time.Millisecond, func(base string) { nextInvocationID(base) }), testlink.ErrTimeoutUnproven.Error()},
+		"host deadline":                          {1, func(ctx context.Context, _ net.Conn) { <-ctx.Done() }, testlink.ErrFunctionTimeout.Error()},
+		"bootstrap posts the timeout error type": {10, agentGuest(0, postSpoofedTimeout), testlink.ErrTimeoutUnproven.Error()},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			srv := codeServer(t)
+			run := invokeComposed(t, tc.timeout, srv.URL+"/code", srv.Client(), tc.guest)
+			rep := daemonReply(t, run.replies[0])
+			require.Equal(t, testlink.OutcomeUnknown, rep.Outcome)
+			require.Equal(t, tc.reason, rep.Error, "the outcome is unknown for the wrong reason")
+			require.Equal(t, http.StatusInternalServerError, run.resp.StatusCode)
+			require.Equal(t, "ServiceException", run.resp.Header.Get("X-Amzn-Errortype"))
+			require.Empty(t, run.resp.Header.Get("X-Amz-Function-Error"))
+			require.NotContains(t, string(run.body), "Sandbox.Timedout")
+		})
+	}
+}
+
+// postSpoofedTimeout is a bootstrap reporting Sandbox.Timedout as its own error.
+func postSpoofedTimeout(base string) {
+	id, ok := nextInvocationID(base)
+	if !ok {
+		return
+	}
+	req, err := http.NewRequest(http.MethodPost, base+"invocation/"+id+"/error",
+		strings.NewReader(`{"errorType":"Sandbox.Timedout","errorMessage":"spoof"}`))
+	if err != nil {
+		return
+	}
+	req.Header.Set("Lambda-Runtime-Function-Error-Type", "Sandbox.Timedout")
+	if resp, err := http.DefaultClient.Do(req); err == nil {
+		_ = resp.Body.Close()
+	}
+}
+
+// A malformed redirect quoting a presigned credential reaches neither the
+// daemon's reply, the caller, nor the gateway's log.
+func TestLambdaTestLinkSeam_RedirectCredentialNotLeaked(t *testing.T) {
+	srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Location", "%zz?X-Amz-Credential="+seamCredential+"&X-Amz-Signature=deadbeef")
+		w.WriteHeader(http.StatusFound)
+	}))
+	t.Cleanup(srv.Close)
+	run := invokeComposed(t, 3, srv.URL+"/code", srv.Client(), func(ctx context.Context, _ net.Conn) { <-ctx.Done() })
+
+	rep := daemonReply(t, run.replies[0])
+	require.Equal(t, testlink.OutcomeRefused, rep.Outcome)
+	require.Equal(t, "code download: request failed", rep.Error)
+	require.Contains(t, run.log, "code download: request failed", "the refusal was not logged")
+	require.NotContains(t, string(run.replies[0]), seamCredential)
+	require.NotContains(t, string(run.body), seamCredential)
+	require.NotContains(t, run.log, seamCredential)
+	require.Equal(t, http.StatusInternalServerError, run.resp.StatusCode)
+}
+
+// The bootstrap sees the ARN the caller invoked, qualifier included, and the
+// daemon is asked to run the function's configured handler.
+func TestLambdaTestLinkSeam_RuntimeSeesInvokedARNAndHandler(t *testing.T) {
+	cases := map[string]struct{ target, suffix string }{
+		"unqualified":     {"/2015-03-31/functions/f/invocations", ""},
+		"path qualifier":  {"/2015-03-31/functions/f:$LATEST/invocations", ":$LATEST"},
+		"query qualifier": {"/2015-03-31/functions/f/invocations?Qualifier=%24LATEST", ":$LATEST"},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			srv := codeServer(t)
+			seen := make(chan string, 1)
+			run := invokeComposedAt(t, tc.target, seamFunction{timeout: 10}, srv.URL+"/code", srv.Client(), agentGuest(0, func(base string) {
+				resp, err := http.Get(base + "invocation/next")
+				if err != nil {
+					seen <- ""
+					return
+				}
+				_ = resp.Body.Close()
+				seen <- resp.Header.Get("Lambda-Runtime-Invoked-Function-Arn")
+				id := resp.Header.Get("Lambda-Runtime-Aws-Request-Id")
+				if post, err := http.Post(base+"invocation/"+id+"/response", "application/json", strings.NewReader(`{}`)); err == nil {
+					_ = post.Body.Close()
+				}
+			}))
+			require.Len(t, run.replies, 1)
+			require.Equal(t, testlink.OutcomeDone, daemonReply(t, run.replies[0]).Outcome)
+			require.Equal(t, http.StatusOK, run.resp.StatusCode)
+			var req testlink.Request
+			require.NoError(t, json.Unmarshal(run.requests[0], &req))
+			require.Equal(t, seamHandler, req.Handler)
+			want := arn.FormatLambdaFunction(req.Region, req.AccountID, "f") + tc.suffix
+			require.Equal(t, want, <-seen)
+		})
+	}
+}
+
+// A function with environment variables is refused before any code link is
+// presigned, any grant is issued or anything is sent to a node.
+func TestLambdaTestLinkSeam_EnvironmentVariablesRefusedBeforeDispatch(t *testing.T) {
+	srv := codeServer(t)
+	run := invokeComposedAt(t, "/2015-03-31/functions/f/invocations", seamFunction{timeout: 10, envCipher: "sealed"},
+		srv.URL+"/code", srv.Client(), func(ctx context.Context, _ net.Conn) { <-ctx.Done() })
+	require.Equal(t, http.StatusInternalServerError, run.resp.StatusCode)
+	require.Equal(t, "ServiceException", run.resp.Header.Get("X-Amzn-Errortype"))
+	require.Contains(t, string(run.body), "does not support function environment variables")
+	require.Zero(t, run.presigns, "a code link was presigned")
+	require.Zero(t, run.grants, "a grant was issued")
+	require.Empty(t, run.requests, "the request reached a node")
+}

tests/integration/lambda_test.go

@@ -9,9 +9,14 @@ import (
 	"crypto/rand"
 	"crypto/sha256"
 	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
 	"errors"
 	"io"
 	"net/http"
+	"strings"
+	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -42,12 +47,44 @@ const lambdaTrustPolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow
 
 // The gateway's NATS connection and IAM service exist only once
 // StartGatewayWithServices has built them, after the registration list.
-type lateMeta struct{ lambdafunction.Client }
+type lateMeta struct {
+	lambdafunction.Client
+	// active reports every record Active, standing in for runtime availability.
+	active atomic.Bool
+}
+
+func (m *lateMeta) Get(ctx context.Context, accountID, region, name string) (lambdafunction.Record, uint64, error) {
+	rec, rev, err := m.Client.Get(ctx, accountID, region, name)
+	if err == nil && m.active.Load() {
+		rec.State = lambdafunction.StateActive
+	}
+	return rec, rev, err
+}
+
+// echoExec stands in for the execution owner: it returns the payload it got.
+type echoExec struct {
+	mu   sync.Mutex
+	last lambdaawsapi.SyncInvocation
+}
+
+func (e *echoExec) InvokeSync(_ context.Context, inv lambdaawsapi.SyncInvocation) (lambdaawsapi.SyncResult, error) {
+	e.mu.Lock()
+	e.last = inv
+	e.mu.Unlock()
+	return lambdaawsapi.SyncResult{Payload: inv.Payload, FunctionError: "Handled", ExecutedVersion: "$LATEST"}, nil
+}
 
 type lateRoles struct{ passrole.Resolver }
 
 // startLambdaDaemonLite subscribes the production MetaService to the subjects
 // the gateway's Lambda client dials, on JetStream KV like the live daemon.
+func mustHex(t *testing.T, s string) []byte {
+	t.Helper()
+	b, err := hex.DecodeString(s)
+	require.NoError(t, err)
+	return b
+}
+
 func startLambdaDaemonLite(t *testing.T, nc *nats.Conn) {
 	t.Helper()
 	js, err := jetstream.New(nc)
@@ -90,7 +127,7 @@ func TestLambda_SignedCreateAndGetDownloadsTheStoredCode(t *testing.T) {
 	envKey, err := masterkey.New(keyBytes)
 	require.NoError(t, err)
 
-	meta, roles := &lateMeta{}, &lateRoles{}
+	meta, roles, exec := &lateMeta{}, &lateRoles{}, &echoExec{}
 	var trusts func(accountID, roleARN string) (bool, error)
 	gw := StartGatewayWithServices(t, []awsdispatch.Registration{lambdaawsapi.NewRegistration(lambdaawsapi.Deps{
 		Meta:   meta,
@@ -98,6 +135,7 @@ func TestLambda_SignedCreateAndGetDownloadsTheStoredCode(t *testing.T) {
 		Roles:  roles,
 		Trusts: func(accountID, roleARN string) (bool, error) { return trusts(accountID, roleARN) },
 		AEAD:   envKey.AEAD,
+		Exec:   exec,
 	})})
 	iamSvc, ok := gwfix.DepsOf(gw.Config).IAM.(*iamdomain.IAMServiceImpl)
 	require.True(t, ok, "harness IAM service type")
@@ -124,6 +162,19 @@ func TestLambda_SignedCreateAndGetDownloadsTheStoredCode(t *testing.T) {
 	})
 	require.NoError(t, err)
 	require.Equal(t, base64.StdEncoding.EncodeToString(sum[:]), aws.StringValue(created.CodeSha256))
+	account := strings.Split(aws.StringValue(created.FunctionArn), ":")[4]
+	stored, storedRev, err := meta.Client.Get(t.Context(), account, testRegion, "e2e-fn")
+	require.NoError(t, err)
+	js, err := jetstream.New(gw.NATSConn)
+	require.NoError(t, err)
+	kv, err := js.KeyValue(t.Context(), lambdafunction.KVAccountBucket(account))
+	require.NoError(t, err)
+	upEntry, err := kv.Get(t.Context(), lambdafunction.KVUploadKey(testRegion, stored.FunctionID, stored.Code.RevisionID))
+	require.NoError(t, err, "the stored copy is tracked by an upload record")
+	var up lambdafunction.Upload
+	require.NoError(t, json.Unmarshal(upEntry.Value(), &up))
+	require.Equal(t, lambdafunction.UploadAttached, up.State)
+	require.Equal(t, storedRev, up.AttachedRevision)
 
 	got, err := cli.GetFunction(&lambda.GetFunctionInput{FunctionName: aws.String("e2e-fn")})
 	require.NoError(t, err)
@@ -150,6 +201,26 @@ func TestLambda_SignedCreateAndGetDownloadsTheStoredCode(t *testing.T) {
 	require.ErrorAs(t, err, &aerr)
 	require.Equal(t, lambda.ErrCodeResourceNotFoundException, aerr.Code())
 
+	// Every stored function stays Pending, so Invoke is refused before dispatch
+	// until the record is reported Active.
+	invokeIn := &lambda.InvokeInput{FunctionName: aws.String("e2e-fn"), Payload: []byte(`{"a":40,"b":2}`)}
+	_, err = cli.Invoke(invokeIn)
+	require.ErrorAs(t, err, &aerr)
+	require.Equal(t, lambda.ErrCodeResourceConflictException, aerr.Code())
+	meta.active.Store(true)
+	out, err := cli.Invoke(invokeIn)
+	require.NoError(t, err)
+	require.Equal(t, int64(http.StatusOK), aws.Int64Value(out.StatusCode))
+	require.Equal(t, invokeIn.Payload, out.Payload)
+	require.Equal(t, "Handled", aws.StringValue(out.FunctionError))
+	require.Equal(t, "$LATEST", aws.StringValue(out.ExecutedVersion))
+	exec.mu.Lock()
+	require.Equal(t, aws.StringValue(cfg.CodeSha256), base64.StdEncoding.EncodeToString(mustHex(t, exec.last.Function.Code.SHA256)))
+	exec.mu.Unlock()
+	_, err = cli.Invoke(&lambda.InvokeInput{FunctionName: aws.String("e2e-fn"), LogType: aws.String(lambda.LogTypeTail)})
+	require.ErrorAs(t, err, &aerr)
+	require.Equal(t, lambda.ErrCodeInvalidParameterValueException, aerr.Code())
+
 	// SSL_CERT_FILE (set by the fixture) makes the default client trust it.
 	resp, err := http.Get(aws.StringValue(got.Code.Location)) //nolint:gosec,noctx // URL is the presigned location under test
 	require.NoError(t, err)