fix(lambda): validate environment names before reserved keys

cmd/lambda-agent/start_testlink.go

@@ -12,12 +12,14 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
+	"sort"
 	"strings"
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/agent"
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/runtimeapi"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/lambdaenv"
 )
 
 const (
@@ -91,6 +93,9 @@ func (t testlink) run(parent context.Context) (int, error) {
 	}
 	cred := *first.Credential
 	first.Credential = nil
+	if err := lambdaenv.Validate(cred.Environment); err != nil {
+		return exitRefused, errors.New("first frame carries an invalid function environment")
+	}
 	if sdkErr != nil {
 		slog.Warn("lambda-agent: SDK gateway settings unavailable", "err", sdkErr)
 	}
@@ -142,19 +147,31 @@ func (t testlink) run(parent context.Context) (int, error) {
 
 // bootstrapEnv supplies the runtime, execution-role and SDK settings.
 func bootstrapEnv(runtimeAPI, handler string, cred hostlink.Credential, region string, sdk *sdkConfig, caPath string) []string {
-	env := []string{
-		"AWS_LAMBDA_RUNTIME_API=" + runtimeAPI,
-		"_HANDLER=" + handler,
-		"LAMBDA_TASK_ROOT=" + taskRoot,
-		"PATH=/bin:/sbin:/usr/bin",
-		"AWS_ACCESS_KEY_ID=" + cred.AccessKeyID,
-		"AWS_SECRET_ACCESS_KEY=" + cred.SecretAccessKey,
-		"AWS_SESSION_TOKEN=" + cred.SessionToken,
-		"AWS_REGION=" + region,
-		"AWS_DEFAULT_REGION=" + region,
+	vars := map[string]string{
+		"AWS_LAMBDA_RUNTIME_API": runtimeAPI,
+		"_HANDLER":               handler,
+		"LAMBDA_TASK_ROOT":       taskRoot,
+		"PATH":                   "/bin:/sbin:/usr/bin",
+		"AWS_ACCESS_KEY_ID":      cred.AccessKeyID,
+		"AWS_SECRET_ACCESS_KEY":  cred.SecretAccessKey,
+		"AWS_SESSION_TOKEN":      cred.SessionToken,
+		"AWS_REGION":             region,
+		"AWS_DEFAULT_REGION":     region,
 	}
 	if sdk != nil {
-		env = append(env, "AWS_ENDPOINT_URL="+sdk.endpoint, "AWS_CA_BUNDLE="+caPath)
+		vars["AWS_ENDPOINT_URL"], vars["AWS_CA_BUNDLE"] = sdk.endpoint, caPath
+	}
+	for k, v := range cred.Environment {
+		vars[k] = v
+	}
+	keys := make([]string, 0, len(vars))
+	for k := range vars {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	env := make([]string, 0, len(keys))
+	for _, k := range keys {
+		env = append(env, k+"="+vars[k])
 	}
 	return env
 }

cmd/lambda-agent/start_testlink_test.go

@@ -221,6 +221,48 @@ func TestTestlinkProvidesSDKSettingsToBootstrap(t *testing.T) {
 	assert.Equal(t, os.FileMode(0o644), info.Mode().Perm())
 }
 
+func TestTestlinkProvidesFunctionEnvironment(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	var b bootstrapStarts
+	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
+	conn := h.guest(t)
+	defer conn.Close()
+	hs := hostlink.NewTestStream(conn)
+	cred := testCredential
+	cred.Environment = map[string]string{"GREETING": "hello $WORLD", "PATH": "/my/bin"}
+	req := hostlink.Request{Credential: &cred, GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: []byte(`{}`)}
+	require.NoError(t, hs.HostSend(req))
+	_, err := hs.HostResult(ctx)
+	require.NoError(t, err)
+	require.NoError(t, wait(t, done, "agent to stop").err)
+	assert.Equal(t, "hello $WORLD", envValue(b.env, "GREETING"))
+	assert.Equal(t, "/my/bin", envValue(b.env, "PATH"))
+	assert.Equal(t, testCredential.SecretAccessKey, envValue(b.env, "AWS_SECRET_ACCESS_KEY"))
+}
+
+func TestTestlinkRejectsReservedFunctionEnvironment(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	var b bootstrapStarts
+	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
+	conn := h.guest(t)
+	defer conn.Close()
+	cred := testCredential
+	cred.Environment = map[string]string{"AWS_SECRET_ACCESS_KEY": "forged"}
+	req := hostlink.Request{Credential: &cred, GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: []byte(`{}`)}
+	require.NoError(t, hostlink.NewTestStream(conn).HostSend(req))
+	result := wait(t, done, "agent to refuse the environment")
+	assert.Equal(t, exitRefused, result.code)
+	assert.Error(t, result.err)
+	assert.NotContains(t, result.err.Error(), "forged")
+	assert.Zero(t, b.n)
+}
+
 var testCredential = hostlink.Credential{AccessKeyID: "ASIAGUESTKEY", SecretAccessKey: "guest-secret", SessionToken: "guest-token"}
 
 func envValue(env []string, name string) string {

contracts/lambda/v1/README.md

@@ -21,14 +21,14 @@ Tenant code may be able to reach the listener, but it cannot open the agent's co
 **Authentication:** mutual TLS 1.3, bound to the environment ID and realization generation, using material the user runtime cannot read (ADR-0009:S17).
 For each generation the executor creates ephemeral self-signed host and guest certificates whose single URI SAN is `spinifex-lambda://<Region>/<account>/<function ID>/<environment ID>/<generation>/<role>`, with role `host` or `guest`.
 Each end validates the peer certificate against the one pinned peer certificate and separately requires that URI SAN with the opposite role; hostname verification is not relied on.
-ALPN carries the L-GUEST major version, `spinifex-lambda-guest/1`, and an unsupported version fails the handshake. Builds with the `lambdatestlink` tag offer only `spinifex-lambda-testlink/2` on both host and guest, whose first frame carries the execution-role credential, so a test-link end and a release end fail the handshake; release builds are unchanged.
-In the same builds, the gateway reaches the host daemon on the local abstract socket `spinifex-lambda-testlink-control/3`, and each request carries the runtime-image digest the gateway was built with; the daemon refuses a missing or different digest before starting the attempt, and a gateway and daemon on different socket names never connect.
+ALPN carries the L-GUEST major version, `spinifex-lambda-guest/1`, and an unsupported version fails the handshake. Builds with the `lambdatestlink` tag offer only `spinifex-lambda-testlink/3` on both host and guest, whose first frame carries the execution-role credential and function environment, so a test-link end and a release end fail the handshake; release builds are unchanged.
+In the same builds, the gateway reaches the host daemon on the local abstract socket `spinifex-lambda-testlink-control/4`, and each request carries the runtime-image digest the gateway was built with; the daemon refuses a missing or different digest before starting the attempt, and a gateway and daemon on different socket names never connect.
 On that socket a request's raw payload is at most 6 MiB, checked by the gateway before encoding and by the daemon after decoding.
 Its encoded frame is at most the base64 length of 6 MiB plus 64 KiB of metadata headroom and 64 bytes of field slack, and a receiver checks the frame length before allocating; metadata has no separate bound within the frame.
-The credential travels as its own frame of at most 8 KiB, and the reply has the same payload and frame bounds as the request.
+The credential and environment travel as their own frame of at most 40 KiB, and the reply has the same payload and frame bounds as the request.
 The daemon holds at most two connections at once, from accept to close, and leaves the rest unaccepted in the listen backlog.
 Under TLS 1.3 the guest's handshake can complete before the host refuses its certificate, so the guest treats the connection as accepted only after the host's first message.
-In the same builds, the function owner moves a created function with stored code from `Pending` to `Active` once this node's daemon serves the pinned image and the cluster configuration has exactly one node; a function with environment variables or a code copy over the test-link bound stays `Pending`, and `StateReason` names the unmet condition.
+In the same builds, the function owner moves a created function with stored code from `Pending` to `Active` once this node's daemon serves the pinned image and the cluster configuration has exactly one node; a code copy over the test-link bound stays `Pending`, and `StateReason` names the unmet condition.
 Release builds never leave `Pending`.
 The Mulga deploy role installs a test-link build with `deploy_lambda_testlink=true`; this profile does not meet Q-199 fail-closed revocation, and its guests take management addresses whose record can expire (mulga-0hsub).
 

spinifex/agents/lambda/hostlink/link.go

@@ -28,11 +28,12 @@ type Request struct {
 	Credential *Credential `json:"credential,omitempty"`
 }
 
-// Credential is one execution-role session. It is never logged or stored.
+// Credential holds one execution-role session and its function environment.
 type Credential struct {
-	AccessKeyID     string `json:"access_key_id"`
-	SecretAccessKey string `json:"secret_access_key"`
-	SessionToken    string `json:"session_token"`
+	AccessKeyID     string            `json:"access_key_id"`
+	SecretAccessKey string            `json:"secret_access_key"`
+	SessionToken    string            `json:"session_token"`
+	Environment     map[string]string `json:"environment,omitempty"`
 }
 
 // LogValue keeps every value out of logs.

spinifex/agents/lambda/hostlink/protocol_testlink.go

@@ -4,5 +4,5 @@ package hostlink
 
 // Protocol is the only ALPN protocol a test-link host and guest offer. It
 // differs from the release protocol because the first frame carries the
-// execution credential, which an older guest would ignore.
-const Protocol = "spinifex-lambda-testlink/2"
+// execution credential and function environment, which an older guest would ignore.
+const Protocol = "spinifex-lambda-testlink/3"

spinifex/domains/lambda/awsapi/handler_test.go

@@ -553,7 +553,6 @@ func TestCreateFunction_ParameterValidation(t *testing.T) {
 		"missing role":    func(b map[string]any) { b["Role"] = "arn:aws:iam::" + gwtest.AccountID + ":role/ghost" },
 		"not a role arn":  func(b map[string]any) { b["Role"] = "arn:aws:iam::" + gwtest.AccountID + ":user/alice" },
 		"malformed body":  func(b map[string]any) { b["Timeout"] = "soon" },
-		"empty env name":  func(b map[string]any) { b["Environment"] = map[string]any{"Variables": map[string]string{"": "x"}} },
 		"description len": func(b map[string]any) { b["Description"] = strings.Repeat("d", 257) },
 	}
 	for name, mutate := range cases {
@@ -566,6 +565,31 @@ func TestCreateFunction_ParameterValidation(t *testing.T) {
 	}
 }
 
+func TestCreateFunction_EnvironmentValidation(t *testing.T) {
+	for name, tc := range map[string]struct {
+		vars map[string]string
+		code string
+	}{
+		"empty name":           {map[string]string{"": "secret"}, awserrors.ErrorValidationException},
+		"one character":        {map[string]string{"A": "secret"}, awserrors.ErrorValidationException},
+		"invalid character":    {map[string]string{"BAD-NAME": "secret"}, awserrors.ErrorValidationException},
+		"invalid and reserved": {map[string]string{"A": "secret", "AWS_REGION": "secret"}, awserrors.ErrorValidationException},
+		"reserved":             {map[string]string{"AWS_SECRET_ACCESS_KEY": "secret"}, awserrors.ErrorInvalidParameterValueException},
+		"nul value":            {map[string]string{"VALUE": "secret\x00"}, awserrors.ErrorInvalidParameterValueException},
+		"too large":            {map[string]string{"VALUE": strings.Repeat("x", 4092)}, awserrors.ErrorInvalidParameterValueException},
+	} {
+		t.Run(name, func(t *testing.T) {
+			e := newEnv(t)
+			_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) {
+				b["Environment"] = map[string]any{"Variables": tc.vars}
+			}))
+			testutil.RequireAWSCode(t, err, tc.code)
+			assert.NotContains(t, err.Error(), "secret")
+			assert.Zero(t, e.store.count())
+		})
+	}
+}
+
 func TestCreateFunction_RoleErrorsDoNotLeakAcrossAccounts(t *testing.T) {
 	e := newEnv(t)
 	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) { b["Role"] = foreignRoleARN }))

spinifex/domains/lambda/awsapi/validate.go

@@ -4,6 +4,7 @@ import (
 	"archive/zip"
 	"bytes"
 	"encoding/json"
+	"errors"
 	"reflect"
 	"regexp"
 	"strings"
@@ -13,16 +14,16 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/domains/lambda/codestore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/lambdaenv"
 )
 
 const (
-	supportedRuntime  = "provided.al2023"
-	defaultTimeout    = 3
-	maxTimeout        = 900
-	defaultMemory     = 128
-	maxMemory         = 10240
-	maxDescription    = 256
-	maxEnvironmentLen = 4096
+	supportedRuntime = "provided.al2023"
+	defaultTimeout   = 3
+	maxTimeout       = 900
+	defaultMemory    = 128
+	maxMemory        = 10240
+	maxDescription   = 256
 	// maxBodyBytes bounds a CreateFunction body: the base64 of the largest
 	// code copy plus room for the rest of the document.
 	maxBodyBytes = (codestore.MaxCodeBytes/3+1)*4 + 1<<20
@@ -258,16 +259,14 @@ func envVars(e *lambda.Environment) (map[string]string, error) {
 		return nil, nil
 	}
 	vars := make(map[string]string, len(e.Variables))
-	size := 0
 	for k, v := range e.Variables {
-		if k == "" {
-			return nil, invalid("Environment variable names must not be empty")
-		}
 		vars[k] = aws.StringValue(v)
-		size += len(k) + len(vars[k])
 	}
-	if size > maxEnvironmentLen {
-		return nil, invalid("Environment variables exceed %d bytes", maxEnvironmentLen)
+	if err := lambdaenv.Validate(vars); err != nil {
+		if errors.Is(err, lambdaenv.ErrName) {
+			return nil, awserrors.Errorf(awserrors.ErrorValidationException, "Environment variable names must match [a-zA-Z][a-zA-Z0-9_]+")
+		}
+		return nil, invalid("%s", err)
 	}
 	return vars, nil
 }

spinifex/foundation/aws/lambdaenv/env.go

@@ -0,0 +1,50 @@
+// Package lambdaenv validates function environment variables.
+package lambdaenv
+
+import (
+	"errors"
+	"regexp"
+	"strings"
+)
+
+var (
+	ErrName     = errors.New("invalid environment variable name")
+	ErrReserved = errors.New("reserved environment variable name")
+	ErrValue    = errors.New("environment variable contains a NUL byte")
+	ErrSize     = errors.New("environment variables exceed 4096 bytes")
+)
+
+var namePattern = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9_]+$`)
+
+var reserved = map[string]struct{}{
+	"AWS_ACCESS_KEY": {}, "AWS_ACCESS_KEY_ID": {}, "AWS_SECRET_ACCESS_KEY": {}, "AWS_SESSION_TOKEN": {},
+	"AWS_DEFAULT_REGION": {}, "AWS_REGION": {}, "AWS_EXECUTION_ENV": {},
+	"AWS_LAMBDA_RUNTIME_API": {}, "AWS_LAMBDA_FUNCTION_NAME": {}, "AWS_LAMBDA_FUNCTION_MEMORY_SIZE": {},
+	"AWS_LAMBDA_FUNCTION_VERSION": {}, "AWS_LAMBDA_INITIALIZATION_TYPE": {},
+	"AWS_LAMBDA_LOG_GROUP_NAME": {}, "AWS_LAMBDA_LOG_STREAM_NAME": {},
+	"AWS_LAMBDA_MAX_CONCURRENCY": {}, "AWS_LAMBDA_METADATA_API": {}, "AWS_LAMBDA_METADATA_TOKEN": {},
+	"LAMBDA_TASK_ROOT": {}, "LAMBDA_RUNTIME_DIR": {},
+}
+
+// Validate checks values without including customer names or values in errors.
+func Validate(vars map[string]string) error {
+	for k := range vars {
+		if !namePattern.MatchString(k) {
+			return ErrName
+		}
+	}
+	size := 0
+	for k, v := range vars {
+		if _, ok := reserved[k]; ok {
+			return ErrReserved
+		}
+		if strings.ContainsRune(v, '\x00') {
+			return ErrValue
+		}
+		size += len(k) + len(v)
+	}
+	if size > 4096 {
+		return ErrSize
+	}
+	return nil
+}

spinifex/foundation/aws/lambdaenv/env_test.go

@@ -0,0 +1,33 @@
+package lambdaenv_test
+
+import (
+	"errors"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/lambdaenv"
+)
+
+func TestValidate(t *testing.T) {
+	for name, tc := range map[string]struct {
+		vars map[string]string
+		want error
+	}{
+		"absent": {},
+		"literal values and configurable settings": {vars: map[string]string{"GREETING": "hello $WORLD", "PATH": "/my/bin", "AWS_ENDPOINT_URL": "https://example.test"}},
+		"maximum size":                   {vars: map[string]string{"AA": strings.Repeat("x", 4094)}},
+		"empty name":                     {vars: map[string]string{"": "x"}, want: lambdaenv.ErrName},
+		"one-character name":             {vars: map[string]string{"A": "x"}, want: lambdaenv.ErrName},
+		"invalid character":              {vars: map[string]string{"bad-name": "x"}, want: lambdaenv.ErrName},
+		"reserved credential":            {vars: map[string]string{"AWS_SECRET_ACCESS_KEY": "x"}, want: lambdaenv.ErrReserved},
+		"invalid name precedes reserved": {vars: map[string]string{"A": "x", "AWS_REGION": "x"}, want: lambdaenv.ErrName},
+		"nul value":                      {vars: map[string]string{"VALUE": "x\x00"}, want: lambdaenv.ErrValue},
+		"over maximum":                   {vars: map[string]string{"AA": strings.Repeat("x", 4095)}, want: lambdaenv.ErrSize},
+	} {
+		t.Run(name, func(t *testing.T) {
+			if got := lambdaenv.Validate(tc.vars); !errors.Is(got, tc.want) {
+				t.Fatalf("Validate() = %v, want %v", got, tc.want)
+			}
+		})
+	}
+}

spinifex/runtime/compute/lambda/guestconn/protocol_testlink_test.go

@@ -18,8 +18,12 @@ import (
 
 // releaseOnly is cfg as a release build would offer it: ALPNv1 alone.
 func releaseOnly(cfg *tls.Config) *tls.Config {
+	return protocolOnly(cfg, lambdav1.ALPNv1)
+}
+
+func protocolOnly(cfg *tls.Config, protocol string) *tls.Config {
 	cfg = cfg.Clone()
-	cfg.NextProtos = []string{lambdav1.ALPNv1}
+	cfg.NextProtos = []string{protocol}
 	verify := cfg.VerifyConnection
 	cfg.VerifyConnection = func(cs tls.ConnectionState) error {
 		cs.NegotiatedProtocol = hostlink.Protocol // let only the ALPN offer differ
@@ -28,6 +32,42 @@ func releaseOnly(cfg *tls.Config) *tls.Config {
 	return cfg
 }
 
+func TestTestLinkRejectsPreviousProtocolBothWays(t *testing.T) {
+	const previous = "spinifex-lambda-testlink/2"
+	t.Run("old guest", func(t *testing.T) {
+		ln := listen(t)
+		g := newGeneration(t, gen1)
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		hostSide := acceptAsync(g, ln, ctx, 2*time.Second)
+		id := guestFor(t, g, ln.Addr().String())
+		raw, err := net.Dial("tcp", ln.Addr().String())
+		require.NoError(t, err)
+		defer raw.Close()
+		require.Error(t, tls.Client(raw, protocolOnly(id.ClientConfig(), previous)).HandshakeContext(ctx))
+		require.Error(t, (<-hostSide).err)
+	})
+	t.Run("old host", func(t *testing.T) {
+		ln := listen(t)
+		g := newGeneration(t, gen1)
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		hostErr := make(chan error, 1)
+		go func() {
+			c, err := ln.Accept()
+			if err != nil {
+				hostErr <- err
+				return
+			}
+			defer c.Close()
+			hostErr <- tls.Server(c, protocolOnly(g.ServerConfig(), previous)).HandshakeContext(ctx)
+		}()
+		_, err := hostlink.Dial(ctx, guestFor(t, g, ln.Addr().String()))
+		require.Error(t, err)
+		require.Error(t, <-hostErr)
+	})
+}
+
 func TestTestLinkProtocolIsNotTheReleaseProtocol(t *testing.T) {
 	assert.NotEqual(t, lambdav1.ALPNv1, hostlink.Protocol)
 }

spinifex/runtime/compute/lambda/testlink/control.go

@@ -23,7 +23,7 @@ import (
 // ControlName is the abstract Unix socket name the node's daemon serves. An
 // abstract name has no directory entry; the kernel frees it when the last
 // reference to the listening socket closes.
-const ControlName = "spinifex-lambda-testlink-control/3"
+const ControlName = "spinifex-lambda-testlink-control/4"
 
 // Service accounts at each end of the control socket.
 const (
@@ -43,9 +43,8 @@ func joinAfterFunc(ctx context.Context, f func()) func() {
 	}
 }
 
-// MaxCredentialBytes bounds the encoded credential; a session's three values
-// fit well inside it.
-const MaxCredentialBytes = 8 << 10
+// MaxCredentialBytes includes the credential and JSON-escaped function environment.
+const MaxCredentialBytes = 40 << 10
 
 // Declared control-socket bounds, independent of any message bus. Request and
 // Reply payloads are capped raw; the frame cap leaves MetadataHeadroomBytes
@@ -61,8 +60,8 @@ const (
 	MaxAdmitted = 2
 )
 
-// Credential is one execution-role session for one attempt; it goes to the
-// guest in the first frame and never on NATS, to disk or into a log.
+// Credential carries one execution-role session and its function environment
+// to the guest, never on NATS, to disk or into a log.
 type Credential = hostlink.Credential
 
 // ErrNotSent means the exchange failed before any request byte was written, so

spinifex/runtime/compute/lambda/testlink/control_test.go

@@ -83,6 +83,17 @@ func TestControl_OneRequestOneReply(t *testing.T) {
 	assert.Equal(t, testCred, s.cred.Load())
 }
 
+func TestControl_FullEnvironmentFitsWithCredential(t *testing.T) {
+	name := controlName(t)
+	s, _ := serveControl(t, name, os.Getuid(), nil)
+	cred := testCred
+	cred.SessionToken = string(bytes.Repeat([]byte("t"), 8<<10-128))
+	cred.Environment = map[string]string{"AA": string(bytes.Repeat([]byte{1}, 4094))}
+	_, err := client(name, os.Getuid()).Exchange(t.Context(), []byte(`{}`), cred, soon())
+	require.NoError(t, err)
+	assert.Equal(t, cred, s.cred.Load())
+}
+
 func TestControl_RequestSizeBoundary(t *testing.T) {
 	name := controlName(t)
 	s, _ := serveControl(t, name, os.Getuid(), nil)
@@ -219,7 +230,7 @@ func TestControl_BadFramesAreRefused(t *testing.T) {
 		"incomplete credential": append(rawFrame([]byte("{}")),
 			rawFrame([]byte(`{"access_key_id":"ASIA"}`))...),
 		"oversized credential": append(rawFrame([]byte("{}")),
-			rawFrame(bytes.Repeat([]byte("c"), MaxCredentialBytes+1))...),
+			rawFrame(bytes.Repeat([]byte("c"), MaxCredentialBytes+1))[:4]...),
 	}
 	for name, frame := range cases {
 		t.Run(name, func(t *testing.T) {
@@ -441,6 +452,7 @@ func TestControl_WriteErrorAfterDeliveryIsUnknown(t *testing.T) {
 const (
 	legacyControlName = "spinifex-lambda-testlink-control"
 	v2ControlName     = "spinifex-lambda-testlink-control/2"
+	v3ControlName     = "spinifex-lambda-testlink-control/3"
 )
 
 // A gateway and daemon on different control protocol names never connect, in
@@ -453,6 +465,8 @@ func TestControl_MixedProtocolNamesNeverConnect(t *testing.T) {
 		"new daemon, unversioned gateway": {prefix + ControlName, prefix + legacyControlName},
 		"/2 daemon, new gateway":          {prefix + v2ControlName, prefix + ControlName},
 		"new daemon, /2 gateway":          {prefix + ControlName, prefix + v2ControlName},
+		"/3 daemon, new gateway":          {prefix + v3ControlName, prefix + ControlName},
+		"new daemon, /3 gateway":          {prefix + ControlName, prefix + v3ControlName},
 	} {
 		t.Run(name, func(t *testing.T) {
 			s, _ := serveControl(t, tc.daemon, os.Getuid(), nil)
@@ -478,7 +492,9 @@ func TestControl_AcceptStoppedRunsBeforeTheDrain(t *testing.T) {
 		AcceptStopped: func() { close(stopped) }}
 	done := make(chan error, 1)
 	go func() { done <- srv.Serve(context.Background(), ln) }()
-	go func() { _, _ = client(name, os.Getuid()).Exchange(context.Background(), []byte("{}"), testCred, soon()) }()
+	go func() {
+		_, _ = client(name, os.Getuid()).Exchange(context.Background(), []byte("{}"), testCred, soon())
+	}()
 	select {
 	case <-inHandle:
 	case <-time.After(5 * time.Second):

spinifex/runtime/compute/lambda/testlink/launch_record_test.go

@@ -64,7 +64,9 @@ func TestDaemon_StartThenFailedLaunchedWriteIsUnknown(t *testing.T) {
 	owner := &failLaunched{Grants: d.Grants}
 	d.Grants = owner
 
-	rep := d.Handle(context.Background(), g.request(srv.URL, code), testCred)
+	cred := testCred
+	cred.Environment = map[string]string{"GREETING": "private-function-environment"}
+	rep := d.Handle(context.Background(), g.request(srv.URL, code), cred)
 	// An identity is read only from a live process's pidfd, so one was started.
 	require.Len(t, owner.got, 1, "no process was started, so this run proves nothing about after-start replies")
 	require.NotZero(t, owner.got[0].PID)
@@ -72,7 +74,7 @@ func TestDaemon_StartThenFailedLaunchedWriteIsUnknown(t *testing.T) {
 	require.Equal(t, OutcomeUnknown, rep.Outcome, rep.Error)
 	require.Equal(t, ErrLaunchNotRecorded.Error(), rep.Error)
 
-	secrets := []string{testCred.AccessKeyID, testCred.SecretAccessKey, testCred.SessionToken, srv.URL}
+	secrets := []string{cred.AccessKeyID, cred.SecretAccessKey, cred.SessionToken, cred.Environment["GREETING"], srv.URL}
 	files := 0
 	require.NoError(t, filepath.WalkDir(d.WorkDir, func(p string, e fs.DirEntry, err error) error {
 		if err != nil || e.IsDir() {

spinifex/runtime/compute/lambda/testlink/readiness.go

@@ -13,9 +13,6 @@ import (
 // handed an Active function it cannot run.
 func (d *Daemon) Readiness(imagePin string, nodes int) function.Readiness {
 	return func(_ context.Context, rec function.Record) error {
-		if rec.EnvCipher != "" {
-			return notReady("The function cannot run here: test-link Lambda does not support environment variables.")
-		}
 		if !validHandler(rec.Handler) {
 			return notReady("The function cannot run here: test-link Lambda runs only a handler of 1 to 128 characters with no whitespace.")
 		}

spinifex/runtime/compute/lambda/testlink/readiness_test.go

@@ -133,8 +133,7 @@ func TestReadiness_RefusesFunctionsTheTestLinkCannotRun(t *testing.T) {
 	storedFunctionHandler(t, svc, "edgehandler", strings.Repeat("h", 128), "", 9)
 
 	require.NoError(t, svc.ActivatePending(t.Context()))
-	assert.Equal(t, "The function cannot run here: test-link Lambda does not support environment variables.", functionState(t, svc, "env").StateReason)
-	assert.Equal(t, function.StatePending, functionState(t, svc, "env").State)
+	assert.Equal(t, function.StateActive, functionState(t, svc, "env").State)
 	assert.Equal(t, "The function cannot run here: its code is larger than test-link Lambda accepts.", functionState(t, svc, "huge").StateReason)
 	assert.Equal(t, function.StatePending, functionState(t, svc, "huge").State)
 	assert.Equal(t, function.StateActive, functionState(t, svc, "edge").State, "the largest accepted code copy activates")
@@ -159,7 +158,9 @@ func TestReadiness_ServingEndsWhenAcceptingStops(t *testing.T) {
 	}
 	d.serve(t.Context(), ln, srv)
 	require.True(t, d.Serving())
-	go func() { _, _ = (&ControlClient{Name: name, DaemonUID: os.Getuid(), RequestLimit: MaxFrameBytes, ReplyLimit: MaxFrameBytes, IOTimeout: 5 * time.Second}).Exchange(context.Background(), []byte("{}"), testCred, time.Now().Add(5*time.Second)) }()
+	go func() {
+		_, _ = (&ControlClient{Name: name, DaemonUID: os.Getuid(), RequestLimit: MaxFrameBytes, ReplyLimit: MaxFrameBytes, IOTimeout: 5 * time.Second}).Exchange(context.Background(), []byte("{}"), testCred, time.Now().Add(5*time.Second))
+	}()
 	select {
 	case <-inHandle:
 	case <-time.After(5 * time.Second):

spinifex/runtime/roles/awsgw/awsgw.go

@@ -407,7 +407,7 @@ func launchService(config *config.ClusterConfig) error {
 		Roles:  iamdomain.NewRoleResolver(iamService),
 		Trusts: lambdaRoleTrusts(iamService),
 		AEAD:   lambdaKey.AEAD,
-		Exec:   lambdaTestExecutor(natsConn, config.Node, lambdaCodeStore, stsService),
+		Exec:   lambdaTestExecutor(natsConn, config.Node, lambdaCodeStore, stsService, lambdaKey.AEAD),
 		Mappings: func(ctx context.Context) (lambdaawsapi.MappingStore, error) {
 			return lambdaMappings(ctx)
 		},

spinifex/runtime/roles/awsgw/lambda_testlink.go

@@ -4,6 +4,7 @@ package awsgw
 
 import (
 	"context"
+	"crypto/cipher"
 	"log/slog"
 	"time"
 
@@ -18,7 +19,7 @@ import (
 
 // lambdaTestExecutor sends each Invoke to this node's test-link daemon over
 // the control socket. Without the daemon's service account it does not serve.
-func lambdaTestExecutor(nc *nats.Conn, node string, codes *codestore.Manager, sessions lambdaEnvironmentRoleIssuer) awsapi.Executor {
+func lambdaTestExecutor(nc *nats.Conn, node string, codes *codestore.Manager, sessions lambdaEnvironmentRoleIssuer, aead cipher.AEAD) awsapi.Executor {
 	daemonUID, err := testlink.LookupUID(testlink.DaemonUser)
 	if err != nil {
 		slog.Error("Lambda test link: control peer account unavailable; Invoke is not served", "err", err)
@@ -31,6 +32,7 @@ func lambdaTestExecutor(nc *nats.Conn, node string, codes *codestore.Manager, se
 		Control:      testLinkControl(daemonUID),
 		RequestLimit: testlink.MaxFrameBytes,
 		Sessions:     sessions,
+		AEAD:         aead,
 		Node:         node,
 		ImageDigest:  lambdaImageDigest,
 		Codes: func(ctx context.Context) (codePresigner, error) {

spinifex/runtime/roles/awsgw/lambda_testlink_executor.go

@@ -4,6 +4,7 @@ package awsgw
 
 import (
 	"context"
+	"crypto/cipher"
 	"crypto/rand"
 	"encoding/hex"
 	"encoding/json"
@@ -21,6 +22,7 @@ import (
 	stsdomain "github.com/mulgadc/spinifex/spinifex/domains/sts"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/lambdaenv"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/testlink"
 )
 
@@ -58,12 +60,13 @@ type testLinkExecutor struct {
 	// RequestLimit bounds the encoded request frame, without its credential.
 	RequestLimit int64
 	Sessions     lambdaEnvironmentRoleIssuer
+	AEAD         cipher.AEAD
 	Node         string
 	// ImageDigest is the runtime image every request requires of the daemon.
 	ImageDigest string
-	Codes  func(context.Context) (codePresigner, error)
-	Grants func(context.Context) (grantIssuer, error)
-	now    func() time.Time
+	Codes       func(context.Context) (codePresigner, error)
+	Grants      func(context.Context) (grantIssuer, error)
+	now         func() time.Time
 }
 
 var _ awsapi.Executor = (*testLinkExecutor)(nil)
@@ -78,11 +81,16 @@ func (e *testLinkExecutor) InvokeSync(ctx context.Context, in awsapi.SyncInvocat
 		return awsapi.SyncResult{}, &awsapi.RefusalError{Err: awserrors.Errorf(awserrors.ErrorServiceException,
 			"the test execution path has no valid runtime image pin")}
 	}
-	// The test link cannot deliver environment variables, so it refuses a
-	// function that has any before presigning, granting or sending.
+	var environment map[string]string
 	if rec.EnvCipher != "" {
-		return awsapi.SyncResult{}, &awsapi.RefusalError{Err: awserrors.Errorf(awserrors.ErrorServiceException,
-			"the test execution path does not support function environment variables")}
+		if e.AEAD == nil {
+			return awsapi.SyncResult{}, &awsapi.RefusalError{Err: awserrors.Errorf(awserrors.ErrorServiceException, "the function environment is unavailable")}
+		}
+		var err error
+		environment, err = function.OpenEnv(e.AEAD, rec.AccountID, rec.FunctionID, rec.EnvCipher)
+		if err != nil || lambdaenv.Validate(environment) != nil {
+			return awsapi.SyncResult{}, &awsapi.RefusalError{Err: awserrors.Errorf(awserrors.ErrorServiceException, "the function environment is unavailable")}
+		}
 	}
 	if rec.Code.SHA256 == "" || rec.Code.Size <= 0 || rec.Code.Size > testlink.MaxCodeBytes {
 		return awsapi.SyncResult{}, &awsapi.RefusalError{Err: fmt.Errorf("code copy of %d bytes is outside the test bound", rec.Code.Size)}
@@ -152,6 +160,7 @@ func (e *testLinkExecutor) InvokeSync(ctx context.Context, in awsapi.SyncInvocat
 		AccessKeyID:     aws.StringValue(session.Credentials.AccessKeyId),
 		SecretAccessKey: aws.StringValue(session.Credentials.SecretAccessKey),
 		SessionToken:    aws.StringValue(session.Credentials.SessionToken),
+		Environment:     environment,
 	}
 	revoke := func() error {
 		rctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), revokeTimeout)

spinifex/runtime/roles/awsgw/lambda_testlink_executor_test.go

@@ -4,6 +4,8 @@ package awsgw
 
 import (
 	"context"
+	"crypto/aes"
+	"crypto/cipher"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -165,6 +167,61 @@ func TestLambdaTestLinkExecutor_RefusesWithoutSending(t *testing.T) {
 	}
 }
 
+func TestLambdaTestLinkExecutor_OpensEnvironmentBeforeDispatch(t *testing.T) {
+	block, err := aes.NewCipher(make([]byte, 32))
+	require.NoError(t, err)
+	aead, err := cipher.NewGCM(block)
+	require.NoError(t, err)
+	vars := map[string]string{"GREETING": "private environment value"}
+	sealed, err := function.SealEnv(aead, "123456789012", "fn1", vars)
+	require.NoError(t, err)
+	input := invocation([]byte(`{}`))
+	input.Function.EnvCipher = sealed
+	n := &fakeControl{max: 1 << 20, reply: &testlink.Reply{Outcome: testlink.OutcomeDone}}
+	e := executor(n, nil, grantStore(t))
+	e.AEAD = aead
+	_, err = e.InvokeSync(t.Context(), input)
+	require.NoError(t, err)
+	require.Len(t, n.creds, 1)
+	require.Equal(t, vars, n.creds[0].Environment)
+	require.NotContains(t, string(n.sent[0]), vars["GREETING"])
+}
+
+func TestLambdaTestLinkExecutor_RefusesUnreadableEnvironment(t *testing.T) {
+	block, err := aes.NewCipher(make([]byte, 32))
+	require.NoError(t, err)
+	aead, err := cipher.NewGCM(block)
+	require.NoError(t, err)
+	wrongAccount, err := function.SealEnv(aead, "000000000001", "fn1", map[string]string{"GREETING": "private value"})
+	require.NoError(t, err)
+	reserved, err := function.SealEnv(aead, "123456789012", "fn1", map[string]string{"AWS_REGION": "private value"})
+	require.NoError(t, err)
+	for name, tc := range map[string]struct {
+		sealed string
+		aead   cipher.AEAD
+	}{
+		"key missing":       {wrongAccount, nil},
+		"wrong account":     {wrongAccount, aead},
+		"reserved variable": {reserved, aead},
+		"unreadable":        {"not ciphertext", aead},
+	} {
+		t.Run(name, func(t *testing.T) {
+			input := invocation(nil)
+			input.Function.EnvCipher = tc.sealed
+			n, store, sessions := &fakeControl{max: 1 << 20}, grantStore(t), &fakeSessions{}
+			e := executorWith(n, sessions, nil, store)
+			e.AEAD = tc.aead
+			_, err := e.InvokeSync(t.Context(), input)
+			var refusal *awsapi.RefusalError
+			require.ErrorAs(t, err, &refusal)
+			require.NotContains(t, err.Error(), "private value")
+			require.Empty(t, n.sent)
+			require.Empty(t, sessions.issued)
+			require.Empty(t, grantState(t, store))
+		})
+	}
+}
+
 // A gateway built without a valid image pin refuses before presigning,
 // granting, issuing or connecting.
 func TestLambdaTestLinkExecutor_InvalidImagePinRefusesBeforeConnecting(t *testing.T) {

spinifex/runtime/roles/awsgw/lambda_testlink_off.go

@@ -3,6 +3,7 @@
 package awsgw
 
 import (
+	"crypto/cipher"
 	"github.com/mulgadc/spinifex/spinifex/domains/lambda/awsapi"
 	"github.com/mulgadc/spinifex/spinifex/domains/lambda/codestore"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
@@ -11,7 +12,7 @@ import (
 
 // Without the test link there is no executor, so Invoke refuses before it
 // presigns, sends or allocates anything, and Lambda stays unrouted.
-func lambdaTestExecutor(*nats.Conn, string, *codestore.Manager, lambdaEnvironmentRoleIssuer) awsapi.Executor {
+func lambdaTestExecutor(*nats.Conn, string, *codestore.Manager, lambdaEnvironmentRoleIssuer, cipher.AEAD) awsapi.Executor {
 	return nil
 }
 

spinifex/runtime/roles/awsgw/lambda_testlink_off_test.go

@@ -51,7 +51,7 @@ func (e *spyExec) InvokeSync(context.Context, lambdaawsapi.SyncInvocation) (lamb
 // A default build composes the gateway with Lambda unrouted and no executor: a
 // signed Invoke is refused before any Lambda handler runs and starts no attempt.
 func TestDefaultBuild_InvokeNeverReachesLambda(t *testing.T) {
-	require.Nil(t, lambdaTestExecutor(nil, "node1", nil, nil), "a default build has no Lambda executor")
+	require.Nil(t, lambdaTestExecutor(nil, "node1", nil, nil, nil), "a default build has no Lambda executor")
 
 	meta, exec := &spyFunctions{}, &spyExec{}
 	lambda := lambdaawsapi.NewRegistration(lambdaawsapi.Deps{Meta: meta, Exec: exec})

spinifex/runtime/roles/awsgw/lambda_testlink_seam_env_test.go

@@ -5,6 +5,7 @@ package awsgw
 
 import (
 	"context"
+	"crypto/cipher"
 	"crypto/rand"
 	"encoding/hex"
 	"encoding/json"
@@ -17,6 +18,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/mulgadc/bluebottle/pkg/masterkey"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
 	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
@@ -51,6 +53,7 @@ type seamEnv struct {
 	roleARN  string
 	grants   *execution.Store
 	sessions jetstream.KeyValue
+	aead     cipher.AEAD
 }
 
 func newSeamEnv(t *testing.T) *seamEnv {
@@ -58,6 +61,8 @@ func newSeamEnv(t *testing.T) *seamEnv {
 	_, nc, _ := testutil.StartTestJetStream(t)
 	masterKey, err := iamdomain.GenerateMasterKey()
 	require.NoError(t, err)
+	lambdaKey, err := masterkey.New(masterKey)
+	require.NoError(t, err)
 	iamSvc, err := iamdomain.NewIAMServiceImpl(t.Context(), nc, masterKey)
 	require.NoError(t, err)
 	stsSvc, err := stsdomain.NewSTSServiceImpl(t.Context(), nc, iamSvc, masterKey, seamNoJWKS{})
@@ -78,7 +83,7 @@ func newSeamEnv(t *testing.T) *seamEnv {
 	require.NoError(t, err)
 	sessions, err := js.KeyValue(t.Context(), stsdomain.KVBucketSessionCredentials)
 	require.NoError(t, err)
-	return &seamEnv{nc: nc, iam: iamSvc, sts: stsSvc, roleARN: aws.StringValue(role.Role.Arn), grants: grants, sessions: sessions}
+	return &seamEnv{nc: nc, iam: iamSvc, sts: stsSvc, roleARN: aws.StringValue(role.Role.Arn), grants: grants, sessions: sessions, aead: lambdaKey.AEAD}
 }
 
 // resolve authenticates cred through the real verifier and resolves its

spinifex/runtime/roles/awsgw/lambda_testlink_seam_test.go

@@ -29,6 +29,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/lambda/testlink"
+	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/require"
 )
 
@@ -212,7 +213,7 @@ func composedExecutor(t *testing.T, env *seamEnv, codeURL string, client *http.C
 	grants := &countedGrants{grantIssuer: env.grants}
 	presigns := &atomic.Int32{}
 	daemon := startSeamDaemon(t, &testlink.Daemon{Node: "node1", Grants: env.grants, HTTP: client}, guest)
-	exec := &testLinkExecutor{Control: daemon.client, RequestLimit: testlink.MaxFrameBytes, Sessions: env.sts, Node: "node1", ImageDigest: testImageDigest,
+	exec := &testLinkExecutor{Control: daemon.client, RequestLimit: testlink.MaxFrameBytes, Sessions: env.sts, AEAD: env.aead, Node: "node1", ImageDigest: testImageDigest,
 		Codes:  func(context.Context) (codePresigner, error) { return urlPresigner{codeURL, presigns}, nil },
 		Grants: func(context.Context) (grantIssuer, error) { return grants, nil },
 	}
@@ -375,16 +376,42 @@ func TestLambdaTestLinkSeam_RuntimeSeesInvokedARNAndHandler(t *testing.T) {
 	}
 }
 
-// A function with environment variables is refused before any code link is
-// presigned, any grant is issued or anything is sent to a node.
-func TestLambdaTestLinkSeam_EnvironmentVariablesRefusedBeforeDispatch(t *testing.T) {
+func TestLambdaTestLinkSeam_EnvironmentReachesGuestWithoutLeaking(t *testing.T) {
+	env := newSeamEnv(t)
+	var subscriptions []*nats.Subscription
+	for _, subject := range []string{">", "$JS.>", "$KV.>"} {
+		sub, err := env.nc.SubscribeSync(subject)
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = sub.Unsubscribe() })
+		subscriptions = append(subscriptions, sub)
+	}
+	require.NoError(t, env.nc.Flush())
+	vars := map[string]string{"GREETING": "private-environment-marker"}
+	sealed, err := function.SealEnv(env.aead, seamAccount, "fn1", vars)
+	require.NoError(t, err)
 	srv := codeServer(t)
-	run := invokeComposedAt(t, "/2015-03-31/functions/f/invocations", seamFunction{timeout: 10, envCipher: "sealed"},
-		srv.URL+"/code", srv.Client(), func(ctx context.Context, _ net.Conn) { <-ctx.Done() })
-	require.Equal(t, http.StatusInternalServerError, run.resp.StatusCode)
-	require.Equal(t, "ServiceException", run.resp.Header.Get("X-Amzn-Errortype"))
-	require.Contains(t, string(run.body), "does not support function environment variables")
-	require.Zero(t, run.presigns, "a code link was presigned")
-	require.Zero(t, run.grants, "a grant was issued")
-	require.Empty(t, run.requests, "the request reached a node")
+	creds := make(chan hostlink.Credential, 1)
+	run := invokeComposedIn(t, env, "/2015-03-31/functions/f/invocations", seamFunction{timeout: 10, envCipher: sealed},
+		srv.URL+"/code", srv.Client(), agentGuestWith(guestOpts{creds: creds}, func(base string) { respond(base, `{}`) }))
+	require.Equal(t, http.StatusOK, run.resp.StatusCode, string(run.body))
+	require.Equal(t, vars, recv(t, creds, "guest credential").Environment)
+	for _, request := range run.requests {
+		require.NotContains(t, string(request), vars["GREETING"])
+	}
+	for _, reply := range run.replies {
+		require.NotContains(t, string(reply), vars["GREETING"])
+	}
+	require.NoError(t, env.nc.Flush())
+	for _, sub := range subscriptions {
+		for {
+			msg, err := sub.NextMsg(50 * time.Millisecond)
+			if errors.Is(err, nats.ErrTimeout) {
+				break
+			}
+			require.NoError(t, err)
+			require.NotContains(t, msg.Subject, vars["GREETING"])
+			require.NotContains(t, string(msg.Data), vars["GREETING"])
+		}
+	}
+	require.NotContains(t, run.log, vars["GREETING"])
 }