fix(lambda): bound ListFunctions server work per request

Walk the account stream by exact sequence, up to the end a plain stream
info reports, so each request examines a fixed number of sequences
whatever they hold. Add an account-isolation test over two real
account buckets.

spinifex/domains/lambda/awsapi/get.go

@@ -56,6 +56,27 @@ func (d Deps) lookupTarget(ctx context.Context, c *call, name, q string) (functi
 	return rec, revision, err
 }
 
+// render opens the record's environment and renders its configuration; an
+// environment that cannot be opened fails rather than being left out.
+func (d Deps) render(ctx context.Context, rec function.Record, revision uint64) (*lambda.FunctionConfiguration, error) {
+	var env map[string]string
+	if rec.EnvCipher != "" {
+		if d.AEAD == nil {
+			return nil, serviceError("environment decryption is unavailable")
+		}
+		var err error
+		if env, err = function.OpenEnv(d.AEAD, rec.AccountID, rec.FunctionID, rec.EnvCipher); err != nil {
+			slog.ErrorContext(ctx, "Lambda: open environment failed", "function_id", rec.FunctionID, "err", err)
+			return nil, serviceError("could not read the function environment")
+		}
+	}
+	cfg, err := configuration(rec, revision, env)
+	if err != nil {
+		return nil, serviceError("could not build the function configuration")
+	}
+	return cfg, nil
+}
+
 // loadConfiguration resolves the requested $LATEST function and renders its
 // configuration; it is a metadata read and never touches the code store.
 func (d Deps) loadConfiguration(ctx context.Context, c *call, query string) (function.Record, *lambda.FunctionConfiguration, error) {
@@ -64,19 +85,9 @@ func (d Deps) loadConfiguration(ctx context.Context, c *call, query string) (fun
 		return function.Record{}, nil, err
 	}
 
-	var env map[string]string
-	if rec.EnvCipher != "" {
-		if d.AEAD == nil {
-			return function.Record{}, nil, serviceError("environment decryption is unavailable")
-		}
-		if env, err = function.OpenEnv(d.AEAD, rec.AccountID, rec.FunctionID, rec.EnvCipher); err != nil {
-			slog.ErrorContext(ctx, "Lambda: open environment failed", "function_id", rec.FunctionID, "err", err)
-			return function.Record{}, nil, serviceError("could not read the function environment")
-		}
-	}
-	cfg, err := configuration(rec, revision, env)
+	cfg, err := d.render(ctx, rec, revision)
 	if err != nil {
-		return function.Record{}, nil, serviceError("could not build the function configuration")
+		return function.Record{}, nil, err
 	}
 	return rec, cfg, nil
 }

spinifex/domains/lambda/awsapi/handler_test.go

@@ -60,6 +60,14 @@ type localClient struct {
 	configErr error
 	// configCalls counts UpdateConfiguration requests sent to the owner.
 	configCalls int
+	// deleteCalls counts Delete requests sent to the owner.
+	deleteCalls int
+	// deleteErr fails Delete before the owner sees it; loseDeleteReply
+	// reports a transport error after the owner committed it.
+	deleteErr       error
+	loseDeleteReply bool
+	// listErr fails List before the owner sees it.
+	listErr error
 }
 
 var _ function.Client = (*localClient)(nil)
@@ -169,6 +177,42 @@ func (c *localClient) UpdateConfiguration(ctx context.Context, account, region s
 	return r.Record, r.Revision, nil
 }
 
+func (c *localClient) List(ctx context.Context, account, region, marker string, maxItems int) ([]function.ListedFunction, string, error) {
+	c.mu.Lock()
+	fail := c.listErr
+	c.mu.Unlock()
+	if fail != nil {
+		return nil, "", fail
+	}
+	r, err := c.svc.List(ctx, &function.ListRequest{Region: region, Marker: marker, MaxItems: maxItems}, account)
+	if err != nil {
+		return nil, "", err
+	}
+	return r.Functions, r.NextMarker, nil
+}
+
+func (c *localClient) Delete(ctx context.Context, account, region string, rec function.Record, rev uint64) error {
+	c.mu.Lock()
+	hook, fail, lose := c.beforeCommit, c.deleteErr, c.loseDeleteReply
+	c.beforeCommit = nil
+	c.deleteCalls++
+	c.mu.Unlock()
+	if fail != nil {
+		return fail
+	}
+	if hook != nil {
+		hook()
+	}
+	if _, err := c.svc.Delete(ctx, &function.DeleteRequest{Region: region, FunctionName: rec.FunctionName,
+		FunctionID: rec.FunctionID, Revision: rev}, account); err != nil {
+		return err
+	}
+	if lose {
+		return errors.New("nats: timeout")
+	}
+	return nil
+}
+
 // memStore is an in-memory code store; objects is keyed by Ref.
 type memStore struct {
 	mu      sync.Mutex
@@ -864,7 +908,7 @@ func TestAuthorization(t *testing.T) {
 
 func TestRegistration_ResolvesActionsAndInventory(t *testing.T) {
 	reg := awsapi.NewRegistration(awsapi.Deps{})
-	assert.Equal(t, []string{"CreateEventSourceMapping", "CreateFunction", "DeleteEventSourceMapping", "GetEventSourceMapping", "GetFunction", "GetFunctionCodeSigningConfig", "GetFunctionConfiguration", "Invoke", "ListEventSourceMappings", "ListVersionsByFunction", "UpdateFunctionCode", "UpdateFunctionConfiguration"}, reg.Inventory.Registered)
+	assert.Equal(t, []string{"CreateEventSourceMapping", "CreateFunction", "DeleteEventSourceMapping", "DeleteFunction", "GetEventSourceMapping", "GetFunction", "GetFunctionCodeSigningConfig", "GetFunctionConfiguration", "Invoke", "ListEventSourceMappings", "ListFunctions", "ListVersionsByFunction", "UpdateFunctionCode", "UpdateFunctionConfiguration"}, reg.Inventory.Registered)
 	assert.Equal(t, "UpdateFunctionConfiguration", reg.ResolveAction(httptest.NewRequest(http.MethodPut, "/2015-03-31/functions/x/configuration", nil)))
 	assert.Equal(t, "ListVersionsByFunction", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x/versions", nil)))
 	assert.Equal(t, "DeleteEventSourceMapping", reg.ResolveAction(httptest.NewRequest(http.MethodDelete, "/2015-03-31/event-source-mappings/x", nil)))
@@ -873,7 +917,10 @@ func TestRegistration_ResolvesActionsAndInventory(t *testing.T) {
 	assert.Equal(t, "GetFunction", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x", nil)))
 	assert.Equal(t, "GetFunctionConfiguration", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x/configuration", nil)))
 	assert.Equal(t, "UpdateFunctionCode", reg.ResolveAction(httptest.NewRequest(http.MethodPut, "/2015-03-31/functions/x/code", nil)))
-	assert.Empty(t, reg.ResolveAction(httptest.NewRequest(http.MethodDelete, "/2015-03-31/functions/x", nil)))
+	assert.Equal(t, "DeleteFunction", reg.ResolveAction(httptest.NewRequest(http.MethodDelete, "/2015-03-31/functions/x", nil)))
+	assert.Equal(t, "ListFunctions", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions", nil)))
+	assert.Equal(t, "ListFunctions", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/", nil)))
+	assert.Empty(t, reg.ResolveAction(httptest.NewRequest(http.MethodDelete, "/2015-03-31/functions/x/configuration", nil)))
 }
 
 func TestGetFunctionConfiguration_ReturnsTheSameConfigurationAsGetFunction(t *testing.T) {

spinifex/domains/lambda/awsapi/invoke.go

@@ -57,13 +57,13 @@ func (r *RefusalError) Error() string { return "invocation refused before dispat
 
 func (r *RefusalError) Unwrap() error { return r.Err }
 
-// invokeTarget is the function and qualifier an Invoke request names, once
+// functionTarget is the function and qualifier a request names, once
 // every deterministic check on the request has passed.
-type invokeTarget struct{ Name, Qualifier string }
+type functionTarget struct{ Name, Qualifier string }
 
 // validateInvoke runs every check that needs no stored state: options, size,
 // payload JSON, function name and qualifier. Nothing here reads metadata.
-func validateInvoke(c *call) (*invokeTarget, error) {
+func validateInvoke(c *call) (*functionTarget, error) {
 	if err := checkInvokeOptions(c.Header); err != nil {
 		return nil, err
 	}
@@ -76,6 +76,12 @@ func validateInvoke(c *call) (*invokeTarget, error) {
 		return nil, awserrors.Errorf(awserrors.ErrorInvalidRequestContentException,
 			"Could not parse request body into json")
 	}
+	return qualifiedTarget(c)
+}
+
+// qualifiedTarget resolves the function name and the qualifier from either
+// its suffix or the Qualifier query, which c.Query must hold strictly parsed.
+func qualifiedTarget(c *call) (*functionTarget, error) {
 	if len(c.Params[0]) > maxFunctionNameLen {
 		return nil, invalid("1 validation error detected: Value at 'functionName' failed to satisfy constraint: Member must have length less than or equal to 256")
 	}
@@ -101,7 +107,7 @@ func validateInvoke(c *call) (*invokeTarget, error) {
 		}
 		q = query[0]
 	}
-	return &invokeTarget{Name: name, Qualifier: q}, nil
+	return &functionTarget{Name: name, Qualifier: q}, nil
 }
 
 // invoke implements Invoke for RequestResponse on $LATEST. serve has already

spinifex/domains/lambda/awsapi/listdelete.go

@@ -0,0 +1,107 @@
+package awsapi
+
+import (
+	"context"
+	"log/slog"
+	"net/http"
+	"strconv"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/lambda"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// singleQuery returns the one value of a query member; present but repeated
+// or empty is refused.
+func singleQuery(c *call, member, field string) (string, bool, error) {
+	v, ok := c.Query[member]
+	if !ok {
+		return "", false, nil
+	}
+	if len(v) != 1 || v[0] == "" {
+		return "", true, invalid("1 validation error detected: Value at '%s' failed to satisfy constraint: Member must have length greater than or equal to 1", field)
+	}
+	return v[0], true, nil
+}
+
+// listFunctions implements ListFunctions over $LATEST. Pages hold at most
+// function.MaxListPage functions, and may be short or empty while a
+// NextMarker remains, since each owner request also bounds its scan.
+func (d Deps) listFunctions(ctx context.Context, c *call) (reply, error) {
+	if c.Query.Has("MasterRegion") {
+		return reply{}, unsupported("MasterRegion")
+	}
+	if v, ok, err := singleQuery(c, "FunctionVersion", "functionVersion"); err != nil {
+		return reply{}, err
+	} else if ok && v != lambda.FunctionVersionAll {
+		return reply{}, invalid("1 validation error detected: Value '%s' at 'functionVersion' failed to satisfy constraint: Member must satisfy enum value set: [ALL]", v)
+	}
+	maxItems := function.MaxListPage
+	if v, ok, err := singleQuery(c, "MaxItems", "maxItems"); err != nil {
+		return reply{}, err
+	} else if ok {
+		n, err := strconv.Atoi(v)
+		if err != nil || n < 1 || n > maxListItems {
+			return reply{}, invalid("MaxItems must be between 1 and %d", maxListItems)
+		}
+		maxItems = min(n, function.MaxListPage)
+	}
+	marker, _, err := singleQuery(c, "Marker", "marker")
+	if err != nil {
+		return reply{}, err
+	}
+
+	listed, next, err := d.Meta.List(ctx, c.AccountID, c.Region, marker, maxItems)
+	if err != nil {
+		if _, coded := awserrors.ResolveErrorCode(err); coded {
+			return reply{}, err
+		}
+		slog.ErrorContext(ctx, "Lambda: list owner request failed", "err", err)
+		return reply{}, serviceError("the functions could not be listed; retry the request")
+	}
+	out := &lambda.ListFunctionsOutput{Functions: make([]*lambda.FunctionConfiguration, 0, len(listed))}
+	for _, l := range listed {
+		cfg, err := d.render(ctx, l.Record, l.Revision)
+		if err != nil {
+			return reply{}, err
+		}
+		// ListFunctions returns a subset of the configuration: AWS leaves the
+		// state and last-update members to GetFunction.
+		cfg.State, cfg.StateReason, cfg.StateReasonCode = nil, nil, nil
+		cfg.LastUpdateStatus, cfg.LastUpdateStatusReason, cfg.LastUpdateStatusReasonCode = nil, nil, nil
+		out.Functions = append(out.Functions, cfg)
+	}
+	if next != "" {
+		out.NextMarker = aws.String(next)
+	}
+	return reply{Status: http.StatusOK, Body: out}, nil
+}
+
+// deleteFunction implements DeleteFunction for a whole function. Any explicit
+// qualifier is refused before anything is read, so a request scoped to a
+// version can never delete the function.
+func (d Deps) deleteFunction(ctx context.Context, c *call) (reply, error) {
+	if c.Delete == nil {
+		return reply{}, serviceError("delete request was not validated")
+	}
+	if q := c.Delete.Qualifier; q != "" {
+		if q == "$LATEST" {
+			return reply{}, invalid("$LATEST version cannot be deleted without deleting the function.")
+		}
+		return reply{}, notFound(c.Region, c.AccountID, c.Delete.Name+":"+q)
+	}
+	rec, revision, err := d.lookupTarget(ctx, c, c.Delete.Name, "")
+	if err != nil {
+		return reply{}, err
+	}
+	if err := d.Meta.Delete(ctx, c.AccountID, c.Region, rec, revision); err != nil {
+		// A lost reply may follow the unbinding; the caller reads the function to learn which.
+		if _, coded := awserrors.ResolveErrorCode(err); coded {
+			return reply{}, err
+		}
+		slog.ErrorContext(ctx, "Lambda: delete owner request failed", "function_id", rec.FunctionID, "err", err)
+		return reply{}, serviceError("the function could not be deleted; retry the request")
+	}
+	return reply{Status: http.StatusNoContent}, nil
+}

spinifex/domains/lambda/awsapi/listdelete_test.go

@@ -0,0 +1,385 @@
+package awsapi_test
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/service/lambda"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/internal/testkit/gwtest"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/eventsource"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	listPath  = "/2015-03-31/functions"
+	helloPath = "/2015-03-31/functions/hello"
+	helloARN  = "arn:aws:lambda:" + gwtest.Region + ":" + gwtest.AccountID + ":function:hello"
+)
+
+// listAll walks ListFunctions from path, following NextMarker, and returns the
+// pages it read.
+func listAll(t *testing.T, e *env, path string) []lambda.ListFunctionsOutput {
+	t.Helper()
+	var pages []lambda.ListFunctionsOutput
+	marker := ""
+	for range 100 {
+		p := path
+		if marker != "" {
+			p += "&Marker=" + url.QueryEscape(marker)
+		}
+		rec, err := e.do(http.MethodGet, p, nil)
+		require.NoError(t, err)
+		require.Equal(t, http.StatusOK, rec.Code)
+		var out lambda.ListFunctionsOutput
+		require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out))
+		pages = append(pages, out)
+		if out.NextMarker == nil {
+			return pages
+		}
+		marker = *out.NextMarker
+	}
+	t.Fatal("the walk does not end")
+	return nil
+}
+
+func listedNames(pages []lambda.ListFunctionsOutput) []string {
+	var names []string
+	for _, p := range pages {
+		for _, f := range p.Functions {
+			names = append(names, *f.FunctionName)
+		}
+	}
+	return names
+}
+
+func createActive(t *testing.T, e *env, names ...string) {
+	t.Helper()
+	for _, name := range names {
+		_, err := e.do(http.MethodPost, listPath, createBody(name, nil))
+		require.NoError(t, err)
+	}
+	e.activate(t)
+}
+
+func TestListFunctions_WalksEveryFunctionAcrossPagesOnBothPaths(t *testing.T) {
+	e := newUpdateEnv(t)
+	createActive(t, e, "second", "third")
+
+	pages := listAll(t, e, listPath+"?MaxItems=1")
+	assert.Len(t, pages, 3, "a page that fills on the last function has no NextMarker")
+	assert.ElementsMatch(t, []string{"hello", "second", "third"}, listedNames(pages))
+	slash := listAll(t, e, listPath+"/?MaxItems=1")
+	assert.Equal(t, listedNames(pages), listedNames(slash))
+
+	rec, err := e.do(http.MethodGet, listPath, nil)
+	require.NoError(t, err)
+	assert.NotContains(t, rec.Body.String(), "NextMarker")
+}
+
+// ListFunctions renders each function as GetFunctionConfiguration does, with
+// its environment opened, less the state and last-update members.
+func TestListFunctions_RendersTheConfigurationSubset(t *testing.T) {
+	e := newUpdateEnv(t)
+	rec, err := e.do(http.MethodGet, helloPath+"/configuration", nil)
+	require.NoError(t, err)
+	want := decodeConfig(t, rec.Body.Bytes())
+	want.State, want.StateReason, want.StateReasonCode = nil, nil, nil
+	want.LastUpdateStatus, want.LastUpdateStatusReason, want.LastUpdateStatusReasonCode = nil, nil, nil
+
+	rec, err = e.do(http.MethodGet, listPath, nil)
+	require.NoError(t, err)
+	for _, member := range []string{`"State"`, `"StateReason`, `"LastUpdateStatus`} {
+		assert.NotContains(t, rec.Body.String(), member)
+	}
+	var out lambda.ListFunctionsOutput
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out))
+	require.Len(t, out.Functions, 1)
+	assert.Equal(t, want, *out.Functions[0])
+	assert.Equal(t, "prod", *out.Functions[0].Environment.Variables["MODE"])
+}
+
+func TestListFunctions_EmptyAccountReturnsAnEmptyList(t *testing.T) {
+	e := newEnv(t)
+	rec, err := e.do(http.MethodGet, listPath+"?FunctionVersion=ALL", nil)
+	require.NoError(t, err)
+	assert.JSONEq(t, `{"Functions":[]}`, rec.Body.String())
+}
+
+func TestListFunctions_PageHoldsAtMostFifty(t *testing.T) {
+	e := newEnv(t)
+	for i := range function.MaxListPage + 1 {
+		_, err := e.do(http.MethodPost, listPath, createBody(fmt.Sprintf("fn-%02d", i), nil))
+		require.NoError(t, err)
+	}
+	pages := listAll(t, e, listPath+"?MaxItems=10000")
+	require.Len(t, pages, 2)
+	assert.Len(t, pages[0].Functions, function.MaxListPage)
+	assert.Len(t, listedNames(pages), function.MaxListPage+1)
+}
+
+// Two accounts holding the same function names see only their own functions,
+// with their own environments, on first and continuation pages, and neither
+// can continue the other's walk.
+func TestListFunctions_AccountsAreIsolated(t *testing.T) {
+	e := newEnv(t)
+	const other = "210987654321"
+	e.deps.Roles = roleSet{roleARN: true, "arn:aws:iam::" + other + ":role/lambda-exec": true}
+	names := map[string][]string{
+		gwtest.AccountID: {"shared-1", "only-a", "shared-2", "shared-3"},
+		other:            {"shared-2", "shared-1", "only-b", "shared-3"},
+	}
+	for _, acct := range []string{gwtest.AccountID, other} {
+		e.h.AccountID = acct
+		for _, name := range names[acct] {
+			rec, err := e.do(http.MethodPost, listPath, createBody(name, func(b map[string]any) {
+				b["Role"] = "arn:aws:iam::" + acct + ":role/lambda-exec"
+				b["Environment"] = map[string]any{"Variables": map[string]string{"OWNER": acct}}
+			}))
+			require.NoError(t, err)
+			require.Equal(t, http.StatusCreated, rec.Code, rec.Body.String())
+		}
+	}
+	e.activate(t)
+
+	markers := map[string]string{}
+	for _, acct := range []string{gwtest.AccountID, other} {
+		e.h.AccountID = acct
+		pages := listAll(t, e, listPath+"?MaxItems=2")
+		require.Len(t, pages, 2, "the walk has a continuation page")
+		markers[acct] = *pages[0].NextMarker
+		assert.Equal(t, names[acct], listedNames(pages))
+		for _, p := range pages {
+			for _, f := range p.Functions {
+				assert.Equal(t, "arn:aws:lambda:"+gwtest.Region+":"+acct+":function:"+*f.FunctionName, *f.FunctionArn)
+				assert.Equal(t, "arn:aws:iam::"+acct+":role/lambda-exec", *f.Role)
+				assert.Equal(t, acct, *f.Environment.Variables["OWNER"])
+			}
+		}
+	}
+	for acct, foreign := range map[string]string{gwtest.AccountID: markers[other], other: markers[gwtest.AccountID]} {
+		e.h.AccountID = acct
+		_, err := e.do(http.MethodGet, listPath+"?Marker="+url.QueryEscape(foreign), nil)
+		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+	}
+}
+
+func TestListFunctions_ParameterValidation(t *testing.T) {
+	e := newUpdateEnv(t)
+	for name, query := range map[string]string{
+		"MaxItems zero":           "MaxItems=0",
+		"MaxItems over 10000":     "MaxItems=10001",
+		"MaxItems not a number":   "MaxItems=x",
+		"MaxItems repeated":       "MaxItems=1&MaxItems=2",
+		"empty Marker":            "Marker=",
+		"malformed Marker":        "Marker=not-a-marker",
+		"MasterRegion":            "MasterRegion=us-east-1",
+		"unknown FunctionVersion": "FunctionVersion=LATEST",
+		"malformed query":         "MaxItems=%zz",
+	} {
+		t.Run(name, func(t *testing.T) {
+			_, err := e.do(http.MethodGet, listPath+"?"+query, nil)
+			testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+		})
+	}
+}
+
+func TestListFunctions_AuthorizesOnEveryResource(t *testing.T) {
+	e := newUpdateEnv(t)
+	e.h = gwtest.Scoped(gwtest.Statement("Allow", "lambda:ListFunctions", helloARN))
+	_, err := e.do(http.MethodGet, listPath, nil)
+	gwtest.AssertDenied(t, err)
+	e.h = gwtest.Scoped(gwtest.Statement("Allow", "lambda:ListFunctions", "*"))
+	_, err = e.do(http.MethodGet, listPath, nil)
+	require.NoError(t, err)
+}
+
+func TestListFunctions_OwnerFailuresAreRetryable(t *testing.T) {
+	e := newUpdateEnv(t)
+	e.meta.listErr = fmt.Errorf("NATS request to %s: %w", function.SubjectList, nats.ErrNoResponders)
+	_, err := e.do(http.MethodGet, listPath, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
+	assert.False(t, awserrors.IsTerminal(err))
+
+	e.meta.listErr = nil
+	e.deps.AEAD = nil
+	_, err = e.do(http.MethodGet, listPath, nil)
+	// An environment that cannot be opened fails the page.
+	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
+}
+
+// deletion reads the retained obligation for functionID.
+func (e *env) deletion(t *testing.T, functionID string) (function.Deletion, bool) {
+	t.Helper()
+	entry, err := e.kv(t).Get(t.Context(), function.KVDeletedKey(gwtest.Region, functionID))
+	if errors.Is(err, jetstream.ErrKeyNotFound) {
+		return function.Deletion{}, false
+	}
+	require.NoError(t, err)
+	var d function.Deletion
+	require.NoError(t, json.Unmarshal(entry.Value(), &d))
+	return d, true
+}
+
+func TestDeleteFunction_RemovesTheFunctionAndRetainsItsRecordsAndCode(t *testing.T) {
+	e := newUpdateEnv(t)
+	before, _ := e.record(t, "hello")
+	copies := e.store.count()
+	// Deletion reads neither the code store nor the environment.
+	codes, aead := e.deps.Codes, e.deps.AEAD
+	e.deps.Codes, e.deps.AEAD = nil, nil
+
+	rec, err := e.do(http.MethodDelete, helloPath, nil)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusNoContent, rec.Code)
+	assert.Empty(t, rec.Body.Bytes())
+
+	_, err = e.do(http.MethodGet, helloPath+"/configuration", nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+	assert.Empty(t, listedNames(listAll(t, e, listPath+"?MaxItems=1")))
+	_, err = e.do(http.MethodDelete, helloPath, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+
+	d, ok := e.deletion(t, before.FunctionID)
+	require.True(t, ok)
+	assert.Equal(t, "hello", d.FunctionName)
+	assert.Contains(t, e.uploads(t), before.FunctionID)
+	assert.Equal(t, copies, e.store.count(), "no code copy is removed")
+
+	e.deps.Codes, e.deps.AEAD = codes, aead
+	_, err = e.do(http.MethodPost, listPath, createBody("hello", nil))
+	require.NoError(t, err)
+	again, _ := e.record(t, "hello")
+	assert.NotEqual(t, before.FunctionID, again.FunctionID)
+	_, ok = e.deletion(t, before.FunctionID)
+	assert.True(t, ok, "the old obligation survives the name's reuse")
+}
+
+// Every explicit qualifier, in the query or the name, and every query that
+// cannot be read exactly, is refused before the owner is asked to delete.
+func TestDeleteFunction_RefusesEveryQualifierWithoutMutation(t *testing.T) {
+	cases := map[string]struct {
+		path string
+		code string
+	}{
+		"query $LATEST":         {helloPath + "?Qualifier=%24LATEST", awserrors.ErrorInvalidParameterValueException},
+		"query version":         {helloPath + "?Qualifier=1", awserrors.ErrorResourceNotFoundException},
+		"query alias":           {helloPath + "?Qualifier=live", awserrors.ErrorResourceNotFoundException},
+		"empty query":           {helloPath + "?Qualifier=", awserrors.ErrorInvalidParameterValueException},
+		"bare query key":        {helloPath + "?Qualifier", awserrors.ErrorInvalidParameterValueException},
+		"repeated query":        {helloPath + "?Qualifier=1&Qualifier=1", awserrors.ErrorInvalidParameterValueException},
+		"malformed escape":      {helloPath + "?Qualifier=%zz", awserrors.ErrorInvalidParameterValueException},
+		"malformed other pair":  {helloPath + "?x=%zz&Qualifier=1", awserrors.ErrorInvalidParameterValueException},
+		"semicolon":             {helloPath + "?Qualifier=1;x=2", awserrors.ErrorInvalidParameterValueException},
+		"invalid pattern":       {helloPath + "?Qualifier=a.b", awserrors.ErrorInvalidParameterValueException},
+		"name suffix $LATEST":   {helloPath + ":%24LATEST", awserrors.ErrorInvalidParameterValueException},
+		"name suffix version":   {helloPath + ":1", awserrors.ErrorResourceNotFoundException},
+		"ARN suffix version":    {"/2015-03-31/functions/" + url.PathEscape(helloARN+":1"), awserrors.ErrorResourceNotFoundException},
+		"empty name suffix":     {helloPath + ":", awserrors.ErrorInvalidParameterValueException},
+		"mismatched qualifiers": {helloPath + ":1?Qualifier=2", awserrors.ErrorInvalidParameterValueException},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			e := newUpdateEnv(t)
+			before, beforeRev := e.record(t, "hello")
+			_, err := e.do(http.MethodDelete, tc.path, nil)
+			testutil.RequireAWSCode(t, err, tc.code)
+			assert.Zero(t, e.meta.deleteCalls)
+			requireConfigUnchanged(t, e, before, beforeRev)
+			_, ok := e.deletion(t, before.FunctionID)
+			assert.False(t, ok)
+		})
+	}
+}
+
+func TestDeleteFunction_Authorization(t *testing.T) {
+	allowAll := gwtest.Statement("Allow", "*", "*")
+	t.Run("denied on the function ARN", func(t *testing.T) {
+		e := newUpdateEnv(t)
+		e.h = gwtest.Scoped(allowAll, gwtest.Statement("Deny", "lambda:DeleteFunction", helloARN))
+		before, beforeRev := e.record(t, "hello")
+		_, err := e.do(http.MethodDelete, helloPath, nil)
+		gwtest.AssertDenied(t, err)
+		requireConfigUnchanged(t, e, before, beforeRev)
+	})
+	t.Run("another function's grant does not reach it", func(t *testing.T) {
+		e := newUpdateEnv(t)
+		e.h = gwtest.Scoped(gwtest.Statement("Allow", "lambda:DeleteFunction", helloARN+"-other"))
+		_, err := e.do(http.MethodDelete, helloPath, nil)
+		gwtest.AssertDenied(t, err)
+	})
+	t.Run("allowed on the function ARN", func(t *testing.T) {
+		e := newUpdateEnv(t)
+		e.h = gwtest.Scoped(gwtest.Statement("Allow", "lambda:DeleteFunction", helloARN))
+		rec, err := e.do(http.MethodDelete, helloPath, nil)
+		require.NoError(t, err)
+		assert.Equal(t, http.StatusNoContent, rec.Code)
+	})
+}
+
+func TestDeleteFunction_LostReplyIsAnErrorButTheDeleteStands(t *testing.T) {
+	e := newUpdateEnv(t)
+	e.meta.loseDeleteReply = true
+	_, err := e.do(http.MethodDelete, helloPath, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
+	assert.False(t, awserrors.IsTerminal(err))
+	_, err = e.do(http.MethodGet, helloPath+"/configuration", nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+}
+
+func TestDeleteFunction_OwnerWithoutTheSubjectIsRetryableAndChangesNothing(t *testing.T) {
+	e := newUpdateEnv(t)
+	before, beforeRev := e.record(t, "hello")
+	e.meta.deleteErr = fmt.Errorf("NATS request to %s: %w", function.SubjectDelete, nats.ErrNoResponders)
+	_, err := e.do(http.MethodDelete, helloPath, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorServiceException)
+	assert.False(t, awserrors.IsTerminal(err))
+	requireConfigUnchanged(t, e, before, beforeRev)
+}
+
+func TestDeleteFunction_UpdateBetweenReadAndDeleteWins(t *testing.T) {
+	e := newConfigEnv(t)
+	e.meta.beforeCommit = func() {
+		_, err := e.do(http.MethodPut, configPath, map[string]any{"Handler": "handler.winner"})
+		assert.NoError(t, err)
+	}
+	_, err := e.do(http.MethodDelete, helloPath, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
+	after, _ := e.record(t, "hello")
+	assert.Equal(t, "handler.winner", after.Handler)
+}
+
+func TestDeleteFunction_LeavesMappingsInPlace(t *testing.T) {
+	e, store, _ := newMappingEnv(t)
+	rec, err := e.do(http.MethodPost, mappingsPath, mappingBody(nil))
+	require.NoError(t, err)
+	uuid, _ := decodeMapping(t, rec.Body.Bytes())["UUID"].(string)
+	before, beforeRev, err := store.Get(t.Context(), uuid)
+	require.NoError(t, err)
+
+	_, err = e.do(http.MethodDelete, helloPath, nil)
+	require.NoError(t, err)
+	after, afterRev, err := store.Get(t.Context(), uuid)
+	require.NoError(t, err)
+	assert.Equal(t, beforeRev, afterRev)
+	assert.Equal(t, before, after)
+	assert.Equal(t, eventsource.StateEnabled, after.State)
+}
+
+func TestDeleteFunction_LaterInvokeIsNotFound(t *testing.T) {
+	e, ex := newInvokeEnv(t)
+	_, err := e.do(http.MethodDelete, helloPath, nil)
+	require.NoError(t, err)
+	_, err = e.invoke(t.Context(), invokePath, nil, nil)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+	assert.Empty(t, ex.calls)
+}

spinifex/domains/lambda/awsapi/registration.go

@@ -149,6 +149,20 @@ func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
 		}
 	case "GetEventSourceMapping", "DeleteEventSourceMapping":
 		resource = mappingARN(inv.Region, inv.AccountID, params[0])
+	case "ListFunctions":
+		if c.Query, err = url.ParseQuery(r.URL.RawQuery); err != nil {
+			return invalid("The request query string could not be parsed.")
+		}
+	case "DeleteFunction":
+		// A dropped Qualifier would widen the request to the whole function,
+		// so the query is parsed strictly and the target resolved before anything else.
+		if c.Query, err = url.ParseQuery(r.URL.RawQuery); err != nil {
+			return invalid("The request query string could not be parsed.")
+		}
+		if c.Delete, err = qualifiedTarget(c); err != nil {
+			return err
+		}
+		resource = qualifiedFunctionARN(inv.Region, inv.AccountID, c.Delete.Name, c.Delete.Qualifier)
 	case "Invoke":
 		// Invoke validates deterministically before authorizing, and its
 		// resource comes from that same validated target. URL.Query drops
@@ -234,6 +248,11 @@ func (d Deps) checkPassRole(inv dispatch.Invocation, roleARN string) error {
 }
 
 func writeJSON(w http.ResponseWriter, out reply) {
+	if out.Status == http.StatusNoContent {
+		maps.Copy(w.Header(), out.Header)
+		w.WriteHeader(out.Status)
+		return
+	}
 	var body []byte
 	var err error
 	if raw, ok := out.Body.(rawJSON); ok {

spinifex/domains/lambda/awsapi/routes.go

@@ -22,7 +22,10 @@ type call struct {
 	Create *lambda.CreateFunctionInput
 	// Invoke is Invoke's validated target, resolved ahead of the
 	// authorization gate whose resource it names.
-	Invoke *invokeTarget
+	Invoke *functionTarget
+	// Delete is DeleteFunction's validated target, resolved from the strictly
+	// parsed query ahead of the authorization gate.
+	Delete *functionTarget
 	// UpdateConfig is the decoded UpdateFunctionConfiguration body, read
 	// ahead of the PassRole check on a new role.
 	UpdateConfig *lambda.UpdateFunctionConfigurationInput
@@ -49,6 +52,18 @@ var routes = []route{
 		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
 			return d.createFunction(ctx, c)
 		}},
+	{Method: "GET", Pattern: "/2015-03-31/functions", Action: "ListFunctions",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.listFunctions(ctx, c)
+		}},
+	{Method: "GET", Pattern: "/2015-03-31/functions/", Action: "ListFunctions",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.listFunctions(ctx, c)
+		}},
+	{Method: "DELETE", Pattern: "/2015-03-31/functions/{FunctionName}", Action: "DeleteFunction",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.deleteFunction(ctx, c)
+		}},
 	{Method: "GET", Pattern: "/2015-03-31/functions/{FunctionName}", Action: "GetFunction",
 		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
 			return d.getFunction(ctx, c)

spinifex/domains/lambda/function/client.go

@@ -26,6 +26,10 @@ type Client interface {
 	CodeUpdated(ctx context.Context, accountID, region string, rec Record, revision uint64, codeRevisionID, sha256Hex string, size int64) (Record, uint64, error)
 	// UpdateConfiguration commits cfg for the function at revision and returns the updated record and revision.
 	UpdateConfiguration(ctx context.Context, accountID, region string, rec Record, revision uint64, cfg Configuration) (Record, uint64, error)
+	// List returns up to maxItems of the Region's functions after marker and the next marker, empty at the end.
+	List(ctx context.Context, accountID, region, marker string, maxItems int) ([]ListedFunction, string, error)
+	// Delete unbinds the function read at revision, retaining its cleanup obligation.
+	Delete(ctx context.Context, accountID, region string, rec Record, revision uint64) error
 }
 
 // NATSClient is the Client over NATS request/reply to the daemon.
@@ -98,3 +102,19 @@ func (c *NATSClient) UpdateConfiguration(ctx context.Context, accountID, region
 	}
 	return resp.Record, resp.Revision, nil
 }
+
+func (c *NATSClient) List(ctx context.Context, accountID, region, marker string, maxItems int) ([]ListedFunction, string, error) {
+	resp, err := natsmsg.NATSRequest[ListResponse](ctx, c.conn, SubjectList,
+		&ListRequest{Region: region, Marker: marker, MaxItems: maxItems}, metaRequestTimeout, accountID)
+	if err != nil {
+		return nil, "", err
+	}
+	return resp.Functions, resp.NextMarker, nil
+}
+
+func (c *NATSClient) Delete(ctx context.Context, accountID, region string, rec Record, revision uint64) error {
+	_, err := natsmsg.NATSRequest[DeleteResponse](ctx, c.conn, SubjectDelete,
+		&DeleteRequest{Region: region, FunctionName: rec.FunctionName, FunctionID: rec.FunctionID, Revision: revision},
+		metaRequestTimeout, accountID)
+	return err
+}

spinifex/domains/lambda/function/delete.go

@@ -0,0 +1,87 @@
+package function
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// Delete records the function's cleanup obligation, then unbinds its name by
+// compare-and-swap at the revision the caller read. Code copies and upload
+// records are retained; nothing here removes them.
+func (s *MetaService) Delete(ctx context.Context, req *DeleteRequest, accountID string) (*DeleteResponse, error) {
+	if !regionRE.MatchString(req.Region) {
+		return nil, badRegion()
+	}
+	if !ValidName(req.FunctionName) || !idRE.MatchString(req.FunctionID) || accountID == "" {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, "delete request is incomplete")
+	}
+	kv, err := s.bucket(ctx, accountID)
+	if err != nil {
+		return nil, err
+	}
+	key := KVFunctionKey(req.Region, req.FunctionName)
+	entry, err := kv.Get(ctx, key)
+	if errors.Is(err, jetstream.ErrKeyNotFound) {
+		return nil, awserrors.Errorf(awserrors.ErrorResourceNotFoundException, "Function not found: %s", req.FunctionName)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("lambda: get function record: %w", err)
+	}
+	rec, err := decode(entry.Value())
+	if err != nil {
+		return nil, err
+	}
+	if entry.Revision() != req.Revision || rec.FunctionID != req.FunctionID {
+		return nil, conflict("Function was modified by another request: %s", req.FunctionName)
+	}
+	if err := s.recordDeletion(ctx, kv, accountID, rec); err != nil {
+		return nil, err
+	}
+	if err := kv.Delete(ctx, key, jetstream.LastRevision(req.Revision)); err != nil {
+		if errors.Is(err, jetstream.ErrKeyExists) {
+			return nil, conflict("Function was modified by another request: %s", req.FunctionName)
+		}
+		return nil, fmt.Errorf("lambda: delete function record: %w", err)
+	}
+	return &DeleteResponse{}, nil
+}
+
+// recordDeletion writes the obligation for rec. One already present, from an
+// earlier attempt whose outcome was uncertain, is accepted only for the same
+// function; anything else refuses the delete.
+func (s *MetaService) recordDeletion(ctx context.Context, kv jetstream.KeyValue, accountID string, rec Record) error {
+	want := Deletion{
+		FormatVersion: FormatVersion, AccountID: accountID, Region: rec.Region,
+		FunctionName: rec.FunctionName, FunctionID: rec.FunctionID, RecordedAt: s.now().UTC(),
+	}
+	data, err := json.Marshal(want)
+	if err != nil {
+		return err
+	}
+	key := KVDeletedKey(rec.Region, rec.FunctionID)
+	_, err = kv.Create(ctx, key, data)
+	if err == nil {
+		return nil
+	}
+	if !errors.Is(err, jetstream.ErrKeyExists) {
+		return fmt.Errorf("lambda: record function deletion: %w", err)
+	}
+	entry, err := kv.Get(ctx, key)
+	if err != nil {
+		return fmt.Errorf("lambda: read function deletion: %w", err)
+	}
+	var have Deletion
+	if err := json.Unmarshal(entry.Value(), &have); err != nil {
+		return fmt.Errorf("lambda: function deletion record is unreadable: %w", err)
+	}
+	if have.FormatVersion != FormatVersion || have.AccountID != accountID || have.Region != rec.Region ||
+		have.FunctionName != rec.FunctionName || have.FunctionID != rec.FunctionID {
+		return fmt.Errorf("lambda: function deletion record %s names another function", key)
+	}
+	return nil
+}

spinifex/domains/lambda/function/delete_test.go

@@ -0,0 +1,296 @@
+package function
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// serveAs creates name with stored code under functionID and activates it.
+func serveAs(t *testing.T, svc *MetaService, name, functionID string) *GetResponse {
+	t.Helper()
+	rev := mustCreate(t, svc, name, functionID)
+	_, err := svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: name, Revision: rev, SHA256: testDigest, Size: 9}, testAccount)
+	require.NoError(t, err)
+	svc.SetReadiness(func(context.Context, Record) error { return nil })
+	require.NoError(t, svc.ActivatePending(t.Context()))
+	got := mustGet(t, svc, name)
+	require.Equal(t, StateActive, got.Record.State)
+	return got
+}
+
+func deleteReq(got *GetResponse) *DeleteRequest {
+	return &DeleteRequest{Region: testRegion, FunctionName: got.Record.FunctionName, FunctionID: got.Record.FunctionID, Revision: got.Revision}
+}
+
+// deletion reads the retained obligation for functionID, or reports it absent.
+func deletion(t *testing.T, svc *MetaService, functionID string) (Deletion, bool) {
+	t.Helper()
+	kv, err := svc.bucket(t.Context(), testAccount)
+	require.NoError(t, err)
+	entry, err := kv.Get(t.Context(), KVDeletedKey(testRegion, functionID))
+	if errors.Is(err, jetstream.ErrKeyNotFound) {
+		return Deletion{}, false
+	}
+	require.NoError(t, err)
+	var d Deletion
+	require.NoError(t, json.Unmarshal(entry.Value(), &d))
+	return d, true
+}
+
+func requireRetained(t *testing.T, svc *MetaService, name, functionID string) {
+	t.Helper()
+	d, ok := deletion(t, svc, functionID)
+	require.True(t, ok, "deletion obligation for %s is missing", functionID)
+	assert.Equal(t, Deletion{FormatVersion: FormatVersion, AccountID: testAccount, Region: testRegion,
+		FunctionName: name, FunctionID: functionID, RecordedAt: d.RecordedAt}, d)
+	_, ok = upload(t, svc, functionID)
+	assert.True(t, ok, "upload record for %s was removed", functionID)
+}
+
+func TestDelete_UnbindsTheNameAndRetainsTheObligationAndUploads(t *testing.T) {
+	svc, c := newTestService(t)
+	got := serveAs(t, svc, "hello", "f1")
+	_, err := svc.Delete(t.Context(), deleteReq(got), testAccount)
+	require.NoError(t, err)
+
+	assert.False(t, mustGet(t, svc, "hello").Found)
+	requireRetained(t, svc, "hello", "f1")
+	d, _ := deletion(t, svc, "f1")
+	assert.Equal(t, c.now(), d.RecordedAt)
+	up, _ := upload(t, svc, "f1")
+	assert.Equal(t, UploadAttached, up.State, "the upload record is left exactly as it was")
+}
+
+func TestDelete_MissingFunctionIsNotFound(t *testing.T) {
+	svc, _ := newTestService(t)
+	_, err := svc.Delete(t.Context(), &DeleteRequest{Region: testRegion, FunctionName: "hello", FunctionID: "f1", Revision: 1}, testAccount)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+	_, ok := deletion(t, svc, "f1")
+	assert.False(t, ok)
+}
+
+func TestDelete_StaleRevisionOrAnotherFunctionIsRefusedWithoutAnObligation(t *testing.T) {
+	svc, _ := newTestService(t)
+	got := serveAs(t, svc, "hello", "f1")
+	for name, req := range map[string]*DeleteRequest{
+		"stale revision": {Region: testRegion, FunctionName: "hello", FunctionID: "f1", Revision: got.Revision - 1},
+		"other function": {Region: testRegion, FunctionName: "hello", FunctionID: "f0", Revision: got.Revision},
+	} {
+		_, err := svc.Delete(t.Context(), req, testAccount)
+		testutil.RequireAWSCode(t, err, awserrors.ErrorResourceConflictException)
+		assert.Equal(t, got.Revision, mustGet(t, svc, "hello").Revision, name)
+		_, ok := deletion(t, svc, req.FunctionID)
+		assert.False(t, ok, name)
+	}
+}
+
+func TestDelete_RefusesMalformedRequests(t *testing.T) {
+	svc, _ := newTestService(t)
+	for name, req := range map[string]*DeleteRequest{
+		"bad region": {Region: "Bad.Region", FunctionName: "hello", FunctionID: "f1", Revision: 1},
+		"bad name":   {Region: testRegion, FunctionName: "a.b", FunctionID: "f1", Revision: 1},
+		"bad id":     {Region: testRegion, FunctionName: "hello", FunctionID: "f.1", Revision: 1},
+	} {
+		_, err := svc.Delete(t.Context(), req, testAccount)
+		require.Error(t, err, name)
+		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+	}
+}
+
+// The delete passes its read, then pauses before writing while an update
+// commits; the conditional delete loses and the function stays bound. The
+// obligation it left proves nothing was deleted.
+func TestDelete_InterleavedUpdateLosesTheConditionalDelete(t *testing.T) {
+	svc, c := newTestService(t)
+	got := serveAs(t, svc, "hello", "f1")
+	paused, release := make(chan struct{}), make(chan struct{})
+	var calls atomic.Int32
+	svc.now = func() time.Time {
+		if calls.Add(1) == 1 {
+			close(paused)
+			<-release
+		}
+		return c.now()
+	}
+	deleteErr := make(chan error, 1)
+	go func() {
+		_, err := svc.Delete(t.Context(), deleteReq(got), testAccount)
+		deleteErr <- err
+	}()
+	<-paused
+	cfg := Configuration{Role: got.Record.Role, Handler: "handler.two", Timeout: 3, MemorySize: 128}
+	won, err := svc.UpdateConfiguration(t.Context(), configReq(got.Revision, cfg), testAccount)
+	require.NoError(t, err)
+	close(release)
+	testutil.RequireAWSCode(t, <-deleteErr, awserrors.ErrorResourceConflictException)
+
+	after := mustGet(t, svc, "hello")
+	require.True(t, after.Found)
+	assert.Equal(t, won.Revision, after.Revision)
+	assert.Equal(t, "f1", after.Record.FunctionID)
+	_, ok := deletion(t, svc, "f1")
+	assert.True(t, ok, "the obligation from the losing attempt is retained")
+}
+
+func TestDelete_ObligationNamingAnotherFunctionRefusesTheDelete(t *testing.T) {
+	svc, _ := newTestService(t)
+	got := serveAs(t, svc, "hello", "f1")
+	kv, err := svc.bucket(t.Context(), testAccount)
+	require.NoError(t, err)
+	data, err := json.Marshal(Deletion{FormatVersion: FormatVersion, AccountID: testAccount, Region: testRegion, FunctionName: "other", FunctionID: "f1"})
+	require.NoError(t, err)
+	_, err = kv.Create(t.Context(), KVDeletedKey(testRegion, "f1"), data)
+	require.NoError(t, err)
+
+	_, err = svc.Delete(t.Context(), deleteReq(got), testAccount)
+	require.Error(t, err)
+	_, coded := awserrors.ResolveErrorCode(err)
+	assert.False(t, coded, "an inconsistent obligation is an internal fault, retried by the caller")
+	assert.True(t, mustGet(t, svc, "hello").Found)
+}
+
+// A delete of a create still storing its code wins; the late CodeStored and
+// RollbackCreate of that create change nothing, and its upload stays owed.
+func TestDelete_PendingCreateRacingDeletionChangesNothingLater(t *testing.T) {
+	svc, _ := newTestService(t)
+	rev := mustCreate(t, svc, "hello", "f1")
+	_, err := svc.Delete(t.Context(), &DeleteRequest{Region: testRegion, FunctionName: "hello", FunctionID: "f1", Revision: rev}, testAccount)
+	require.NoError(t, err)
+	recreated := serveAs(t, svc, "hello", "f2")
+
+	_, err = svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: rev, SHA256: testDigest, Size: 9}, testAccount)
+	require.Error(t, err)
+	_, err = svc.RollbackCreate(t.Context(), &RollbackCreateRequest{Region: testRegion, FunctionName: "hello", Revision: rev}, testAccount)
+	require.Error(t, err)
+
+	got := mustGet(t, svc, "hello")
+	assert.Equal(t, recreated.Revision, got.Revision)
+	assert.Equal(t, "f2", got.Record.FunctionID)
+	requireOwed(t, svc, "f1")
+	requireRetained(t, svc, "hello", "f1")
+}
+
+func TestDelete_SameNameRecreationKeepsEachObligationApart(t *testing.T) {
+	svc, _ := newTestService(t)
+	first := serveAs(t, svc, "hello", "f1")
+	_, err := svc.Delete(t.Context(), deleteReq(first), testAccount)
+	require.NoError(t, err)
+	second := serveAs(t, svc, "hello", "f2")
+	_, ok := deletion(t, svc, "f2")
+	assert.False(t, ok, "a recreated function has no obligation of its own")
+
+	_, err = svc.Delete(t.Context(), deleteReq(second), testAccount)
+	require.NoError(t, err)
+	requireRetained(t, svc, "hello", "f1")
+	requireRetained(t, svc, "hello", "f2")
+}
+
+// faultKV fails one write to key: before it commits, or after, losing the
+// acknowledgement.
+type faultKV struct {
+	jetstream.KeyValue
+
+	key   string
+	after bool
+	fired atomic.Bool
+}
+
+var errInjected = errors.New("nats: timeout")
+
+func (f *faultKV) trip(key string) bool { return key == f.key && f.fired.CompareAndSwap(false, true) }
+
+func (f *faultKV) Create(ctx context.Context, key string, value []byte, opts ...jetstream.KVCreateOpt) (uint64, error) {
+	if !f.trip(key) {
+		return f.KeyValue.Create(ctx, key, value, opts...)
+	}
+	if f.after {
+		if _, err := f.KeyValue.Create(ctx, key, value, opts...); err != nil {
+			return 0, err
+		}
+	}
+	return 0, errInjected
+}
+
+func (f *faultKV) Delete(ctx context.Context, key string, opts ...jetstream.KVDeleteOpt) error {
+	if !f.trip(key) {
+		return f.KeyValue.Delete(ctx, key, opts...)
+	}
+	if f.after {
+		if err := f.KeyValue.Delete(ctx, key, opts...); err != nil {
+			return err
+		}
+	}
+	return errInjected
+}
+
+// Each write of a delete is interrupted before it commits, or commits with its
+// acknowledgement lost. A fresh owner then finds the binding and obligation the
+// stream actually holds, a retry converges, and a recreated name stays apart.
+func TestDelete_InterruptionAndLostAcknowledgementAtEachWrite(t *testing.T) {
+	cases := []struct {
+		name       string
+		key        string
+		after      bool
+		bound      bool
+		obligation bool
+	}{
+		{"obligation write interrupted", KVDeletedKey(testRegion, "f1"), false, true, false},
+		{"obligation acknowledgement lost", KVDeletedKey(testRegion, "f1"), true, true, true},
+		{"name delete interrupted", KVFunctionKey(testRegion, "hello"), false, true, true},
+		{"name delete acknowledgement lost", KVFunctionKey(testRegion, "hello"), true, false, true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			_, nc, _ := testutil.StartTestJetStream(t)
+			js := testutil.NewJetStream(t, nc)
+			svc := NewMetaService(js)
+			got := serveAs(t, svc, "hello", "f1")
+			kv, err := svc.bucket(t.Context(), testAccount)
+			require.NoError(t, err)
+			svc.mu.Lock()
+			svc.buckets[testAccount] = &faultKV{KeyValue: kv, key: tc.key, after: tc.after}
+			svc.mu.Unlock()
+
+			_, err = svc.Delete(t.Context(), deleteReq(got), testAccount)
+			require.ErrorIs(t, err, errInjected)
+			_, coded := awserrors.ResolveErrorCode(err)
+			assert.False(t, coded, "an uncertain write is a retryable fault")
+
+			reopened := NewMetaService(js)
+			now := mustGet(t, reopened, "hello")
+			assert.Equal(t, tc.bound, now.Found, "binding")
+			if tc.bound {
+				assert.Equal(t, got.Revision, now.Revision)
+			}
+			_, ok := deletion(t, reopened, "f1")
+			assert.Equal(t, tc.obligation, ok, "obligation")
+			_, ok = upload(t, reopened, "f1")
+			assert.True(t, ok, "upload record retained")
+
+			_, err = reopened.Delete(t.Context(), deleteReq(got), testAccount)
+			if tc.bound {
+				require.NoError(t, err, "the retry completes the delete")
+			} else {
+				testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+			}
+			assert.False(t, mustGet(t, reopened, "hello").Found)
+			requireRetained(t, reopened, "hello", "f1")
+
+			recreated := serveAs(t, reopened, "hello", "f2")
+			assert.Equal(t, "f2", recreated.Record.FunctionID)
+			requireRetained(t, reopened, "hello", "f1")
+			_, ok = deletion(t, reopened, "f2")
+			assert.False(t, ok)
+		})
+	}
+}

spinifex/domains/lambda/function/list.go

@@ -0,0 +1,141 @@
+package function
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"math"
+	"strings"
+
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// List bounds: a page returns at most MaxListPage functions and examines at
+// most maxListScan stream sequences, whatever each one holds.
+const (
+	MaxListPage = 50
+	maxListScan = 200
+)
+
+// listCursor is the decoded marker: the last stream sequence a page examined,
+// bound to the account and Region it was issued for.
+type listCursor struct {
+	Version   int    `json:"v"`
+	AccountID string `json:"a"`
+	Region    string `json:"r"`
+	After     uint64 `json:"s"`
+}
+
+func encodeCursor(c listCursor) (string, error) {
+	data, err := json.Marshal(c)
+	if err != nil {
+		return "", err
+	}
+	return base64.RawURLEncoding.EncodeToString(data), nil
+}
+
+func decodeCursor(marker, accountID, region string) (uint64, error) {
+	bad := awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, "Invalid marker: %s", marker)
+	data, err := base64.RawURLEncoding.DecodeString(marker)
+	if err != nil {
+		return 0, bad
+	}
+	var c listCursor
+	if err := json.Unmarshal(data, &c); err != nil {
+		return 0, bad
+	}
+	if c.Version != 1 || c.AccountID != accountID || c.Region != region || c.After == 0 || c.After == math.MaxUint64 {
+		return 0, bad
+	}
+	return c.After, nil
+}
+
+// List returns the Region's functions in stream order after the marker. Each
+// step is one leader get of an exact sequence, bounded by a fast stream-state
+// read, so a page costs at most its scan bound whatever the bucket holds.
+// NextMarker is set unless the page reached the stream end it read.
+func (s *MetaService) List(ctx context.Context, req *ListRequest, accountID string) (*ListResponse, error) {
+	if !regionRE.MatchString(req.Region) {
+		return nil, badRegion()
+	}
+	if accountID == "" || req.MaxItems < 1 || req.MaxItems > MaxListPage {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, "list request is incomplete")
+	}
+	var after uint64
+	if req.Marker != "" {
+		var err error
+		if after, err = decodeCursor(req.Marker, accountID, req.Region); err != nil {
+			return nil, err
+		}
+	}
+	kv, err := s.bucket(ctx, accountID)
+	if err != nil {
+		return nil, err
+	}
+	stream := "KV_" + kv.Bucket()
+	// An unfiltered stream info without deleted details reads counters only.
+	info, err := s.js.Stream(ctx, stream)
+	if err != nil {
+		return nil, fmt.Errorf("lambda: list function records: %w", err)
+	}
+	state := info.CachedInfo().State
+	if state.FirstSeq > 0 {
+		after = max(after, state.FirstSeq-1)
+	}
+	keyPrefix := "$KV." + kv.Bucket() + "." + kvFunctionKeyPrefix + req.Region + "."
+	out := &ListResponse{Functions: []ListedFunction{}}
+	for gets := 0; gets < maxListScan && after < state.LastSeq; gets++ {
+		seq := after + 1
+		m, err := kvstore.LeaderMsgGet(ctx, s.js, stream, kvstore.MsgGetRequest{Seq: seq})
+		if errors.Is(err, jetstream.ErrMsgNotFound) {
+			after = seq
+			continue
+		}
+		if err != nil {
+			return nil, fmt.Errorf("lambda: list function records: %w", err)
+		}
+		if m.Sequence != seq {
+			return nil, fmt.Errorf("lambda: list function records: %w", jetstream.ErrInvalidJetStreamResponse)
+		}
+		if !strings.HasPrefix(m.Subject, keyPrefix) || removed(m.Header) {
+			after = seq
+			continue
+		}
+		if len(out.Functions) == req.MaxItems {
+			// A full page reached a function it cannot return; the marker
+			// stays before it.
+			break
+		}
+		rec, err := decode(m.Data)
+		if err != nil {
+			return nil, err
+		}
+		out.Functions = append(out.Functions, ListedFunction{Record: rec, Revision: seq})
+		after = seq
+	}
+	if after >= state.LastSeq {
+		return out, nil
+	}
+	if out.NextMarker, err = encodeCursor(listCursor{Version: 1, AccountID: accountID, Region: req.Region, After: after}); err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// removed reports a key's delete or purge marker, as the KV Get of nats.go
+// reads one.
+func removed(h nats.Header) bool {
+	if op := h.Get("KV-Operation"); op != "" {
+		return op == "DEL" || op == "PURGE"
+	}
+	switch h.Get(jetstream.MarkerReasonHeader) {
+	case "MaxAge", "Purge", "Remove":
+		return true
+	}
+	return false
+}

spinifex/domains/lambda/function/list_test.go

@@ -0,0 +1,325 @@
+package function
+
+import (
+	"fmt"
+	"math"
+	"testing"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func listPage(t *testing.T, svc *MetaService, region, marker string, maxItems int) *ListResponse {
+	t.Helper()
+	resp, err := svc.List(t.Context(), &ListRequest{Region: region, Marker: marker, MaxItems: maxItems}, testAccount)
+	require.NoError(t, err)
+	return resp
+}
+
+// walk follows NextMarker to the end and returns the names in order and the
+// number of pages.
+func walk(t *testing.T, svc *MetaService, region string, maxItems int) ([]string, int) {
+	t.Helper()
+	var names []string
+	marker, pages := "", 0
+	for {
+		resp := listPage(t, svc, region, marker, maxItems)
+		pages++
+		require.LessOrEqual(t, len(resp.Functions), maxItems)
+		for _, f := range resp.Functions {
+			names = append(names, f.Record.FunctionName)
+		}
+		if resp.NextMarker == "" {
+			return names, pages
+		}
+		marker = resp.NextMarker
+		require.Less(t, pages, 1000, "the walk does not end")
+	}
+}
+
+func TestList_EmptyAccountHasNoFunctionsAndNoMarker(t *testing.T) {
+	svc, _ := newTestService(t)
+	resp := listPage(t, svc, testRegion, "", MaxListPage)
+	assert.Empty(t, resp.Functions)
+	assert.NotNil(t, resp.Functions)
+	assert.Empty(t, resp.NextMarker)
+}
+
+func TestList_PagesReturnEveryFunctionOnceWithItsRevision(t *testing.T) {
+	svc, _ := newTestService(t)
+	want := make([]string, 0, 7)
+	for i := range 7 {
+		name := fmt.Sprintf("fn-%d", i)
+		mustCreate(t, svc, name, fmt.Sprintf("f%d", i))
+		want = append(want, name)
+	}
+	names, pages := walk(t, svc, testRegion, 2)
+	assert.Equal(t, want, names, "stream order, each once")
+	assert.Equal(t, 4, pages)
+
+	page := listPage(t, svc, testRegion, "", 1)
+	require.Len(t, page.Functions, 1)
+	got := mustGet(t, svc, "fn-0")
+	assert.Equal(t, got.Revision, page.Functions[0].Revision)
+	assert.Equal(t, got.Record, page.Functions[0].Record)
+}
+
+// A page that fills on the last function looks ahead, finds the walk over,
+// and returns no marker; one that fills before it keeps the next one.
+func TestList_FullPageAtTheEndHasNoMarker(t *testing.T) {
+	svc, _ := newTestService(t)
+	for i := range 4 {
+		mustCreate(t, svc, fmt.Sprintf("fn-%d", i), fmt.Sprintf("f%d", i))
+	}
+	_, pages := walk(t, svc, testRegion, 2)
+	assert.Equal(t, 2, pages)
+	assert.Empty(t, listPage(t, svc, testRegion, "", 4).NextMarker)
+
+	first := listPage(t, svc, testRegion, "", 3)
+	require.NotEmpty(t, first.NextMarker)
+	rest := listPage(t, svc, testRegion, first.NextMarker, 3)
+	require.Len(t, rest.Functions, 1, "the record the full page looked at is not skipped")
+	assert.Equal(t, "fn-3", rest.Functions[0].Record.FunctionName)
+	assert.Empty(t, rest.NextMarker)
+}
+
+func TestList_SkipsDeletedFunctionsUploadsObligationsAndOtherRegions(t *testing.T) {
+	svc, _ := newTestService(t)
+	gone := serveAs(t, svc, "gone", "f0")
+	mustCreate(t, svc, "kept", "f1")
+	_, err := svc.Delete(t.Context(), deleteReq(gone), testAccount)
+	require.NoError(t, err)
+	other := createReq("elsewhere", "f2")
+	other.Region = "us-east-1"
+	_, err = svc.Create(t.Context(), other, testAccount)
+	require.NoError(t, err)
+
+	names, _ := walk(t, svc, testRegion, MaxListPage)
+	assert.Equal(t, []string{"kept"}, names)
+	names, _ = walk(t, svc, "us-east-1", MaxListPage)
+	assert.Equal(t, []string{"elsewhere"}, names)
+}
+
+// cursorAfter is the last sequence a marker says its page examined.
+func cursorAfter(t *testing.T, marker string) uint64 {
+	t.Helper()
+	after, err := decodeCursor(marker, testAccount, testRegion)
+	require.NoError(t, err)
+	return after
+}
+
+// walkFrom follows NextMarker from marker to the end, checking that no page
+// examines more than maxListScan sequences, and returns the names it listed.
+func walkFrom(t *testing.T, svc *MetaService, marker string, maxItems int) ([]string, int) {
+	t.Helper()
+	var names []string
+	var after uint64
+	if marker != "" {
+		after = cursorAfter(t, marker)
+	}
+	for pages := 1; ; pages++ {
+		resp := listPage(t, svc, testRegion, marker, maxItems)
+		require.LessOrEqual(t, len(resp.Functions), maxItems)
+		for _, f := range resp.Functions {
+			names = append(names, f.Record.FunctionName)
+		}
+		if resp.NextMarker == "" {
+			return names, pages
+		}
+		next := cursorAfter(t, resp.NextMarker)
+		require.LessOrEqual(t, next-after, uint64(maxListScan), "a page examined more than its scan bound")
+		require.True(t, next > after || len(resp.Functions) > 0, "a page made no progress")
+		marker, after = resp.NextMarker, next
+		require.Less(t, pages, 1000, "the walk does not end")
+	}
+}
+
+// tombstones appends n deleted functions to the stream.
+func tombstones(t *testing.T, svc *MetaService, n int) {
+	t.Helper()
+	kv, err := svc.bucket(t.Context(), testAccount)
+	require.NoError(t, err)
+	for i := range n {
+		name := fmt.Sprintf("tomb-%d", i)
+		mustCreate(t, svc, name, fmt.Sprintf("t%d", i))
+		require.NoError(t, kv.Delete(t.Context(), KVFunctionKey(testRegion, name)))
+	}
+}
+
+// More delete markers than one request may examine give empty pages that
+// still continue, and the walk reaches the live function after them.
+func TestList_ScanBoundGivesShortPagesThatContinue(t *testing.T) {
+	svc, _ := newTestService(t)
+	tombstones(t, svc, maxListScan+5)
+	mustCreate(t, svc, "live", "f1")
+
+	first := listPage(t, svc, testRegion, "", MaxListPage)
+	assert.Empty(t, first.Functions)
+	require.NotEmpty(t, first.NextMarker)
+	assert.Equal(t, uint64(maxListScan), cursorAfter(t, first.NextMarker))
+	names, _ := walkFrom(t, svc, first.NextMarker, MaxListPage)
+	assert.Equal(t, []string{"live"}, names)
+}
+
+// Messages that are not this Region's function records count toward the scan
+// bound like any other sequence: uploads, obligations, other keys and other
+// Regions' functions.
+func TestList_ExcludedRecordsCountTowardTheScanBound(t *testing.T) {
+	svc, _ := newTestService(t)
+	mustCreate(t, svc, "first", "f1")
+	kv, err := svc.bucket(t.Context(), testAccount)
+	require.NoError(t, err)
+	for i := range maxListScan + 30 {
+		var key string
+		switch i % 4 {
+		case 0:
+			key = KVFunctionKey("us-east-1", fmt.Sprintf("x-%d", i))
+		case 1:
+			key = fmt.Sprintf("upload.%s.f9.%d", testRegion, i)
+		case 2:
+			key = KVDeletedKey(testRegion, fmt.Sprintf("d%d", i))
+		default:
+			key = fmt.Sprintf("other.%d", i)
+		}
+		_, err := kv.Put(t.Context(), key, []byte("{}"))
+		require.NoError(t, err)
+	}
+	mustCreate(t, svc, "last", "f2")
+
+	names, pages := walkFrom(t, svc, "", MaxListPage)
+	assert.Equal(t, []string{"first", "last"}, names)
+	assert.GreaterOrEqual(t, pages, 2)
+}
+
+// A key rewritten many times leaves gaps behind its latest revision; missing
+// sequences count toward the bound and the walk crosses them.
+func TestList_SparseSequencesCountTowardTheScanBound(t *testing.T) {
+	svc, _ := newTestService(t)
+	mustCreate(t, svc, "first", "f1")
+	kv, err := svc.bucket(t.Context(), testAccount)
+	require.NoError(t, err)
+	for range 2*maxListScan + 7 {
+		_, err := kv.Put(t.Context(), "scratch", []byte("{}"))
+		require.NoError(t, err)
+	}
+	mustCreate(t, svc, "middle", "f2")
+	for range maxListScan {
+		_, err := kv.Put(t.Context(), "scratch", []byte("{}"))
+		require.NoError(t, err)
+	}
+	mustCreate(t, svc, "last", "f3")
+
+	page := listPage(t, svc, testRegion, "", MaxListPage)
+	require.Len(t, page.Functions, 1)
+	assert.Equal(t, uint64(maxListScan), cursorAfter(t, page.NextMarker))
+	for _, size := range []int{1, MaxListPage} {
+		names, _ := walkFrom(t, svc, "", size)
+		assert.Equal(t, []string{"first", "middle", "last"}, names, "page size %d", size)
+	}
+}
+
+// A large live set is returned once each, in stream order, in pages that
+// never exceed the requested size.
+func TestList_LargeLiveSetPagesEveryFunctionOnce(t *testing.T) {
+	svc, _ := newTestService(t)
+	want := make([]string, 0, 3*MaxListPage+7)
+	for i := range cap(want) {
+		name := fmt.Sprintf("fn-%03d", i)
+		mustCreate(t, svc, name, fmt.Sprintf("f%d", i))
+		want = append(want, name)
+	}
+	for _, size := range []int{1, 7, MaxListPage} {
+		names, _ := walkFrom(t, svc, "", size)
+		assert.Equal(t, want, names, "page size %d", size)
+	}
+}
+
+// A full page looking ahead through delete markers to the end of the stream
+// returns no marker; when the scan bound runs out first, it continues and a
+// later page finds the function beyond them.
+func TestList_LookAheadThroughDeleteMarkers(t *testing.T) {
+	t.Run("reaches the end", func(t *testing.T) {
+		svc, _ := newTestService(t)
+		mustCreate(t, svc, "live", "f1")
+		tombstones(t, svc, 3)
+		page := listPage(t, svc, testRegion, "", 1)
+		require.Len(t, page.Functions, 1)
+		assert.Empty(t, page.NextMarker)
+	})
+	t.Run("scan bound exhausted", func(t *testing.T) {
+		svc, _ := newTestService(t)
+		mustCreate(t, svc, "live", "f1")
+		tombstones(t, svc, maxListScan)
+		mustCreate(t, svc, "beyond", "f2")
+		page := listPage(t, svc, testRegion, "", 1)
+		require.Len(t, page.Functions, 1)
+		assert.Equal(t, "live", page.Functions[0].Record.FunctionName)
+		require.NotEmpty(t, page.NextMarker, "the bound ran out before the walk was known to end")
+		names, _ := walkFrom(t, svc, page.NextMarker, 1)
+		assert.Equal(t, []string{"beyond"}, names)
+	})
+}
+
+// An update moves a function to a later sequence, so this walk, begun before
+// the update, meets it again after the function it had not yet reached.
+func TestList_UpdateDuringWalkMovesFunctionLater(t *testing.T) {
+	svc, _ := newTestService(t)
+	first := serveAs(t, svc, "a", "f1")
+	serveAs(t, svc, "b", "f2")
+	page := listPage(t, svc, testRegion, "", 1)
+	require.Equal(t, "a", page.Functions[0].Record.FunctionName)
+	cfg := Configuration{Role: first.Record.Role, Handler: "handler.two", Timeout: 3, MemorySize: 128}
+	_, err := svc.UpdateConfiguration(t.Context(), &ConfigurationUpdatedRequest{Region: testRegion, FunctionName: "a",
+		FunctionID: "f1", Revision: first.Revision, Configuration: cfg}, testAccount)
+	require.NoError(t, err)
+
+	var rest []string
+	for marker := page.NextMarker; marker != ""; {
+		p := listPage(t, svc, testRegion, marker, 1)
+		for _, f := range p.Functions {
+			rest = append(rest, f.Record.FunctionName)
+		}
+		marker = p.NextMarker
+	}
+	assert.Equal(t, []string{"b", "a"}, rest)
+}
+
+func TestList_RefusesForeignMalformedAndExhaustedMarkers(t *testing.T) {
+	svc, _ := newTestService(t)
+	for i := range 3 {
+		mustCreate(t, svc, fmt.Sprintf("fn-%d", i), fmt.Sprintf("f%d", i))
+	}
+	marker := listPage(t, svc, testRegion, "", 1).NextMarker
+	require.NotEmpty(t, marker)
+
+	foreign, err := svc.List(t.Context(), &ListRequest{Region: testRegion, Marker: marker, MaxItems: 1}, "000000000002")
+	assert.Nil(t, foreign)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+	_, err = svc.List(t.Context(), &ListRequest{Region: "us-east-1", Marker: marker, MaxItems: 1}, testAccount)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+
+	exhausted, err := encodeCursor(listCursor{Version: 1, AccountID: testAccount, Region: testRegion, After: math.MaxUint64})
+	require.NoError(t, err)
+	zero, err := encodeCursor(listCursor{Version: 1, AccountID: testAccount, Region: testRegion})
+	require.NoError(t, err)
+	future, err := encodeCursor(listCursor{Version: 2, AccountID: testAccount, Region: testRegion, After: 1})
+	require.NoError(t, err)
+	for _, bad := range []string{"not base64!", "e30", exhausted, zero, future} {
+		_, err := svc.List(t.Context(), &ListRequest{Region: testRegion, Marker: bad, MaxItems: 1}, testAccount)
+		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+	}
+}
+
+func TestList_RefusesMalformedRequests(t *testing.T) {
+	svc, _ := newTestService(t)
+	for _, req := range []*ListRequest{
+		{Region: "Bad.Region", MaxItems: 1},
+		{Region: testRegion, MaxItems: 0},
+		{Region: testRegion, MaxItems: MaxListPage + 1},
+	} {
+		_, err := svc.List(t.Context(), req, testAccount)
+		testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+	}
+}

spinifex/domains/lambda/function/record.go

@@ -63,6 +63,7 @@ const (
 	KVBucketAccountHistory = 1
 	kvFunctionKeyPrefix    = "fn."
 	kvUploadKeyPrefix      = "upload."
+	kvDeletedKeyPrefix     = "deleted."
 )
 
 // KVUploadKey returns the key of the record that tracks one code upload. The
@@ -111,6 +112,10 @@ const (
 	SubjectCodeUpdated     = "lambda.function.code-updated"
 	// SubjectConfigurationUpdated commits new settings for an Active function.
 	SubjectConfigurationUpdated = "lambda.function.configuration-updated"
+	// SubjectList pages the function records of one Region.
+	SubjectList = "lambda.function.list"
+	// SubjectDelete unbinds a function name and retains its cleanup obligation.
+	SubjectDelete = "lambda.function.delete"
 )
 
 // CreateRequest carries the record the gateway built; the daemon owns the
@@ -218,3 +223,53 @@ type ConfigurationUpdatedResponse struct {
 	Record   Record `json:"record"`
 	Revision uint64 `json:"revision"`
 }
+
+// KVDeletedKey returns the key of the obligation a deleted function leaves
+// behind. A recreated function receives a new random ID, and an obligation
+// whose identity does not match the deleted function is refused.
+func KVDeletedKey(region, functionID string) string {
+	return kvDeletedKeyPrefix + region + "." + functionID
+}
+
+// Deletion is the retained cleanup obligation of a deleted function: its code
+// copies and upload records are kept until a later owner proves them unused.
+// It is written before the name is unbound, so it alone proves no deletion.
+type Deletion struct {
+	FormatVersion int       `json:"formatVersion"`
+	AccountID     string    `json:"accountId"`
+	Region        string    `json:"region"`
+	FunctionName  string    `json:"functionName"`
+	FunctionID    string    `json:"functionId"`
+	RecordedAt    time.Time `json:"recordedAt"`
+}
+
+// ListRequest asks for one page of a Region's functions after Marker, which
+// is empty for the first page or a NextMarker the owner returned.
+type ListRequest struct {
+	Region   string `json:"region"`
+	Marker   string `json:"marker,omitempty"`
+	MaxItems int    `json:"maxItems"`
+}
+
+// ListedFunction is one function record and its revision.
+type ListedFunction struct {
+	Record   Record `json:"record"`
+	Revision uint64 `json:"revision"`
+}
+
+// ListResponse is one page; NextMarker is empty once the walk has ended.
+type ListResponse struct {
+	Functions  []ListedFunction `json:"functions"`
+	NextMarker string           `json:"nextMarker,omitempty"`
+}
+
+// DeleteRequest unbinds the function the caller read at Revision.
+type DeleteRequest struct {
+	Region       string `json:"region"`
+	FunctionName string `json:"functionName"`
+	FunctionID   string `json:"functionId"`
+	Revision     uint64 `json:"revision"`
+}
+
+// DeleteResponse is empty: success means the name was unbound.
+type DeleteResponse struct{}

spinifex/ingress/aws/pipeline/lambda_invoke_wire_test.go

@@ -59,6 +59,14 @@ func (activeMeta) UpdateConfiguration(context.Context, string, string, function.
 	return function.Record{}, 0, errors.New("unused")
 }
 
+func (activeMeta) List(context.Context, string, string, string, int) ([]function.ListedFunction, string, error) {
+	return nil, "", errors.New("unused")
+}
+
+func (activeMeta) Delete(context.Context, string, string, function.Record, uint64) error {
+	return errors.New("unused")
+}
+
 // countingExec records how many invocations reached it.
 type countingExec struct{ calls atomic.Int32 }
 

spinifex/runtime/daemon/daemon.go

@@ -1363,6 +1363,8 @@ func (d *Daemon) subscribeAll() error {
 			natsSub{lambdafunction.SubjectCodeUpdateBegin, handleNATSRequest(d.node, d.lambdaMetaService.BeginCodeUpdate), "spinifex-workers"},
 			natsSub{lambdafunction.SubjectCodeUpdated, handleNATSRequest(d.node, d.lambdaMetaService.CodeUpdated), "spinifex-workers"},
 			natsSub{lambdafunction.SubjectConfigurationUpdated, handleNATSRequest(d.node, d.lambdaMetaService.UpdateConfiguration), "spinifex-workers"},
+			natsSub{lambdafunction.SubjectList, handleNATSRequest(d.node, d.lambdaMetaService.List), "spinifex-workers"},
+			natsSub{lambdafunction.SubjectDelete, handleNATSRequest(d.node, d.lambdaMetaService.Delete), "spinifex-workers"},
 		)
 	}
 

spinifex/runtime/daemon/daemon_lambda_listdelete_test.go

@@ -0,0 +1,72 @@
+package daemon
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/internal/testkit"
+	lambdafunction "github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// A daemon without the list and delete subjects refuses both as retryable
+// transport errors and changes nothing; once served, they decode and commit.
+func TestLambdaListDelete_PreviousDaemonRefusesWithoutMutation(t *testing.T) {
+	_, nc, js := testutil.StartTestJetStream(t)
+	svc := lambdafunction.NewMetaService(js)
+	client := lambdafunction.NewNATSClient(nc)
+	ctx := t.Context()
+	const region = "ap-southeast-2"
+
+	serveLambdaSubjects(t, nc, svc, true)
+	_, err := client.Create(ctx, lambdaTestAccount, region, lambdafunction.Record{
+		FunctionID: "f1", FunctionName: "hello", Role: "arn:aws:iam::000000000001:role/r", Runtime: "provided.al2023",
+		Handler: "bootstrap", Timeout: 3, MemorySize: 128, Code: lambdafunction.Code{RevisionID: "rev-1"},
+	})
+	require.NoError(t, err)
+	rec, rev, err := client.Get(ctx, lambdaTestAccount, region, "hello")
+	require.NoError(t, err)
+
+	_, _, err = client.List(ctx, lambdaTestAccount, region, "", 1)
+	require.ErrorIs(t, err, nats.ErrNoResponders)
+	err = client.Delete(ctx, lambdaTestAccount, region, rec, rev)
+	require.ErrorIs(t, err, nats.ErrNoResponders)
+	_, coded := awserrors.ResolveErrorCode(err)
+	assert.False(t, coded, "the gateway maps an uncoded failure to a retryable ServiceException")
+	same, sameRev, err := client.Get(ctx, lambdaTestAccount, region, "hello")
+	require.NoError(t, err)
+	assert.Equal(t, rev, sameRev)
+	assert.Equal(t, rec, same)
+	kv, err := js.KeyValue(ctx, lambdafunction.KVAccountBucket(lambdaTestAccount))
+	require.NoError(t, err)
+	_, err = kv.Get(ctx, lambdafunction.KVDeletedKey(region, "f1"))
+	assert.ErrorIs(t, err, jetstream.ErrKeyNotFound, "no obligation was recorded")
+
+	for subject, h := range map[string]nats.MsgHandler{
+		lambdafunction.SubjectList:   asMsgHandler(handleNATSRequest("node-1", svc.List)),
+		lambdafunction.SubjectDelete: asMsgHandler(handleNATSRequest("node-1", svc.Delete)),
+	} {
+		sub, err := nc.QueueSubscribe(subject, "spinifex-workers", h)
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = sub.Unsubscribe() })
+	}
+	require.NoError(t, nc.Flush())
+	listed, next, err := client.List(ctx, lambdaTestAccount, region, "", 1)
+	require.NoError(t, err)
+	require.Len(t, listed, 1)
+	assert.Equal(t, rec, listed[0].Record)
+	assert.Equal(t, rev, listed[0].Revision)
+	assert.Empty(t, next)
+
+	_, _, err = client.List(ctx, lambdaTestAccount, region, "bogus", 1)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorInvalidParameterValueException)
+
+	require.NoError(t, client.Delete(ctx, lambdaTestAccount, region, rec, rev))
+	_, _, err = client.Get(ctx, lambdaTestAccount, region, "hello")
+	require.ErrorIs(t, err, lambdafunction.ErrNotFound)
+	err = client.Delete(ctx, lambdaTestAccount, region, rec, rev)
+	testutil.RequireAWSCode(t, err, awserrors.ErrorResourceNotFoundException)
+}

spinifex/runtime/roles/awsgw/lambda_testlink_off_test.go

@@ -52,6 +52,14 @@ func (*spyFunctions) UpdateConfiguration(context.Context, string, string, functi
 	return function.Record{}, 0, errors.New("unused")
 }
 
+func (*spyFunctions) List(context.Context, string, string, string, int) ([]function.ListedFunction, string, error) {
+	return nil, "", errors.New("unused")
+}
+
+func (*spyFunctions) Delete(context.Context, string, string, function.Record, uint64) error {
+	return errors.New("unused")
+}
+
 // spyExec counts invocations that would start an attempt.
 type spyExec struct{ calls atomic.Int32 }
 

spinifex/runtime/roles/awsgw/lambda_testlink_reply_test.go

@@ -60,6 +60,14 @@ func (storedFunction) UpdateConfiguration(context.Context, string, string, funct
 	return function.Record{}, 0, errors.New("unused")
 }
 
+func (storedFunction) List(context.Context, string, string, string, int) ([]function.ListedFunction, string, error) {
+	return nil, "", errors.New("unused")
+}
+
+func (storedFunction) Delete(context.Context, string, string, function.Record, uint64) error {
+	return errors.New("unused")
+}
+
 // invokeThrough sends one Invoke through the Lambda handler to the real
 // test-link executor; fakeControl stands in for the daemon and answers with rep.
 func invokeThrough(t *testing.T, rep testlink.Reply) (*httptest.ResponseRecorder, error) {

spinifex/runtime/roles/awsgw/lambda_testlink_seam_test.go

@@ -85,6 +85,14 @@ func (seamFunction) UpdateConfiguration(context.Context, string, string, functio
 	return function.Record{}, 0, errors.New("unused")
 }
 
+func (seamFunction) List(context.Context, string, string, string, int) ([]function.ListedFunction, string, error) {
+	return nil, "", errors.New("unused")
+}
+
+func (seamFunction) Delete(context.Context, string, string, function.Record, uint64) error {
+	return errors.New("unused")
+}
+
 type urlPresigner struct {
 	url   string
 	calls *atomic.Int32