test(iam): account for UpdateUser as named by bare name

UpdateUser addresses its user by bare name, so the cross-tenant ARN table
cannot target another account; list it with the other byName IAM actions.
Add a focused test that a rename stays within the caller's account.

contracts/ecs/v1/messages.go

@@ -126,8 +126,8 @@ type Assign struct {
 	ENIPrivateIP  string `json:"eniPrivateIp,omitempty"`
 	ENISubnetID   string `json:"eniSubnetId,omitempty"`
 	// TaskRoleARN is the task's IAM role (from the taskdef); when set, the agent
-	// serves its credentials at 169.254.170.2 under CredID, defaulting CredID to
-	// the taskID when the scheduler leaves it empty.
+	// serves its credentials at 169.254.170.2 under CredID, a scheduler-minted
+	// secret. An assign without a CredID gets no credentials.
 	CredID      string `json:"credId,omitempty"`
 	TaskRoleARN string `json:"taskRoleArn,omitempty"`
 	// ExecutionRoleARN is the task's execution role (from the taskdef); when set,

docs/containers/eks/README.md

@@ -143,6 +143,25 @@ No user-facing operation replaces a control plane, and automatic replacement doe
 Recreate the cluster to adopt it.
 User-workload injection through a cluster created with an older image is not measured.
 
+### Add-on versions
+
+Add-on versions use the AWS form `v<x.y.z>-eksbuild.<n>`, for example `v1.40.1-eksbuild.1`.
+`DescribeAddonVersions` returns versions in this form, newest first, and every other add-on response reports the version the same way.
+Spinifex ships one EKS build of each upstream version, so `<n>` is `1` today.
+
+`DescribeAddonVersions` honours `addonName`, `kubernetesVersion`, `types`, `publishers`, `owners`, `maxResults` and `nextToken`.
+Each version carries `compatibilities` with the `clusterVersion`, the `platformVersions`, and a `defaultVersion` flag that is true for exactly one version per add-on and cluster version.
+A `kubernetesVersion` that Spinifex does not run returns no add-ons.
+
+`CreateAddon` and `UpdateAddon` accept the AWS form of a version in the catalogue.
+They also accept the bare upstream form, such as `1.40.1`, and Spinifex logs each use at Info.
+This is a Spinifex leniency that AWS does not offer, and it exists so that older scripts keep working.
+Any other value is refused with a message that names `addonVersion` and lists the supported versions, for example `v1.40.1-eksbuild.2`, `1.40` or `v1.40.1`.
+
+In Terraform, pass the AWS form to `aws_eks_addon.addon_version`, either as a literal or from `data.aws_eks_addon_version`.
+The AWS provider validates the format before it calls the API, so it rejects the bare form.
+Leaving `addon_version` unset still installs the catalogue default.
+
 ## Prerequisites
 
 > [!IMPORTANT]
@@ -359,7 +378,7 @@ Control plane readiness and kubectl authentication above lists which clusters re
 
 **`create-cluster` rejected with an authentication-mode error.** `CONFIG_MAP` is the one mode Spinifex rejects, because it has no `aws-auth` ConfigMap path. Set `authentication_mode` to `API` or `API_AND_CONFIG_MAP`, or leave it unset, and grant access with access entries.
 
-**Addon install fails.** Only addons bundled into the registered `eks-node` image install successfully. If you pin an addon version in Terraform, leave `addon_version` unset so Spinifex selects the catalog default.
+**Addon install fails.** Only addons bundled into the registered `eks-node` image install successfully. If you pin an addon version in Terraform, use the AWS form from `DescribeAddonVersions` (see Add-on versions above), or leave `addon_version` unset so Spinifex selects the catalog default.
 
 ## Reference Architectures
 

docs/coverage/README.md

@@ -17,7 +17,7 @@ tags:
 
 ## Overview
 
-The platform serves **405 operations** across the AWS APIs below. Every page names the operations its service implements from the pinned model, generated from the dispatch tables on each build rather than written by hand.
+The platform serves **406 operations** across the AWS APIs below. Every page names the operations its service implements from the pinned model, generated from the dispatch tables on each build rather than written by hand.
 
 | Service | Operations |
 |---|---:|
@@ -27,9 +27,9 @@ The platform serves **405 operations** across the AWS APIs below. Every page nam
 | [ECS](/coverage/ecs) | 31 |
 | [EKS](/coverage/eks) | 27 |
 | [ELBv2](/coverage/elbv2) | 36 |
-| [IAM](/coverage/iam) | 81 |
+| [IAM](/coverage/iam) | 82 |
 | [RDS](/coverage/rds) | 27 |
 | [Route 53](/coverage/route53) | 13 |
 | [S3](/coverage/s3) | 26 |
 | [STS](/coverage/sts) | 5 |
-| **Total** | **405** |
+| **Total** | **406** |

docs/coverage/coverage.json

@@ -2938,7 +2938,8 @@
           "UpdateAccessKey",
           "UpdateAssumeRolePolicy",
           "UpdateRole",
-          "UpdateRoleDescription"
+          "UpdateRoleDescription",
+          "UpdateUser"
         ],
         "implemented": [
           "AddRoleToInstanceProfile",
@@ -3021,7 +3022,8 @@
           "UpdateAccessKey",
           "UpdateAssumeRolePolicy",
           "UpdateRole",
-          "UpdateRoleDescription"
+          "UpdateRoleDescription",
+          "UpdateUser"
         ],
         "stubbed": [],
         "unsupported": [],
@@ -3121,7 +3123,6 @@
           "UpdateServerCertificate",
           "UpdateServiceSpecificCredential",
           "UpdateSigningCertificate",
-          "UpdateUser",
           "UploadSSHPublicKey",
           "UploadServerCertificate",
           "UploadSigningCertificate"

docs/coverage/iam/README.md

@@ -18,7 +18,7 @@ tags:
 
 ## Overview
 
-Spinifex implements **81 operations** in the IAM `2010-05-08` API model.
+Spinifex implements **82 operations** in the IAM `2010-05-08` API model.
 
 ### Scope
 
@@ -109,3 +109,4 @@ All IAM operations are account-scoped. The root user of an account bypasses poli
 | `UpdateAssumeRolePolicy` |
 | `UpdateRole` |
 | `UpdateRoleDescription` |
+| `UpdateUser` |

docs/coverage/internal-report.md

@@ -987,7 +987,7 @@ spot-price: On owned hardware there is no spot-to-on-demand price differential,
 
 ## iam
 
-`2010-05-08` — 81 of 180 modelled operations implemented (45.0%), 0 stubbed, 19 not applicable, 0 registered outside the pinned model.
+`2010-05-08` — 82 of 180 modelled operations implemented (45.6%), 0 stubbed, 19 not applicable, 0 registered outside the pinned model.
 
 ### Not implemented
 
@@ -1069,7 +1069,6 @@ spot-price: On owned hardware there is no spot-to-on-demand price differential,
 - `UpdateOpenIDConnectProviderThumbprint`
 - `UpdateSAMLProvider`
 - `UpdateServerCertificate`
-- `UpdateUser`
 - `UploadServerCertificate`
 
 ### Not applicable — do not build

docs/package-boundary/eks-addon-lifecycle.md

@@ -20,7 +20,7 @@ There is no generation, so the record mixes desired fields with the latest obser
 | Durable record | `clusters/{cluster}/addons/{addon}` holds `Record` (`domains/eks/addon/record.go`): `addonName`, `addonVersion`, `status`, `serviceAccountRoleArn`, `configurationValues`, `health`, `arn`, `tags`, `createdAt`, `modifiedAt` (the optional ones `omitempty`). `clusters/{cluster}/addons/{addon}/manifest` holds `Manifest` (`domains/eks/addon/record.go`): `addonName`, `addonVersion`, `serviceAccountRoleArn`, `configurationValues`. Listing skips the manifest sub-key by segment count. | `TestAddonPersistedLayout_RecordAndManifestKeysAndFieldNames`, `TestAddonRecord_DecodesHandWrittenJSON`, `domains/eks/addon/store_test.go` `TestRecord_RoundTrip`, `TestList_SkipsManifestSubKeys` |
 | Writers | Record: the owner in `domains/eks/addon/owner.go`, reached from the `CreateAddon`, `UpdateAddon` and `DeleteAddon` handlers, the cluster reconciler's report relay and the GPU node group launch: `Owner.Create` (unconditional put), `Owner.Update` and its failure marking (revision CAS), `Owner.Delete` (get then delete), `Owner.ApplyReport` (revision CAS), `Owner.EnsureGPUDevicePlugin` through `Owner.Create`; and the cluster prefix purge (`DeleteClusterChildren`). Manifest: `StagingInstaller.Install` (unconditional put on create and update), `StagingInstaller.Uninstall` (`domains/eks/addon/delivery.go`), `Owner.Delete` and the prefix purge. | `TestAddonPersistedLayout_RecordAndManifestKeysAndFieldNames`, `TestUpdateAddon_ReportsSuccessfulWhileRecordIsUpdating`, `TestDeleteAddon_ReturnsDeletingAfterErasingRecordAndManifest`, `TestDeleteClusterChildren_SweepsAddonRecordsAndManifestsScopedToCluster`, `domains/eks/addon/store_test.go` `TestCasUpdate`, `TestDeleteRecord_RemovesRecordAndManifest` |
 | Readers | `ListAddons`, `DescribeAddon`, `CreateAddon` (existence check) and the reconciler read the record. `ListStagedAddonManifests` reads every `/manifest` key for the cluster (`addon.ListManifests`); it is served on `eks.ListStagedAddonManifests` (`daemon/daemon.go`) and reached only through the gateway's internal-addons route. | `handlers/eks/addons_test.go` `TestAddon_DescribeListDelete`, `TestListStagedAddonManifests` |
-| Catalogue | Static in-binary catalogue (`domains/eks/addon/catalog.go`): `aws-load-balancer-controller`, `argocd`, `aws-ebs-csi-driver`, and hidden-but-creatable `nvidia-device-plugin` and `spinifex-noop`; one version each, the first being the default. Create and update reject unknown names and versions with `InvalidParameterValue`. | `domains/eks/addon/catalog_test.go`, `handlers/eks/addons_test.go` `TestDescribeAddonVersions_ReturnsCatalog`, `TestCreateAddon_UnknownAddonRejected`, `TestCreateAddon_UnknownVersionRejected`, `TestUpdateAddon_UnknownVersionRejected` |
+| Catalogue | Static in-binary catalogue (`domains/eks/addon/catalog.go`): `aws-load-balancer-controller`, `argocd`, `aws-ebs-csi-driver`, and hidden-but-creatable `nvidia-device-plugin` and `spinifex-noop`; one version each, the first being the default. Versions are stored and used internally in the bare upstream form, and API responses show the AWS form `v<x.y.z>-eksbuild.<n>` (`Spec.AWSVersion`). Create and update accept the AWS form or the exact bare form and reject unknown names and other version strings with `InvalidParameterValue`, which the gateway returns as `InvalidParameterException`. | `domains/eks/addon/catalog_test.go`, `handlers/eks/addons_test.go` `TestDescribeAddonVersions_ReturnsCatalog`, `TestCreateAddon_UnknownAddonRejected`, `TestCreateAddon_UnknownVersionRejected`, `TestUpdateAddon_UnknownVersionRejected` |
 | Desired state | Version, `serviceAccountRoleArn`, `configurationValues` and create-time tags. `resolveConflicts`, `podIdentityAssociations` and `clientRequestToken` are accepted and dropped; `preserve` on delete is ignored. `configurationValues` reaches the guest but the agent discards it (`mulga-eks-addon-sync.sh` `sync_once`), so it has no effect. | `TestAddonPersistedLayout_RecordAndManifestKeysAndFieldNames`, `TestDeleteAddon_ReturnsDeletingAfterErasingRecordAndManifest` |
 | Observed state | `status` and `health` are overwritten by guest reports on the same record as the desired fields. `health` is the latest failure message, projected as one `AddonIssue` with only `message`. Statuses held: `CREATING`, `ACTIVE`, `UPDATING`, `DEGRADED`, `CREATE_FAILED`; `DELETING` is defined but never stored. | `TestAddonRecord_DecodesHandWrittenJSON`, `handlers/eks/addon_status_reconciler_test.go` `TestApplyAddonStatusReport` |
 | Generation | None. Reports carry `version` and `ts`, and neither is read: any report for the add-on name applies to whatever is currently desired. | `TestAddonStatusReport_MismatchedVersionIsAccepted` |

docs/terraform-workbooks/eks-gitops-argocd/README.md

@@ -79,7 +79,8 @@ Terraform manages the cluster, the addons, the ACM cert, and the HTTPS Ingress;
 - The `argocd` and `aws-ebs-csi-driver` bundles must be baked into the `eks-node` AMI — `describe-addon` returns `no baked bundle` otherwise.
 - The EBS-CSI default StorageClass uses `provisioner: ebs.csi.aws.com`; a PVC against it provisions a **Viperblock-backed EBS volume**.
 - If the PVC stays `Pending`, the app still serves (the counter falls back to in-memory) — see [Troubleshooting](#troubleshooting).
-- Leave addon versions unset (the AWS provider and the catalog disagree on the version-string format).
+- Addon versions are unset, so Spinifex installs each catalog default.
+  To pin one, use the AWS form that `DescribeAddonVersions` returns, such as `v3.0.23-eksbuild.1`.
 - The Argo CD `Application` is a `kubernetes_manifest`, so the `argoproj.io` CRDs must exist before `apply` — apply the parent and let the addon reach `ACTIVE` first.
 
 **Prerequisites:**

docs/terraform-workbooks/eks-gitops-argocd/main.tf

@@ -520,8 +520,8 @@ resource "aws_eks_node_group" "workers" {
 #
 # Spinifex wires the controller's AWS credentials and ELB-eligible subnets at the
 # node level, so no IRSA role or subnet tagging is needed here. addon_version is
-# omitted: the AWS provider demands a v-prefixed version that the catalog rejects,
-# so let Spinifex default to its catalog version.
+# omitted so Spinifex installs its catalog default. To pin one, use the AWS form
+# from DescribeAddonVersions, for example v2.11.0-eksbuild.1.
 # ---------------------------------------------------------------------------
 
 resource "aws_eks_addon" "lbc" {
@@ -542,7 +542,7 @@ resource "aws_eks_addon" "lbc" {
 # Installed through the EKS API; Spinifex stages the bundle host-side and the
 # worker renders it into the K3s auto-deploy dir. The workloads module registers
 # the git repo and creates the Argo CD Application that syncs the demo app.
-# addon_version omitted (see the LBC note).
+# addon_version omitted: the catalog default is installed (see the LBC note).
 # ---------------------------------------------------------------------------
 
 resource "aws_eks_addon" "argocd" {
@@ -562,7 +562,7 @@ resource "aws_eks_addon" "argocd" {
 #
 # Ships a default gp3 StorageClass (provisioner ebs.csi.aws.com). A PVC in the
 # demo app's git manifests dynamically provisions a Viperblock-backed EBS volume
-# so the app's state survives pod restarts. addon_version omitted (see LBC note).
+# so the app's state survives pod restarts. addon_version omitted: catalog default.
 # ---------------------------------------------------------------------------
 
 resource "aws_eks_addon" "ebs_csi" {

docs/terraform-workbooks/eks-https-ingress/README.md

@@ -74,7 +74,8 @@ The LBC is installed as a managed **addon** (`aws_eks_addon`). Spinifex wires th
 - **`spinifex.io/managed-ingress = "false"`** disables the K3s built-in traefik/servicelb so the LBC owns ingress, matching AWS parity. Omit the tag (or set it `true`) and the built-in ingress stays on.
 - The ALB targets the workers' **NodePort** (`target-type: instance`). The one SG rule admits the VPC CIDR on that port so the in-VPC ALB can reach the workers; the worker SG is otherwise auto-managed.
 - Spinifex ACM supports **ImportCertificate only** — the cert is generated by the `tls` provider and imported, not requested.
-- **Leave `addon_version` unset** (see the [quickstart/addons notes](/docs/eks-quickstart)) — the AWS provider and the Spinifex catalog disagree on the version string format, so let the server choose.
+- **`addon_version` is unset**, so Spinifex installs its catalog default.
+  To pin a version, use the AWS form that `DescribeAddonVersions` returns, such as `v2.11.0-eksbuild.1`, or read it from `data.aws_eks_addon_version`.
 
 **Prerequisites:**
 

docs/terraform-workbooks/eks-https-ingress/main.tf

@@ -446,8 +446,8 @@ resource "aws_eks_node_group" "workers" {
 #
 # Spinifex wires the controller's AWS credentials and ELB-eligible subnets at the
 # node level, so no IRSA role or subnet tagging is needed here. addon_version is
-# omitted: the AWS provider demands a v-prefixed version that the catalog rejects,
-# so let Spinifex default to its catalog version.
+# omitted so Spinifex installs its catalog default. To pin one, use the AWS form
+# from DescribeAddonVersions, for example v2.11.0-eksbuild.1.
 # ---------------------------------------------------------------------------
 
 resource "aws_eks_addon" "lbc" {

spinifex/agents/ecs/agent/credendpoint.go

@@ -47,6 +47,13 @@ type credResponse struct {
 
 // credEntry caches one task's assumed-role credentials alongside the role it was
 // assumed from.
+// credRole is the task role a credID serves and the task it belongs to; the
+// task ID names the STS session, since the credID is a secret.
+type credRole struct {
+	roleARN string
+	taskID  string
+}
+
 type credEntry struct {
 	roleARN string
 	creds   stsauth.Credentials
@@ -71,7 +78,7 @@ type credEndpoint struct {
 	assume func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error)
 
 	mu         sync.Mutex
-	roles      map[string]string    // credID -> taskRoleARN
+	roles      map[string]credRole  // credID -> task role
 	cache      map[string]credEntry // credID -> cached creds
 	httpClient *http.Client
 
@@ -98,7 +105,7 @@ func newCredEndpoint(provider credentials.CredentialsProvider, region, gatewayUR
 		port:       port,
 		proxyPort:  defaultCredProxyPort,
 		run:        run,
-		roles:      map[string]string{},
+		roles:      map[string]credRole{},
 		cache:      map[string]credEntry{},
 	}
 	c.assume = c.assumeOverGateway
@@ -107,12 +114,12 @@ func newCredEndpoint(provider credentials.CredentialsProvider, region, gatewayUR
 
 // Register associates a credID with the task role it serves; Deregister removes
 // it and drops any cached credentials. Both are no-ops for an empty credID.
-func (c *credEndpoint) Register(credID, roleARN string) {
+func (c *credEndpoint) Register(credID, taskID, roleARN string) {
 	if credID == "" || roleARN == "" {
 		return
 	}
 	c.mu.Lock()
-	c.roles[credID] = roleARN
+	c.roles[credID] = credRole{roleARN: roleARN, taskID: taskID}
 	c.mu.Unlock()
 }
 
@@ -142,25 +149,25 @@ func (c *credEndpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "not found", http.StatusNotFound)
 		return
 	}
-	creds, roleARN, err := c.fetch(r.Context(), credID)
+	creds, role, err := c.fetch(r.Context(), credID)
 	if err != nil {
 		if errors.Is(err, errUnknownCredID) {
 			http.Error(w, "not found", http.StatusNotFound)
 			return
 		}
-		slog.Error("ecs-agent: assume task role failed", "credId", credID, "err", err)
+		slog.Error("ecs-agent: assume task role failed", "task", role.taskID, "role", role.roleARN, "err", err)
 		http.Error(w, "credentials unavailable", http.StatusBadGateway)
 		return
 	}
 	w.Header().Set("Content-Type", "application/json")
 	if err := json.NewEncoder(w).Encode(credResponse{
-		RoleArn:         roleARN,
+		RoleArn:         role.roleARN,
 		AccessKeyID:     creds.AccessKeyID,
 		SecretAccessKey: creds.SecretAccessKey,
 		Token:           creds.SessionToken,
 		Expiration:      creds.Expiration.UTC().Format(time.RFC3339),
 	}); err != nil {
-		slog.Warn("ecs-agent: encode credential response", "credId", credID, "err", err)
+		slog.Warn("ecs-agent: encode credential response", "task", role.taskID, "err", err)
 	}
 }
 
@@ -169,28 +176,28 @@ var errUnknownCredID = fmt.Errorf("unknown credId")
 
 // fetch returns valid credentials for credID, reusing the cache until within the
 // refresh margin and otherwise assuming the task role over the gateway.
-func (c *credEndpoint) fetch(ctx context.Context, credID string) (stsauth.Credentials, string, error) {
+func (c *credEndpoint) fetch(ctx context.Context, credID string) (stsauth.Credentials, credRole, error) {
 	c.mu.Lock()
-	roleARN, known := c.roles[credID]
+	role, known := c.roles[credID]
 	if !known {
 		c.mu.Unlock()
-		return stsauth.Credentials{}, "", errUnknownCredID
+		return stsauth.Credentials{}, credRole{}, errUnknownCredID
 	}
-	if e, ok := c.cache[credID]; ok && e.roleARN == roleARN && credValid(e.creds, credRefreshMargin) {
+	if e, ok := c.cache[credID]; ok && e.roleARN == role.roleARN && credValid(e.creds, credRefreshMargin) {
 		c.mu.Unlock()
-		return e.creds, roleARN, nil
+		return e.creds, role, nil
 	}
 	c.mu.Unlock()
 
-	creds, err := c.assume(ctx, roleARN, sessionName(credID))
+	creds, err := c.assume(ctx, role.roleARN, sessionName(role.taskID))
 	if err != nil {
-		return stsauth.Credentials{}, "", err
+		return stsauth.Credentials{}, role, err
 	}
 
 	c.mu.Lock()
-	c.cache[credID] = credEntry{roleARN: roleARN, creds: creds}
+	c.cache[credID] = credEntry{roleARN: role.roleARN, creds: creds}
 	c.mu.Unlock()
-	return creds, roleARN, nil
+	return creds, role, nil
 }
 
 // AssumeProvider returns a CredentialsProvider that mints credentials by assuming
@@ -390,10 +397,10 @@ func credValid(c stsauth.Credentials, margin time.Duration) bool {
 	return time.Now().Add(margin).Before(c.Expiration)
 }
 
-// sessionName derives an AssumeRole RoleSessionName from credID, satisfying the
+// sessionName derives an AssumeRole RoleSessionName from the task ID, satisfying the
 // STS [\w+=,.@-]{2,64} constraint.
-func sessionName(credID string) string {
-	name := "ecs-" + credID
+func sessionName(taskID string) string {
+	name := "ecs-" + taskID
 	if len(name) > 64 {
 		name = name[:64]
 	}

spinifex/agents/ecs/agent/credendpoint_test.go

@@ -29,7 +29,7 @@ func TestCredEndpoint_ServesRegisteredCredID(t *testing.T) {
 	c, _ := newTestCredEndpoint(stsauth.Credentials{
 		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "tok", Expiration: exp,
 	})
-	c.Register("cred-1", "arn:aws:iam::111122223333:role/task")
+	c.Register("cred-1", "task-1", "arn:aws:iam::111122223333:role/task")
 
 	rr := httptest.NewRecorder()
 	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
@@ -51,6 +51,26 @@ func TestCredEndpoint_ServesRegisteredCredID(t *testing.T) {
 	}
 }
 
+// The STS session name lands in the assumed-role ARN and in STS logs, so it is
+// built from the public task ID and never from the secret credID.
+func TestCredEndpoint_SessionNameIsTaskIDNotCredID(t *testing.T) {
+	c, _ := newTestCredEndpoint(stsauth.Credentials{})
+	var gotSession string
+	c.assume = func(ctx context.Context, roleARN, session string) (stsauth.Credentials, error) {
+		gotSession = session
+		return stsauth.Credentials{AccessKeyID: "AKIA", SecretAccessKey: "secret"}, nil
+	}
+	c.Register("secret-cred", "task-1", "arn:role")
+	rr := httptest.NewRecorder()
+	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/secret-cred", nil))
+	if rr.Code != http.StatusOK {
+		t.Fatalf("status = %d, want 200", rr.Code)
+	}
+	if gotSession != "ecs-task-1" {
+		t.Errorf("session name = %q, want ecs-task-1", gotSession)
+	}
+}
+
 func TestCredEndpoint_UnknownCredIDIs404(t *testing.T) {
 	c, calls := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
 	rr := httptest.NewRecorder()
@@ -65,7 +85,7 @@ func TestCredEndpoint_UnknownCredIDIs404(t *testing.T) {
 
 func TestCredEndpoint_WrongMethodIs405(t *testing.T) {
 	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
-	c.Register("cred-1", "arn:role")
+	c.Register("cred-1", "task-1", "arn:role")
 	rr := httptest.NewRecorder()
 	c.ServeHTTP(rr, httptest.NewRequest(http.MethodPost, "/v2/credentials/cred-1", nil))
 	if rr.Code != http.StatusMethodNotAllowed {
@@ -77,7 +97,7 @@ func TestCredEndpoint_CachesUntilExpiry(t *testing.T) {
 	c, calls := newTestCredEndpoint(stsauth.Credentials{
 		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Hour),
 	})
-	c.Register("cred-1", "arn:role")
+	c.Register("cred-1", "task-1", "arn:role")
 	for hit := range 3 {
 		rr := httptest.NewRecorder()
 		c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
@@ -95,7 +115,7 @@ func TestCredEndpoint_RefreshesNearExpiry(t *testing.T) {
 	c, calls := newTestCredEndpoint(stsauth.Credentials{
 		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Minute),
 	})
-	c.Register("cred-1", "arn:role")
+	c.Register("cred-1", "task-1", "arn:role")
 	for range 2 {
 		rr := httptest.NewRecorder()
 		c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
@@ -107,7 +127,7 @@ func TestCredEndpoint_RefreshesNearExpiry(t *testing.T) {
 
 func TestCredEndpoint_DeregisterDropsCredID(t *testing.T) {
 	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
-	c.Register("cred-1", "arn:role")
+	c.Register("cred-1", "task-1", "arn:role")
 	c.Deregister("cred-1")
 	rr := httptest.NewRecorder()
 	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
@@ -121,7 +141,7 @@ func TestCredEndpoint_AssumeFailureIs502(t *testing.T) {
 	c.assume = func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
 		return stsauth.Credentials{}, context.DeadlineExceeded
 	}
-	c.Register("cred-1", "arn:role")
+	c.Register("cred-1", "task-1", "arn:role")
 	rr := httptest.NewRecorder()
 	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
 	if rr.Code != http.StatusBadGateway {
@@ -133,7 +153,7 @@ func TestCredEndpoint_StartServesOverLoopback(t *testing.T) {
 	c, _ := newTestCredEndpoint(stsauth.Credentials{
 		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Hour),
 	})
-	c.Register("cred-1", "arn:role")
+	c.Register("cred-1", "task-1", "arn:role")
 	c.port = 0 // bind an ephemeral port instead of privileged 80
 	if err := c.Start(); err != nil {
 		t.Fatalf("start: %v", err)

spinifex/agents/ecs/agent/credinject_test.go

@@ -1,8 +1,12 @@
 package agent
 
 import (
+	"net/http"
+	"net/http/httptest"
 	"testing"
 
+	"github.com/mulgadc/spinifex/internal/stsauth"
+
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
 )
 
@@ -13,7 +17,7 @@ func TestTaskCredID(t *testing.T) {
 		want string
 	}{
 		{"no role", ecsv1.Assign{TaskID: "t1"}, ""},
-		{"role, no credID -> taskID", ecsv1.Assign{TaskID: "t1", TaskRoleARN: "arn:role"}, "t1"},
+		{"role, no credID -> none", ecsv1.Assign{TaskID: "t1", TaskRoleARN: "arn:role"}, ""},
 		{"role + credID", ecsv1.Assign{TaskID: "t1", CredID: "c9", TaskRoleARN: "arn:role"}, "c9"},
 	}
 	for _, tc := range cases {
@@ -23,6 +27,37 @@ func TestTaskCredID(t *testing.T) {
 	}
 }
 
+// The task ID is public (it is the task ARN suffix), so naming another task's ID
+// at the credential endpoint must not yield that task's role.
+func TestRegisterTaskCreds_TaskIDDoesNotServeCredentials(t *testing.T) {
+	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "AKIA", SecretAccessKey: "secret"})
+	a := Agent{cred: c}
+	get := func(id string) int {
+		rr := httptest.NewRecorder()
+		c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, credRelativeURI(id), nil))
+		return rr.Code
+	}
+
+	legacy := ecsv1.Assign{TaskID: "victim-task", TaskRoleARN: "arn:aws:iam::000000000010:role/high"}
+	if got := a.registerTaskCreds(&legacy); got != "" {
+		t.Errorf("assign without CredID registered credID %q", got)
+	}
+	if code := get("victim-task"); code != http.StatusNotFound {
+		t.Errorf("GET by task ID of CredID-less assign = %d, want 404", code)
+	}
+
+	minted := ecsv1.Assign{TaskID: "victim-2", CredID: "secret-cred", TaskRoleARN: "arn:aws:iam::000000000010:role/high"}
+	if got := a.registerTaskCreds(&minted); got != "secret-cred" {
+		t.Fatalf("registerTaskCreds = %q, want secret-cred", got)
+	}
+	if code := get("victim-2"); code != http.StatusNotFound {
+		t.Errorf("GET by task ID = %d, want 404", code)
+	}
+	if code := get("secret-cred"); code != http.StatusOK {
+		t.Errorf("GET by CredID = %d, want 200", code)
+	}
+}
+
 func TestWithCredEnv(t *testing.T) {
 	base := map[string]string{"FOO": "bar"}
 

spinifex/agents/ecs/agent/reconcile_test.go

@@ -63,7 +63,7 @@ func TestReconcile_AdoptsRunningLabeledContainer(t *testing.T) {
 	if !adopted["t-001"] {
 		t.Fatalf("t-001 not adopted; got %v", adopted)
 	}
-	if got := a.cred.roles["t-001"]; got != testTaskRole {
+	if got := a.cred.roles["t-001"].roleARN; got != testTaskRole {
 		t.Errorf("cred not re-registered: roles[t-001] = %q, want %q", got, testTaskRole)
 	}
 	if n := countStatus(cp.taskStates(), "t-001", ecsv1.TaskStatusRunning); n != 1 {

spinifex/agents/ecs/agent/taskrun.go

@@ -306,24 +306,24 @@ func (a *Agent) pullResolver(as *ecsv1.Assign) ctrruntime.Resolver {
 // credential endpoint and returns the credID (empty when the task has no role).
 func (a *Agent) registerTaskCreds(as *ecsv1.Assign) string {
 	credID := taskCredID(as)
+	if credID == "" && as.TaskRoleARN != "" {
+		slog.Warn("ecs-agent: task has a role but no credential ID, starting it without credentials", "task", as.TaskID, "role", as.TaskRoleARN)
+	}
 	if credID == "" || a.cred == nil {
 		return credID
 	}
-	a.cred.Register(credID, as.TaskRoleARN)
+	a.cred.Register(credID, as.TaskID, as.TaskRoleARN)
 	return credID
 }
 
 // taskCredID is the credential ID a task's containers fetch credentials under:
-// the scheduler-assigned CredID, falling back to the taskID. Empty when the task
-// carries no IAM role.
+// the scheduler-minted secret CredID. Empty when the task carries no IAM role or
+// no CredID; the public task ID is never used, since any API caller can learn it.
 func taskCredID(as *ecsv1.Assign) string {
 	if as.TaskRoleARN == "" {
 		return ""
 	}
-	if as.CredID != "" {
-		return as.CredID
-	}
-	return as.TaskID
+	return as.CredID
 }
 
 // withCredEnv returns env with AWS_CONTAINER_CREDENTIALS_RELATIVE_URI set for

spinifex/domains/acm/keyalgorithm_test.go

@@ -152,3 +152,30 @@ func TestImportCertificate_SerialZeroRejected(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorValidationException, code)
 	assert.Equal(t, "The serial number in the certificate is not supported by ACM.", msg)
 }
+
+func TestImportCertificate_ExpiredRejected(t *testing.T) {
+	t.Parallel()
+	svc := setupACMService(t)
+	key, err := rsa.GenerateKey(rand.Reader, 1024)
+	require.NoError(t, err)
+	tmpl := &x509.Certificate{
+		SerialNumber: big.NewInt(time.Now().UnixNano()),
+		Subject:      pkix.Name{CommonName: "expired.example.com"},
+		NotBefore:    time.Now().Add(-48 * time.Hour),
+		NotAfter:     time.Now().Add(-24 * time.Hour),
+	}
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
+	require.NoError(t, err)
+	_, err = svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{
+		Certificate: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}),
+		PrivateKey:  pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}),
+	}, testAccountID)
+	code, msg, ok := awserrors.ResolveErrorDetail(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorValidationException, code)
+	assert.Equal(t, "The certificate is expired. You can Import a certificate only during its validity period.", msg)
+
+	list, err := svc.ListCertificates(context.Background(), &acm.ListCertificatesInput{}, testAccountID)
+	require.NoError(t, err)
+	assert.Empty(t, list.CertificateSummaryList)
+}

spinifex/domains/acm/service_impl.go

@@ -223,10 +223,13 @@ func (s *ACMServiceImpl) ImportCertificate(ctx context.Context, input *acm.Impor
 	if leaf.SerialNumber.Sign() == 0 {
 		return nil, awserrors.Errorf(awserrors.ErrorValidationException, "The serial number in the certificate is not supported by ACM.")
 	}
-
 	certArn := aws.StringValue(input.CertificateArn)
-	var inUseBy []string
 	now := time.Now().UTC()
+	if certArn == "" && now.After(leaf.NotAfter) {
+		return nil, awserrors.Errorf(awserrors.ErrorValidationException, "The certificate is expired. You can Import a certificate only during its validity period.")
+	}
+
+	var inUseBy []string
 	createdAt := now
 	tags := tagsToMap(input.Tags)
 	reimport := certArn != ""

spinifex/domains/ec2/awsapi/instance/IamInstanceProfileAssociation.go

@@ -35,7 +35,7 @@ func resolveAndAuthorizeProfile(spec *ec2.IamInstanceProfileSpecification, iamSv
 	}
 	profile, err := iamSvc.ResolveInstanceProfile(accountID, nameOrARN)
 	if err != nil {
-		if err.Error() == awserrors.ErrorIAMNoSuchEntity {
+		if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
 			return nil, errors.New(awserrors.ErrorInvalidIamInstanceProfileNotFound)
 		}
 		return nil, err

spinifex/domains/ec2/awsapi/instance/IamInstanceProfileAssociation_test.go

@@ -45,7 +45,7 @@ func (f *fakeIAMService) ResolveInstanceProfile(accountID, nameOrARN string) (*i
 	if f.resolveFn != nil {
 		return f.resolveFn(accountID, nameOrARN)
 	}
-	return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+	return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", nameOrARN)
 }
 
 func (f *fakeIAMService) CanonicalResourceARN(accountID string, kind arn.IAMResourceType, name string) (string, error) {

spinifex/domains/ec2/guestmetadata/metadata.go

@@ -738,7 +738,7 @@ func (s *IMDSServiceImpl) profileFor(ctx context.Context, eni *eniFacts) (*resol
 	}
 	profile, err := s.iam.ResolveInstanceProfile(ctx, eni.iamAccountID(), inst.iamInstanceProfileArn)
 	if err != nil {
-		if err.Error() == awserrors.ErrorIAMNoSuchEntity {
+		if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
 			return nil, roleMissProfileDeleted, nil // profile deleted; treat as no role
 		}
 		slog.ErrorContext(ctx, "IMDS: resolve instance profile failed", "account_id", eni.iamAccountID(), "arn", inst.iamInstanceProfileArn, "err", err)

spinifex/domains/ec2/guestmetadata/metadata_test.go

@@ -1152,7 +1152,7 @@ func TestHTTP_SecurityCredentialsListBackendError500(t *testing.T) {
 // genuine backend error (TestHTTP_*BackendError500) is a 500.
 func TestHTTP_IAMInfoDeletedProfile404(t *testing.T) {
 	res := &fakeResolver{eni: testENI(), inst: &instanceFacts{iamInstanceProfileArn: "arn:aws:iam::111122223333:instance-profile/gone"}}
-	svc, _ := newTestService(res, &fakeIAM{profileErr: errors.New(awserrors.ErrorIAMNoSuchEntity)}, &fakeAssumer{})
+	svc, _ := newTestService(res, &fakeIAM{profileErr: awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile gone cannot be found.")}, &fakeAssumer{})
 	h := withTapENI(svc.httpHandler(), testENI())
 	token := issueToken(t, h)
 	rec := get(t, h, prefixMetaData+"iam/info", token)

spinifex/domains/ecs/records.go

@@ -272,11 +272,15 @@ type TaskRecord struct {
 	// Group / StartedBy mirror the AWS task fields. A service's tasks carry
 	// Group="service:{name}" so the reconciler counts them and the task-state
 	// hook resolves a RUNNING/STOPPED task back to its owning service.
-	Group                string `json:"group,omitempty"`
-	StartedBy            string `json:"startedBy,omitempty"`
-	TaskDefFamily        string `json:"taskDefFamily"`
-	TaskDefRevision      int    `json:"taskDefRevision"`
-	TaskDefARN           string `json:"taskDefArn"`
+	Group           string `json:"group,omitempty"`
+	StartedBy       string `json:"startedBy,omitempty"`
+	TaskDefFamily   string `json:"taskDefFamily"`
+	TaskDefRevision int    `json:"taskDefRevision"`
+	TaskDefARN      string `json:"taskDefArn"`
+	// CredID is the secret under which the task's containers fetch task-role
+	// credentials from the agent. Set only when the task has a role; it must
+	// never be projected into an API response or logged.
+	CredID               string `json:"credId,omitempty"`
 	ContainerInstanceID  string `json:"containerInstanceId,omitempty"`
 	ContainerInstanceARN string `json:"containerInstanceArn,omitempty"`
 	DesiredStatus        string `json:"desiredStatus"`

spinifex/domains/ecs/service_test.go

@@ -2,7 +2,9 @@ package ecs
 
 import (
 	"context"
+	"encoding/json"
 	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
+	"maps"
 	"strconv"
 	"testing"
 	"time"
@@ -302,6 +304,89 @@ func TestService_RunTask_AssignCarriesTaskRole(t *testing.T) {
 	assert.Equal(t, roleARN, poll.Assignments[0].TaskRoleARN)
 }
 
+// A role task's credential ID is a scheduler-minted secret: not the public task
+// ID, distinct per task, stable on re-delivery, persisted on the task record and
+// absent from every API response. A role-less task gets none.
+func TestService_TaskCredIDIsMintedSecret(t *testing.T) {
+	ctx := context.Background()
+	svc, _ := newTestService(t)
+	_, err := svc.CreateCluster(ctx, &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
+	require.NoError(t, err)
+	_, err = svc.RegisterTaskDefinition(ctx, &ecs.RegisterTaskDefinitionInput{
+		Family:      aws.String("victim"),
+		TaskRoleArn: aws.String("arn:aws:iam::123456789012:role/task-high"),
+		ContainerDefinitions: []*ecs.ContainerDefinition{{
+			Name: aws.String("app"), Image: aws.String("registry/app:1"),
+			Cpu: aws.Int64(128), Memory: aws.Int64(256), Essential: aws.Bool(true),
+		}},
+	}, testAccountID)
+	require.NoError(t, err)
+	registerTaskDef(t, svc, "attacker", 128, 256)
+	registerInstance(t, svc, "web", "i-1", 1024, 2048)
+
+	run, err := svc.RunTask(ctx, &ecs.RunTaskInput{Cluster: aws.String("web"), TaskDefinition: aws.String("victim")}, testAccountID)
+	require.NoError(t, err)
+	start, err := svc.StartTask(ctx, &ecs.StartTaskInput{
+		Cluster: aws.String("web"), TaskDefinition: aws.String("victim"), ContainerInstances: []*string{aws.String("i-1")},
+	}, testAccountID)
+	require.NoError(t, err)
+	_, err = svc.StartTask(ctx, &ecs.StartTaskInput{
+		Cluster: aws.String("web"), TaskDefinition: aws.String("attacker"), ContainerInstances: []*string{aws.String("i-1")},
+	}, testAccountID)
+	require.NoError(t, err)
+
+	poll := func() map[string]ecsv1.Assign {
+		p, perr := svc.PollAssignments(ctx, &PollAssignmentsInput{Cluster: "web", ContainerInstance: "i-1"}, testAccountID)
+		require.NoError(t, perr)
+		byTask := map[string]ecsv1.Assign{}
+		for _, as := range p.Assignments {
+			byTask[as.TaskID] = as
+		}
+		return byTask
+	}
+	first, second := poll(), poll()
+	require.Len(t, first, 3)
+
+	kv, err := svc.bucket(ctx, testAccountID)
+	require.NoError(t, err)
+	var apiJSON []byte
+	for _, out := range []any{run, start} {
+		b, merr := json.Marshal(out)
+		require.NoError(t, merr)
+		apiJSON = append(apiJSON, b...)
+	}
+	var taskARNs []*string
+	for as := range maps.Values(first) {
+		taskARNs = append(taskARNs, aws.String(as.TaskARN))
+	}
+	desc, err := svc.DescribeTasks(ctx, &ecs.DescribeTasksInput{Cluster: aws.String("web"), Tasks: taskARNs}, testAccountID)
+	require.NoError(t, err)
+	b, err := json.Marshal(desc)
+	require.NoError(t, err)
+	apiJSON = append(apiJSON, b...)
+
+	seen := map[string]bool{}
+	for taskID, as := range first {
+		if as.TaskRoleARN == "" {
+			assert.Empty(t, as.CredID, "role-less task got a credential ID")
+			continue
+		}
+		require.NotEmpty(t, as.CredID)
+		assert.NotEqual(t, taskID, as.CredID)
+		assert.False(t, seen[as.CredID], "credential ID reused across tasks")
+		seen[as.CredID] = true
+		assert.Equal(t, as.CredID, second[taskID].CredID, "re-delivered assign changed credential ID")
+
+		var rec TaskRecord
+		found, gerr := getJSON(ctx, kv, TaskKey("web", taskID), &rec)
+		require.NoError(t, gerr)
+		require.True(t, found)
+		assert.Equal(t, as.CredID, rec.CredID)
+		assert.NotContains(t, string(apiJSON), as.CredID, "credential ID leaked in an API response")
+	}
+	assert.Len(t, seen, 2)
+}
+
 // PollAssignments is at-least-once: re-poll without ack re-delivers the assign;
 // acking it (then STOPPED) drains the inbox so it is never re-delivered.
 func TestService_PollAssignments_AckAndReclaim(t *testing.T) {

spinifex/domains/ecs/tasks.go

@@ -237,6 +237,9 @@ func (s *Service) newTaskRecord(accountID, cluster, taskID string, td *taskdefin
 		GPU:                  gpu,
 		CreatedAt:            now,
 	}
+	if td.TaskRoleArn != "" {
+		rec.CredID = uuid.NewV4().String()
+	}
 	for _, c := range td.Containers {
 		rec.Containers = append(rec.Containers, ContainerState{Name: c.Name, Status: TaskStatusPending})
 	}
@@ -257,6 +260,7 @@ func (s *Service) publishAssign(ctx context.Context, kv jetstream.KeyValue, acco
 		TaskDefFamily:    td.Family,
 		TaskDefRevision:  td.Revision,
 		TaskRoleARN:      td.TaskRoleArn,
+		CredID:           rec.CredID,
 		ExecutionRoleARN: td.ExecutionRoleArn,
 		ENIID:            rec.ENIID,
 		ENIMacAddress:    rec.ENIMacAddress,

spinifex/domains/eks/addon/catalog.go

@@ -4,8 +4,14 @@ import (
 	"fmt"
 	"maps"
 	"slices"
+	"strconv"
+	"strings"
 )
 
+// firstBuild is the EKS build number of every bundled version; it moves per
+// version when Spinifex rebuilds a bundle.
+const firstBuild = 1
+
 // NvidiaDevicePlugin is staged by a GPU node group's launch (EnsureGPUDevicePlugin),
 // never user-requested directly.
 const NvidiaDevicePlugin = "nvidia-device-plugin"
@@ -18,6 +24,11 @@ type Spec struct {
 	// Versions lists supported versions newest-first; [0] is the default.
 	Versions       []string
 	DefaultVersion string
+	// Builds maps each upstream version to its EKS build number, the n in
+	// v<upstream>-eksbuild.<n>.
+	Builds map[string]int
+	// Type is the AWS add-on type reported by DescribeAddonVersions.
+	Type string
 	// RequiresIRSA indicates the add-on needs an IAM role for service-account binding.
 	RequiresIRSA bool
 	Description  string
@@ -28,26 +39,26 @@ type Spec struct {
 
 // catalog is the bundled add-on registry. Keep newest version first per slice.
 var catalog = buildCatalog(
-	newSpec("aws-load-balancer-controller", true,
+	typed("networking", newSpec("aws-load-balancer-controller", true,
 		"Provisions ELBv2 load balancers for Kubernetes Service/Ingress resources.",
-		"2.11.0"),
-	newSpec("argocd", false,
+		"2.11.0")),
+	typed("gitops", newSpec("argocd", false,
 		"Declarative GitOps continuous delivery for Kubernetes.",
-		"3.0.23"),
-	newSpec("aws-ebs-csi-driver", true,
+		"3.0.23")),
+	typed("storage", newSpec("aws-ebs-csi-driver", true,
 		"Container Storage Interface driver for Amazon EBS (Viperblock) volumes.",
-		"1.40.1"),
+		"1.40.1")),
 	// Staged automatically for GPU node groups, so hidden from the public
 	// catalogue like spinifex-noop but still creatable by name.
-	hidden(newSpec(NvidiaDevicePlugin, false,
+	hidden(typed("accelerator", newSpec(NvidiaDevicePlugin, false,
 		"NVIDIA device plugin exposing nvidia.com/gpu allocatable via CDI on GPU-tainted nodes.",
-		"0.17.4")),
+		"0.17.4"))),
 	// spinifex-noop is the delivery-transport fixture: a trivial bundle
 	// (Namespace + ConfigMap) the add-on e2e uses to prove stage, render,
 	// auto-deploy, ACTIVE and delete round-trip without a real workload.
-	hidden(newSpec("spinifex-noop", false,
+	hidden(typed("fixture", newSpec("spinifex-noop", false,
 		"No-op delivery-transport fixture (Namespace + ConfigMap).",
-		"0.1.0")),
+		"0.1.0"))),
 )
 
 // hidden marks a spec as internal: still creatable, but absent from the
@@ -57,6 +68,12 @@ func hidden(s Spec) Spec {
 	return s
 }
 
+// typed sets the AWS add-on type reported by DescribeAddonVersions.
+func typed(addonType string, s Spec) Spec {
+	s.Type = addonType
+	return s
+}
+
 // newSpec builds a spec with the first version as its default.
 func newSpec(name string, requiresIRSA bool, description string, versions ...string) Spec {
 	spec := Spec{
@@ -68,6 +85,10 @@ func newSpec(name string, requiresIRSA bool, description string, versions ...str
 	if len(versions) > 0 {
 		spec.DefaultVersion = versions[0]
 	}
+	spec.Builds = make(map[string]int, len(versions))
+	for _, v := range versions {
+		spec.Builds[v] = firstBuild
+	}
 	return spec
 }
 
@@ -111,6 +132,50 @@ func (s Spec) SupportsVersion(version string) bool {
 	return slices.Contains(s.Versions, version)
 }
 
+// awsForm renders a catalog version as v<upstream>-eksbuild.<build>.
+func (s Spec) awsForm(upstream string) string {
+	build, ok := s.Builds[upstream]
+	if !ok {
+		build = firstBuild
+	}
+	return "v" + upstream + "-eksbuild." + strconv.Itoa(build)
+}
+
+// AWSVersion returns the AWS form of a stored version. A version the catalogue
+// does not list, or one already in AWS form, is returned unchanged.
+func (s Spec) AWSVersion(stored string) string {
+	if !slices.Contains(s.Versions, stored) {
+		return stored
+	}
+	return s.awsForm(stored)
+}
+
+// AWSVersions returns every supported version in AWS form, newest first.
+func (s Spec) AWSVersions() []string {
+	out := make([]string, len(s.Versions))
+	for i, v := range s.Versions {
+		out[i] = s.awsForm(v)
+	}
+	return out
+}
+
+// ParseVersion maps a client-supplied version, in AWS form or bare upstream
+// form, to the upstream version stored on the record. Nothing else is accepted.
+func (s Spec) ParseVersion(input string) (string, bool) {
+	if slices.Contains(s.Versions, input) {
+		return input, true
+	}
+	if !strings.HasPrefix(input, "v") {
+		return "", false
+	}
+	for _, v := range s.Versions {
+		if input == s.awsForm(v) {
+			return v, true
+		}
+	}
+	return "", false
+}
+
 // Specs returns every catalogue entry, hidden ones included, sorted by name.
 func Specs() []Spec {
 	names := slices.Sorted(maps.Keys(catalog))

spinifex/domains/eks/addon/catalog_test.go

@@ -100,3 +100,78 @@ func TestValidateCatalog_ChecksTheBundledCatalog(t *testing.T) {
 	t.Cleanup(func() { catalog = previous })
 	require.EqualError(t, ValidateCatalog(), `add-on "broken" has no versions`)
 }
+
+// buildSpec is a two-version spec whose newest version is on EKS build 2, so
+// a test cannot pass by hardcoding build 1.
+func buildSpec() Spec {
+	return Spec{
+		Name:     "fixture",
+		Versions: []string{"2.0.0", "1.0.0"},
+		Builds:   map[string]int{"2.0.0": 2, "1.0.0": 1},
+	}
+}
+
+func TestSpec_AWSVersionUsesTheCatalogBuildNumber(t *testing.T) {
+	t.Parallel()
+	spec := buildSpec()
+	assert.Equal(t, "v2.0.0-eksbuild.2", spec.AWSVersion("2.0.0"))
+	assert.Equal(t, "v1.0.0-eksbuild.1", spec.AWSVersion("1.0.0"))
+}
+
+func TestSpec_AWSVersionIsIdempotent(t *testing.T) {
+	t.Parallel()
+	spec := buildSpec()
+	assert.Equal(t, "v2.0.0-eksbuild.2", spec.AWSVersion("v2.0.0-eksbuild.2"),
+		"a record already holding the AWS form must not be wrapped twice")
+}
+
+func TestSpec_AWSVersionsAreNewestFirst(t *testing.T) {
+	t.Parallel()
+	assert.Equal(t, []string{"v2.0.0-eksbuild.2", "v1.0.0-eksbuild.1"}, buildSpec().AWSVersions())
+}
+
+func TestSpec_ParseVersionAcceptsOnlyTheExactForms(t *testing.T) {
+	t.Parallel()
+	spec := buildSpec()
+	for input, want := range map[string]string{
+		"v2.0.0-eksbuild.2": "2.0.0",
+		"2.0.0":             "2.0.0",
+		"v1.0.0-eksbuild.1": "1.0.0",
+		"1.0.0":             "1.0.0",
+	} {
+		got, ok := spec.ParseVersion(input)
+		assert.True(t, ok, "%q must be accepted", input)
+		assert.Equal(t, want, got, "%q", input)
+	}
+}
+
+func TestSpec_ParseVersionRefusesEverythingElse(t *testing.T) {
+	t.Parallel()
+	spec := buildSpec()
+	for _, input := range []string{
+		"v2.0.0-eksbuild.1", // wrong build for 2.0.0
+		"v2.0.0-eksbuild.3",
+		"v1.0.0-eksbuild.2", // wrong build for 1.0.0
+		"v2.0.0",            // missing build
+		"2.0",               // partial
+		"v2.0.0-eksbuild.",
+		"2.0.0-eksbuild.2", // missing v
+		"V2.0.0-eksbuild.2",
+		" 2.0.0",
+		"9.9.9",
+		"",
+	} {
+		_, ok := spec.ParseVersion(input)
+		assert.False(t, ok, "%q must be refused", input)
+	}
+}
+
+func TestCatalog_EveryVersionHasAnExplicitBuild(t *testing.T) {
+	t.Parallel()
+	for _, spec := range Specs() {
+		for _, v := range spec.Versions {
+			assert.Equal(t, 1, spec.Builds[v], "%s %s needs an explicit build number", spec.Name, v)
+			assert.Equal(t, "v"+v+"-eksbuild.1", spec.AWSVersion(v), "%s %s", spec.Name, v)
+		}
+	}
+}

spinifex/domains/eks/awsapi/addon_versions_query_test.go

@@ -0,0 +1,105 @@
+package awsapi
+
+import (
+	"context"
+	"encoding/json"
+	"sync"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// captureDescribeAddonVersions answers eks.DescribeAddonVersions and returns a
+// function reading the last input the gateway sent to the service.
+func captureDescribeAddonVersions(t *testing.T, nc *nats.Conn) func() eks.DescribeAddonVersionsInput {
+	t.Helper()
+	var (
+		mu   sync.Mutex
+		last eks.DescribeAddonVersionsInput
+	)
+	sub, err := nc.Subscribe("eks.DescribeAddonVersions", func(m *nats.Msg) {
+		mu.Lock()
+		last = eks.DescribeAddonVersionsInput{}
+		_ = json.Unmarshal(m.Data, &last)
+		mu.Unlock()
+		_ = m.Respond([]byte(`{}`))
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	return func() eks.DescribeAddonVersionsInput {
+		mu.Lock()
+		defer mu.Unlock()
+		return last
+	}
+}
+
+// The gateway folds a GET query string into the body as a JSON object of
+// string lists; the model binds every field below from the query.
+func TestDescribeAddonVersions_PassesEveryModelQueryParameter(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	got := captureDescribeAddonVersions(t, nc)
+
+	query := []byte(`{"addonName":["aws-ebs-csi-driver"],"kubernetesVersion":["1.32"],` +
+		`"maxResults":["2"],"nextToken":["tok"],"types":["storage","networking"],` +
+		`"publishers":["eks"],"owners":["aws"]}`)
+	_, err := DescribeAddonVersions(context.Background(), nc, acct, query)
+	require.NoError(t, err)
+
+	in := got()
+	assert.Equal(t, "aws-ebs-csi-driver", aws.StringValue(in.AddonName))
+	assert.Equal(t, "1.32", aws.StringValue(in.KubernetesVersion))
+	assert.Equal(t, int64(2), aws.Int64Value(in.MaxResults))
+	assert.Equal(t, "tok", aws.StringValue(in.NextToken))
+	assert.Equal(t, []string{"storage", "networking"}, aws.StringValueSlice(in.Types))
+	assert.Equal(t, []string{"eks"}, aws.StringValueSlice(in.Publishers))
+	assert.Equal(t, []string{"aws"}, aws.StringValueSlice(in.Owners))
+}
+
+func TestDescribeAddonVersions_NoQueryLeavesEveryFilterUnset(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	got := captureDescribeAddonVersions(t, nc)
+
+	_, err := DescribeAddonVersions(context.Background(), nc, acct, nil)
+	require.NoError(t, err)
+
+	in := got()
+	assert.Nil(t, in.AddonName)
+	assert.Nil(t, in.KubernetesVersion)
+	assert.Nil(t, in.MaxResults)
+	assert.Nil(t, in.NextToken)
+	assert.Empty(t, in.Types)
+}
+
+func TestDescribeAddonVersions_RefusesMalformedQuery(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	captureDescribeAddonVersions(t, nc)
+
+	_, err := DescribeAddonVersions(context.Background(), nc, acct, []byte(`{"maxResults":["many"]}`))
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "maxResults")
+
+	_, err = DescribeAddonVersions(context.Background(), nc, acct, []byte(`not json`))
+	require.Error(t, err)
+}
+
+// The route must hand the folded query to the wrapper, not drop it.
+func TestDescribeAddonVersionsRoute_PassesTheQuery(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	got := captureDescribeAddonVersions(t, nc)
+
+	for _, r := range routes {
+		if r.Action != "DescribeAddonVersions" {
+			continue
+		}
+		_, err := r.Handler(context.Background(), nc, acct, "", nil, []byte(`{"addonName":["argocd"]}`))
+		require.NoError(t, err)
+		assert.Equal(t, "argocd", aws.StringValue(got().AddonName))
+		return
+	}
+	t.Fatal("no DescribeAddonVersions route")
+}

spinifex/domains/eks/awsapi/addons.go

@@ -16,8 +16,23 @@ func ListAddons(ctx context.Context, natsConn *nats.Conn, accountID, cluster str
 }
 
 // DescribeAddonVersions — GET /addons/supported-versions.
-func DescribeAddonVersions(ctx context.Context, natsConn *nats.Conn, accountID string) (*eks.DescribeAddonVersionsOutput, error) {
-	return ekscluster.NewNATSEKSService(natsConn).DescribeAddonVersions(ctx, &eks.DescribeAddonVersionsInput{}, accountID)
+func DescribeAddonVersions(ctx context.Context, natsConn *nats.Conn, accountID string, query []byte) (*eks.DescribeAddonVersionsOutput, error) {
+	q, err := parseListQuery(query)
+	if err != nil {
+		return nil, err
+	}
+	input := &eks.DescribeAddonVersionsInput{
+		AddonName:         q.str("addonName"),
+		KubernetesVersion: q.str("kubernetesVersion"),
+		NextToken:         q.str("nextToken"),
+		Types:             q.strs("types"),
+		Publishers:        q.strs("publishers"),
+		Owners:            q.strs("owners"),
+	}
+	if input.MaxResults, err = q.int64("maxResults"); err != nil {
+		return nil, err
+	}
+	return ekscluster.NewNATSEKSService(natsConn).DescribeAddonVersions(ctx, input, accountID)
 }
 
 // CreateAddon — POST /clusters/{name}/addons.

spinifex/domains/eks/awsapi/query.go

@@ -0,0 +1,61 @@
+package awsapi
+
+import (
+	"encoding/json"
+	"strconv"
+
+	"github.com/aws/aws-sdk-go/aws"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// listQuery holds the query-string inputs of a GET action, which the gateway
+// folds into the body as a JSON object of string lists.
+type listQuery map[string][]string
+
+// parseListQuery decodes the folded query of a GET action. An empty body means
+// the request carried no query string.
+func parseListQuery(body []byte) (listQuery, error) {
+	q := listQuery{}
+	if len(body) == 0 {
+		return q, nil
+	}
+	if err := json.Unmarshal(body, (*map[string][]string)(&q)); err != nil {
+		return nil, awserrors.Errorf(awserrors.ErrorEKSInvalidParameter, "malformed query string")
+	}
+	return q, nil
+}
+
+// str returns the first value of name, or nil when it is absent or empty.
+func (q listQuery) str(name string) *string {
+	if v := q[name]; len(v) > 0 && v[0] != "" {
+		return aws.String(v[0])
+	}
+	return nil
+}
+
+// strs returns every non-empty value of name, or nil when there are none.
+func (q listQuery) strs(name string) []*string {
+	var out []string
+	for _, v := range q[name] {
+		if v != "" {
+			out = append(out, v)
+		}
+	}
+	if len(out) == 0 {
+		return nil
+	}
+	return aws.StringSlice(out)
+}
+
+// int64 returns the first value of name as an integer, or nil when absent.
+func (q listQuery) int64(name string) (*int64, error) {
+	s := q.str(name)
+	if s == nil {
+		return nil, nil
+	}
+	n, err := strconv.ParseInt(*s, 10, 64)
+	if err != nil {
+		return nil, awserrors.Errorf(awserrors.ErrorEKSInvalidParameter, "%s must be an integer", name)
+	}
+	return aws.Int64(n), nil
+}

spinifex/domains/eks/awsapi/registration.go

@@ -82,17 +82,39 @@ func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 	if err == nil {
 		return nil
 	}
-	if action, _, _, ok := router.Lookup(inv.Request.Method, inv.Request.URL.EscapedPath()); ok && IsInternalAction(action) {
+	action, _, _, routed := router.Lookup(inv.Request.Method, inv.Request.URL.EscapedPath())
+	if routed && IsInternalAction(action) {
 		return err
 	}
 	code, msg, ok := awserrors.ResolveErrorDetail(err)
 	if !ok || code != awserrors.ErrorInvalidParameterValue {
 		return err
 	}
-	if msg == "" {
-		return errors.New(awserrors.ErrorEKSInvalidParameter)
+	wire := awserrors.ErrorEKSInvalidParameter
+	if override, ok := parameterRefusalCodes[action]; ok {
+		wire = override
+		if msg == "" {
+			msg = "a parameter of the request is not valid"
+		}
 	}
-	return awserrors.Errorf(awserrors.ErrorEKSInvalidParameter, "%s", msg)
+	if msg == "" {
+		return errors.New(wire)
+	}
+	return awserrors.Errorf(wire, "%s", msg)
+}
+
+// parameterRefusalCodes holds the operations whose model does not declare
+// InvalidParameterException, with the declared code a refused parameter uses.
+var parameterRefusalCodes = map[string]string{
+	"DescribeCluster":              awserrors.ErrorEKSClient,
+	"DeleteCluster":                awserrors.ErrorEKSClient,
+	"DescribeAccessEntry":          awserrors.ErrorEKSInvalidRequest,
+	"DeleteAccessEntry":            awserrors.ErrorEKSInvalidRequest,
+	"DisassociateAccessPolicy":     awserrors.ErrorEKSInvalidRequest,
+	"ListAssociatedAccessPolicies": awserrors.ErrorEKSInvalidRequest,
+	"TagResource":                  awserrors.ErrorEKSBadRequest,
+	"UntagResource":                awserrors.ErrorEKSBadRequest,
+	"ListTagsForResource":          awserrors.ErrorEKSBadRequest,
 }
 
 // serve resolves the route, authorizes the request, then runs the action.

spinifex/domains/eks/awsapi/registration_error_codes_test.go

@@ -0,0 +1,98 @@
+//test:in-package — routes is unexported, and the table must cover exactly the
+// actions the registration serves.
+
+package awsapi
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/bluebottle/pkg/iampolicy"
+	"github.com/mulgadc/spinifex/internal/awsmodel"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// routePath fills a route pattern with placeholder segments. The tag routes
+// take an ARN, which has to be escaped to stay one path segment.
+func routePath(pattern string) string {
+	arn := url.PathEscape("arn:aws:eks:us-east-1:111122223333:cluster/c")
+	path := strings.ReplaceAll(pattern, "*", arn)
+	for {
+		start := strings.Index(path, "{")
+		if start < 0 {
+			return path
+		}
+		end := strings.Index(path[start:], "}")
+		path = path[:start] + "x" + path[start+end+1:]
+	}
+}
+
+// A parameter refusal must reach the wire under a code the operation declares
+// in the EKS model, or SDK clients cannot match it to a typed error.
+func TestRegistration_ParameterRefusalCodeIsDeclaredByOperation(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	sub, err := nc.Subscribe("eks.>", func(m *nats.Msg) {
+		if m.Reply != "" {
+			_ = m.Respond(awserrors.GenerateErrorPayload(awserrors.ErrorInvalidParameterValue))
+		}
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+	model, err := awsmodel.Load(awsmodel.EKS)
+	require.NoError(t, err)
+	registration := NewRegistration(Deps{NATS: nc})
+
+	for _, route := range routes {
+		if IsInternalAction(route.Action) {
+			continue
+		}
+		// The handler takes no input and the model declares no client error.
+		if route.Action == "ListAccessPolicies" {
+			continue
+		}
+		t.Run(route.Action, func(t *testing.T) {
+			operation, ok := model.Operation(route.Action)
+			require.True(t, ok, "%s is not an operation in the EKS model", route.Action)
+			declared := make([]string, 0, len(operation.Errors))
+			for _, ref := range operation.Errors {
+				declared = append(declared, ref.Shape)
+			}
+
+			req := httptest.NewRequest(route.Method, routePath(route.Pattern), strings.NewReader(`{}`))
+			inv := dispatch.Invocation{
+				Request:   req,
+				AccountID: "111122223333",
+				Region:    "us-east-1",
+				Authorize: func(string, string, []string, iampolicy.ConditionKeys) error { return nil },
+				Actor: func() (dispatch.Actor, error) {
+					return dispatch.Actor{AccountID: "111122223333", PrincipalType: "user", UserName: "alice"}, nil
+				},
+			}
+
+			err := registration.Dispatch(httptest.NewRecorder(), inv)
+
+			require.Error(t, err)
+			code, ok := awserrors.ResolveErrorCode(err)
+			require.True(t, ok, "unresolvable error code: %v", err)
+			assert.Contains(t, declared, code,
+				"%s %s refuses a parameter with %s, which the operation does not declare", route.Method, route.Pattern, code)
+		})
+	}
+}
+
+// The access operations declare InvalidRequestException, not a parameter code.
+func TestEKSInvalidRequestCodeIsRegistered(t *testing.T) {
+	assert.Equal(t, "InvalidRequestException", awserrors.ErrorEKSInvalidRequest)
+	assert.Equal(t, http.StatusBadRequest, awserrors.HTTPStatusForError(awserrors.Errorf(awserrors.ErrorEKSInvalidRequest, "x")))
+	assert.Equal(t, "BadRequestException", awserrors.ErrorEKSBadRequest)
+	assert.Equal(t, http.StatusBadRequest, awserrors.HTTPStatusForError(awserrors.Errorf(awserrors.ErrorEKSBadRequest, "x")))
+}

spinifex/domains/eks/awsapi/routes.go

@@ -131,7 +131,7 @@ var routes = []route{
 	// Addons
 	{Method: "GET", Pattern: "/addons/supported-versions", Action: "DescribeAddonVersions",
 		Handler: func(ctx context.Context, nc *nats.Conn, acct, callerARN string, p []string, b []byte) (any, error) {
-			return DescribeAddonVersions(ctx, nc, acct)
+			return DescribeAddonVersions(ctx, nc, acct, b)
 		}},
 	{Method: "GET", Pattern: "/clusters/{clusterName}/addons", Action: "ListAddons",
 		Handler: func(ctx context.Context, nc *nats.Conn, acct, callerARN string, p []string, b []byte) (any, error) {

spinifex/domains/eks/awsapi/wrappers_test.go

@@ -176,7 +176,7 @@ func TestGatewayWrappers_Addons(t *testing.T) {
 	require.NoError(t, err)
 	assert.NotNil(t, out1)
 
-	out2, err := DescribeAddonVersions(context.Background(), nc, acct)
+	out2, err := DescribeAddonVersions(context.Background(), nc, acct, nil)
 	require.NoError(t, err)
 	assert.NotNil(t, out2)
 

spinifex/domains/eks/cluster/addon_describe_versions_test.go

@@ -0,0 +1,171 @@
+package cluster
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/mulgadc/spinifex/spinifex/domains/eks/addon"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func describeVersions(t *testing.T, svc *EKSServiceImpl, in *eks.DescribeAddonVersionsInput) *eks.DescribeAddonVersionsOutput {
+	t.Helper()
+	out, err := svc.DescribeAddonVersions(context.Background(), in, testAccountID)
+	require.NoError(t, err)
+	return out
+}
+
+// The terraform aws_eks_addon_version data source asks for one add-on at the
+// cluster's Kubernetes version, then takes the version whose compatibility for
+// that cluster version is the default (or, with most_recent, the first one).
+func TestDescribeAddonVersions_TerraformProviderCall(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	out := describeVersions(t, svc, &eks.DescribeAddonVersionsInput{
+		AddonName:         aws.String(csiAddon),
+		KubernetesVersion: aws.String(defaultK8sVersion),
+	})
+	require.Len(t, out.Addons, 1)
+	info := out.Addons[0]
+	assert.Equal(t, csiAddon, aws.StringValue(info.AddonName))
+	assert.Nil(t, out.NextToken)
+
+	spec, ok := addon.Lookup(csiAddon)
+	require.True(t, ok)
+	require.Equal(t, spec.AWSVersions(), versionStrings(info), "newest first, AWS form")
+
+	defaults := 0
+	for _, v := range info.AddonVersions {
+		require.Len(t, v.Compatibilities, 1, "one entry, for the requested cluster version")
+		c := v.Compatibilities[0]
+		assert.Equal(t, defaultK8sVersion, aws.StringValue(c.ClusterVersion))
+		assert.Equal(t, []string{"*"}, aws.StringValueSlice(c.PlatformVersions))
+		if aws.BoolValue(c.DefaultVersion) {
+			defaults++
+			assert.Equal(t, spec.AWSVersion(spec.DefaultVersion), aws.StringValue(v.AddonVersion))
+		}
+	}
+	assert.Equal(t, 1, defaults, "exactly one defaultVersion true")
+}
+
+// AWS observed (EKS-01): compatibilities holds only the requested version. The
+// response for a version no add-on supports is not captured; this pins the
+// shape #61 chose, an empty list with no error.
+func TestDescribeAddonVersions_UnsupportedKubernetesVersionIsEmpty(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	out := describeVersions(t, svc, &eks.DescribeAddonVersionsInput{
+		AddonName:         aws.String(csiAddon),
+		KubernetesVersion: aws.String("1.29"),
+	})
+	assert.Empty(t, out.Addons)
+	assert.Nil(t, out.NextToken)
+}
+
+func TestDescribeAddonVersions_FiltersByNameAndHidesFixturesUnlessNamed(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	for _, name := range []string{csiAddon, "argocd", albController} {
+		out := describeVersions(t, svc, &eks.DescribeAddonVersionsInput{AddonName: aws.String(name)})
+		require.Len(t, out.Addons, 1, name)
+		assert.Equal(t, name, aws.StringValue(out.Addons[0].AddonName))
+	}
+	assert.Empty(t, describeVersions(t, svc, &eks.DescribeAddonVersionsInput{AddonName: aws.String("vpc-cni")}).Addons)
+}
+
+// maxResults counts add-ons, not versions; a page boundary never splits an
+// add-on's version list, and the last page carries no token.
+func TestDescribeAddonVersions_PagesByAddon(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	all := describeVersions(t, svc, &eks.DescribeAddonVersionsInput{})
+	require.Greater(t, len(all.Addons), 1)
+	assert.Nil(t, all.NextToken)
+
+	var names []string
+	in := &eks.DescribeAddonVersionsInput{MaxResults: aws.Int64(1)}
+	for range len(all.Addons) {
+		out := describeVersions(t, svc, in)
+		require.Len(t, out.Addons, 1)
+		names = append(names, aws.StringValue(out.Addons[0].AddonName))
+		assert.Equal(t, versionStrings(all.Addons[len(names)-1]), versionStrings(out.Addons[0]))
+		in.NextToken = out.NextToken
+	}
+	assert.Nil(t, in.NextToken, "last page carries no token")
+	var want []string
+	for _, a := range all.Addons {
+		want = append(want, aws.StringValue(a.AddonName))
+	}
+	assert.Equal(t, want, names)
+}
+
+// Every add-on carries the descriptive fields the data source and console read.
+func TestDescribeAddonVersions_CarriesAddonAndVersionMetadata(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	for _, a := range describeVersions(t, svc, &eks.DescribeAddonVersionsInput{}).Addons {
+		name := aws.StringValue(a.AddonName)
+		assert.NotEmpty(t, aws.StringValue(a.Type), name)
+		assert.NotEmpty(t, aws.StringValue(a.Owner), name)
+		assert.NotEmpty(t, aws.StringValue(a.Publisher), name)
+		assert.Nil(t, a.MarketplaceInformation, name)
+		for _, v := range a.AddonVersions {
+			assert.NotEmpty(t, v.Architecture, name)
+			assert.NotNil(t, v.RequiresConfiguration, name)
+			assert.NotNil(t, v.RequiresIamPermissions, name)
+		}
+	}
+}
+
+// types, publishers and owners are model filters; each keeps add-ons whose
+// field matches one of the given values.
+func TestDescribeAddonVersions_FiltersByTypePublisherOwner(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	first := describeVersions(t, svc, &eks.DescribeAddonVersionsInput{}).Addons[0]
+	cases := []struct {
+		name  string
+		match func(*eks.DescribeAddonVersionsInput, []*string)
+		value *string
+	}{
+		{"types", func(in *eks.DescribeAddonVersionsInput, v []*string) { in.Types = v }, first.Type},
+		{"publishers", func(in *eks.DescribeAddonVersionsInput, v []*string) { in.Publishers = v }, first.Publisher},
+		{"owners", func(in *eks.DescribeAddonVersionsInput, v []*string) { in.Owners = v }, first.Owner},
+	}
+	for _, c := range cases {
+		hit := &eks.DescribeAddonVersionsInput{}
+		c.match(hit, aws.StringSlice([]string{"nobody", aws.StringValue(c.value)}))
+		out := describeVersions(t, svc, hit)
+		require.NotEmpty(t, out.Addons, c.name)
+		assert.Contains(t, addonNames(out), aws.StringValue(first.AddonName), c.name)
+
+		miss := &eks.DescribeAddonVersionsInput{}
+		c.match(miss, aws.StringSlice([]string{"nobody"}))
+		assert.Empty(t, describeVersions(t, svc, miss).Addons, c.name)
+	}
+}
+
+func versionStrings(a *eks.AddonInfo) []string {
+	out := make([]string, 0, len(a.AddonVersions))
+	for _, v := range a.AddonVersions {
+		out = append(out, aws.StringValue(v.AddonVersion))
+	}
+	return out
+}
+
+func addonNames(o *eks.DescribeAddonVersionsOutput) []string {
+	out := make([]string, 0, len(o.Addons))
+	for _, a := range o.Addons {
+		out = append(out, aws.StringValue(a.AddonName))
+	}
+	return out
+}

spinifex/domains/eks/cluster/addon_version_format_test.go

@@ -0,0 +1,220 @@
+package cluster
+
+import (
+	"bytes"
+	"context"
+	"log/slog"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/mulgadc/spinifex/spinifex/domains/eks/addon"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	csiAddon       = "aws-ebs-csi-driver"
+	csiBare        = "1.40.1"
+	csiAWSVersion  = "v1.40.1-eksbuild.1"
+	csiWrongBuild  = "v1.40.1-eksbuild.2"
+	versionParam   = "addonVersion"
+	notBeforeAWSOK = "an AWS-format version must be accepted"
+)
+
+func TestDescribeAddonVersions_ReturnsAWSFormNewestFirst(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	out, err := svc.DescribeAddonVersions(context.Background(), &eks.DescribeAddonVersionsInput{}, testAccountID)
+	require.NoError(t, err)
+	require.NotEmpty(t, out.Addons)
+	for _, a := range out.Addons {
+		spec, ok := addon.Lookup(aws.StringValue(a.AddonName))
+		require.True(t, ok)
+		require.Len(t, a.AddonVersions, len(spec.Versions))
+		for i, v := range a.AddonVersions {
+			assert.Equal(t, "v"+spec.Versions[i]+"-eksbuild.1", aws.StringValue(v.AddonVersion),
+				"%s version %d", spec.Name, i)
+		}
+	}
+}
+
+func TestCreateAddon_AcceptsBothFormsAndStoresTheBareForm(t *testing.T) {
+	t.Parallel()
+	for name, input := range map[string]string{"aws form": csiAWSVersion, "bare form": csiBare} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			svc, fake := setupAddonService(t)
+
+			out, err := svc.CreateAddon(context.Background(), &eks.CreateAddonInput{
+				ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+				AddonVersion: aws.String(input),
+			}, testAccountID)
+			require.NoError(t, err, notBeforeAWSOK)
+			assert.Equal(t, csiAWSVersion, aws.StringValue(out.Addon.AddonVersion), "response is in AWS form")
+
+			rec, err := addon.Get(t.Context(), acctKVForTest(t, svc), "c1", csiAddon)
+			require.NoError(t, err)
+			assert.Equal(t, csiBare, rec.AddonVersion, "the record keeps the upstream form")
+			require.Len(t, fake.installs, 1)
+			assert.Equal(t, csiBare, fake.installs[0].AddonVersion, "the bundle lookup sees the upstream form")
+		})
+	}
+}
+
+func TestCreateAddon_DefaultVersionResponseIsAWSForm(t *testing.T) {
+	t.Parallel()
+	svc, _ := setupAddonService(t)
+	out, err := svc.CreateAddon(context.Background(), &eks.CreateAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+	}, testAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, csiAWSVersion, aws.StringValue(out.Addon.AddonVersion))
+}
+
+func TestUpdateAddon_AcceptsBothFormsAndStoresTheBareForm(t *testing.T) {
+	t.Parallel()
+	for name, input := range map[string]string{"aws form": csiAWSVersion, "bare form": csiBare} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			svc, fake := setupAddonService(t)
+			_, err := svc.CreateAddon(context.Background(), &eks.CreateAddonInput{
+				ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+			}, testAccountID)
+			require.NoError(t, err)
+
+			_, err = svc.UpdateAddon(context.Background(), &eks.UpdateAddonInput{
+				ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+				AddonVersion: aws.String(input),
+			}, testAccountID)
+			require.NoError(t, err, notBeforeAWSOK)
+
+			rec, err := addon.Get(t.Context(), acctKVForTest(t, svc), "c1", csiAddon)
+			require.NoError(t, err)
+			assert.Equal(t, csiBare, rec.AddonVersion)
+			require.Len(t, fake.installs, 2)
+			assert.Equal(t, csiBare, fake.installs[1].AddonVersion)
+		})
+	}
+}
+
+func TestDescribeAddon_ReturnsAWSFormForAnyStoredForm(t *testing.T) {
+	t.Parallel()
+	for name, stored := range map[string]string{
+		"record stored bare before the change": csiBare,
+		"record already in AWS form":           csiAWSVersion,
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			svc, _ := setupAddonService(t)
+			_, err := svc.addons().Create(context.Background(), acctKVForTest(t, svc), testAccountID, addon.Desired{
+				Cluster: "c1", Name: csiAddon, Version: stored,
+			})
+			require.NoError(t, err)
+
+			desc, err := svc.DescribeAddon(context.Background(), &eks.DescribeAddonInput{
+				ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+			}, testAccountID)
+			require.NoError(t, err)
+			assert.Equal(t, csiAWSVersion, aws.StringValue(desc.Addon.AddonVersion), "no double wrap")
+
+			list, err := svc.ListAddons(context.Background(), &eks.ListAddonsInput{ClusterName: aws.String("c1")}, testAccountID)
+			require.NoError(t, err)
+			assert.Equal(t, []string{csiAddon}, aws.StringValueSlice(list.Addons))
+		})
+	}
+}
+
+func TestUpdateAddon_ResponseAndRecordAfterBareRecord(t *testing.T) {
+	t.Parallel()
+	svc, _ := setupAddonService(t)
+	_, err := svc.addons().Create(context.Background(), acctKVForTest(t, svc), testAccountID, addon.Desired{
+		Cluster: "c1", Name: csiAddon, Version: csiBare,
+	})
+	require.NoError(t, err)
+
+	_, err = svc.UpdateAddon(context.Background(), &eks.UpdateAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+		ConfigurationValues: aws.String(`{"a":1}`),
+	}, testAccountID)
+	require.NoError(t, err)
+	desc, err := svc.DescribeAddon(context.Background(), &eks.DescribeAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+	}, testAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, csiAWSVersion, aws.StringValue(desc.Addon.AddonVersion))
+}
+
+func TestAddonVersionRefusal_NamesTheParameterAndListsSupportedValues(t *testing.T) {
+	t.Parallel()
+	svc, _ := setupAddonService(t)
+	_, err := svc.CreateAddon(context.Background(), &eks.CreateAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String("argocd"),
+	}, testAccountID)
+	require.NoError(t, err)
+
+	refusals := map[string]func() error{
+		"create": func() error {
+			_, e := svc.CreateAddon(context.Background(), &eks.CreateAddonInput{
+				ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+				AddonVersion: aws.String(csiWrongBuild),
+			}, testAccountID)
+			return e
+		},
+		"update": func() error {
+			_, e := svc.UpdateAddon(context.Background(), &eks.UpdateAddonInput{
+				ClusterName: aws.String("c1"), AddonName: aws.String("argocd"),
+				AddonVersion: aws.String("v3.0.23-eksbuild.2"),
+			}, testAccountID)
+			return e
+		},
+	}
+	want := map[string]string{"create": csiAWSVersion, "update": "v3.0.23-eksbuild.1"}
+	for name, refuse := range refusals {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			err := refuse()
+			require.Error(t, err)
+			code, message, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok)
+			assert.Equal(t, awserrors.ErrorInvalidParameterValue, code, "this change leaves the error code alone")
+			assert.Contains(t, message, versionParam)
+			assert.Contains(t, message, want[name], "the supported AWS-form value is listed")
+		})
+	}
+}
+
+func TestAddonBareVersionInput_LogsItsUseAtInfo(t *testing.T) {
+	var buf bytes.Buffer
+	restore := slog.Default()
+	slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo})))
+	t.Cleanup(func() { slog.SetDefault(restore) })
+
+	svc, _ := setupAddonService(t)
+	_, err := svc.CreateAddon(context.Background(), &eks.CreateAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+	}, testAccountID)
+	require.NoError(t, err)
+
+	buf.Reset()
+	_, err = svc.UpdateAddon(context.Background(), &eks.UpdateAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+		AddonVersion: aws.String(csiBare),
+	}, testAccountID)
+	require.NoError(t, err)
+	logged := buf.String()
+	assert.Contains(t, logged, "level=INFO")
+	assert.Contains(t, logged, "bare")
+	assert.Contains(t, logged, csiAddon)
+	assert.Contains(t, logged, "c1")
+
+	buf.Reset()
+	_, err = svc.UpdateAddon(context.Background(), &eks.UpdateAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+		AddonVersion: aws.String(csiAWSVersion),
+	}, testAccountID)
+	require.NoError(t, err, notBeforeAWSOK)
+	assert.NotContains(t, buf.String(), "bare", "the AWS form is the intended form and logs nothing")
+}

spinifex/domains/eks/cluster/addons.go

@@ -2,8 +2,12 @@ package cluster
 
 import (
 	"context"
+	"encoding/base64"
 	"errors"
 	"fmt"
+	"log/slog"
+	"slices"
+	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/eks"
@@ -35,15 +39,20 @@ func (s *EKSServiceImpl) ListAddons(ctx context.Context, input *eks.ListAddonsIn
 	return &eks.ListAddonsOutput{Addons: aws.StringSlice(names)}, nil
 }
 
-// DescribeAddonVersions returns the static add-on catalog, optionally filtered by name.
-func (s *EKSServiceImpl) DescribeAddonVersions(ctx context.Context, input *eks.DescribeAddonVersionsInput, _ string) (*eks.DescribeAddonVersionsOutput, error) {
-	filter := ""
-	if input != nil {
-		filter = aws.StringValue(input.AddonName)
+// DescribeAddonVersions returns the add-on catalogue filtered by name, owner,
+// publisher, type and the Kubernetes version its versions support, one page
+// of add-ons at a time.
+func (s *EKSServiceImpl) DescribeAddonVersions(_ context.Context, input *eks.DescribeAddonVersionsInput, _ string) (*eks.DescribeAddonVersionsOutput, error) {
+	if input == nil {
+		input = &eks.DescribeAddonVersionsInput{}
 	}
-	specs := addon.Specs()
-	out := make([]*eks.AddonInfo, 0, len(specs))
-	for _, spec := range specs {
+	// Every catalogue version supports only the K3s line the node image ships.
+	if v := aws.StringValue(input.KubernetesVersion); v != "" && v != defaultK8sVersion {
+		return &eks.DescribeAddonVersionsOutput{Addons: []*eks.AddonInfo{}}, nil
+	}
+	filter := aws.StringValue(input.AddonName)
+	var matched []addon.Spec
+	for _, spec := range addon.Specs() {
 		if filter != "" && spec.Name != filter {
 			continue
 		}
@@ -52,9 +61,51 @@ func (s *EKSServiceImpl) DescribeAddonVersions(ctx context.Context, input *eks.D
 		if spec.Hidden && filter == "" {
 			continue
 		}
+		if !matchesAny(input.Types, spec.Type) || !matchesAny(input.Owners, addonOwner) ||
+			!matchesAny(input.Publishers, addonPublisher) {
+			continue
+		}
+		matched = append(matched, spec)
+	}
+	page, next, err := pageAddons(matched, input.MaxResults, input.NextToken)
+	if err != nil {
+		return nil, err
+	}
+	out := make([]*eks.AddonInfo, 0, len(page))
+	for _, spec := range page {
 		out = append(out, addonSpecToAWS(spec))
 	}
-	return &eks.DescribeAddonVersionsOutput{Addons: out}, nil
+	return &eks.DescribeAddonVersionsOutput{Addons: out, NextToken: next}, nil
+}
+
+// matchesAny reports whether the filter is empty or lists value.
+func matchesAny(filter []*string, value string) bool {
+	return len(filter) == 0 || slices.Contains(aws.StringValueSlice(filter), value)
+}
+
+// maxAddonPage bounds maxResults, the model's range being 1 to 100.
+const maxAddonPage = 100
+
+// pageAddons returns the add-ons after the token, at most maxResults of them.
+// The token is the base64 name of the last add-on returned, opaque to clients.
+func pageAddons(specs []addon.Spec, maxResults *int64, token *string) ([]addon.Spec, *string, error) {
+	if t := aws.StringValue(token); t != "" {
+		raw, err := base64.RawURLEncoding.DecodeString(t)
+		if err != nil {
+			return nil, nil, awserrors.Errorf(awserrors.ErrorEKSInvalidParameter, "nextToken is not valid")
+		}
+		after := string(raw)
+		specs = slices.DeleteFunc(slices.Clone(specs), func(sp addon.Spec) bool { return sp.Name <= after })
+	}
+	limit := len(specs)
+	if maxResults != nil {
+		limit = min(max(int(*maxResults), 1), maxAddonPage)
+	}
+	if limit >= len(specs) {
+		return specs, nil, nil
+	}
+	next := base64.RawURLEncoding.EncodeToString([]byte(specs[limit-1].Name))
+	return specs[:limit], &next, nil
 }
 
 // validateAddonServiceAccountRoleArn rejects a non-empty value that is not
@@ -89,11 +140,12 @@ func (s *EKSServiceImpl) CreateAddon(ctx context.Context, input *eks.CreateAddon
 	if !ok {
 		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
 	}
-	version := aws.StringValue(input.AddonVersion)
-	if version == "" {
-		version = spec.DefaultVersion
-	} else if !spec.SupportsVersion(version) {
-		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
+	version := spec.DefaultVersion
+	if requested := aws.StringValue(input.AddonVersion); requested != "" {
+		var err error
+		if version, err = resolveAddonVersion(ctx, spec, requested, cluster); err != nil {
+			return nil, err
+		}
 	}
 	acctKV, err := s.acctKVForNewDependant(ctx, accountID, cluster)
 	if err != nil {
@@ -197,9 +249,13 @@ func (s *EKSServiceImpl) UpdateAddon(ctx context.Context, input *eks.UpdateAddon
 	version := aws.StringValue(input.AddonVersion)
 	if version != "" {
 		spec, ok := addon.Lookup(addonName)
-		if !ok || !spec.SupportsVersion(version) {
+		if !ok {
 			return nil, errors.New(awserrors.ErrorInvalidParameterValue)
 		}
+		var err error
+		if version, err = resolveAddonVersion(ctx, spec, version, cluster); err != nil {
+			return nil, err
+		}
 	}
 	rec, err := s.addons().Update(ctx, acctKV, accountID, cluster, addonName, addon.Change{
 		Version:               version,
@@ -246,7 +302,7 @@ func addonRecordToAWS(cluster string, rec *addon.Record) *eks.Addon {
 	out := &eks.Addon{
 		AddonArn:     aws.String(rec.Arn),
 		AddonName:    aws.String(rec.AddonName),
-		AddonVersion: aws.String(rec.AddonVersion),
+		AddonVersion: aws.String(awsAddonVersion(rec)),
 		ClusterName:  aws.String(cluster),
 		Status:       aws.String(string(rec.Status)),
 		CreatedAt:    aws.Time(rec.CreatedAt),
@@ -269,17 +325,36 @@ func addonRecordToAWS(cluster string, rec *addon.Record) *eks.Addon {
 	return out
 }
 
-// addonSpecToAWS converts a catalog spec to the SDK AddonInfo shape.
+// The owner and publisher Spinifex reports for every catalogue add-on, as AWS
+// does for the add-ons it publishes.
+const (
+	addonOwner     = "aws"
+	addonPublisher = "eks"
+)
+
+// addonSpecToAWS converts a catalog spec to the SDK AddonInfo shape. Every
+// version supports the one cluster version the node image ships, and exactly
+// one of them is that cluster version's default.
 func addonSpecToAWS(spec addon.Spec) *eks.AddonInfo {
 	versions := make([]*eks.AddonVersionInfo, 0, len(spec.Versions))
-	for _, v := range spec.Versions {
+	for _, v := range spec.AWSVersions() {
 		versions = append(versions, &eks.AddonVersionInfo{
 			AddonVersion:           aws.String(v),
+			Architecture:           aws.StringSlice([]string{"amd64"}),
+			RequiresConfiguration:  aws.Bool(false),
 			RequiresIamPermissions: aws.Bool(spec.RequiresIRSA),
+			Compatibilities: []*eks.Compatibility{{
+				ClusterVersion:   aws.String(defaultK8sVersion),
+				PlatformVersions: aws.StringSlice([]string{"*"}),
+				DefaultVersion:   aws.Bool(v == spec.AWSVersion(spec.DefaultVersion)),
+			}},
 		})
 	}
 	return &eks.AddonInfo{
 		AddonName:     aws.String(spec.Name),
+		Type:          aws.String(spec.Type),
+		Owner:         aws.String(addonOwner),
+		Publisher:     aws.String(addonPublisher),
 		AddonVersions: versions,
 	}
 }
@@ -308,3 +383,30 @@ func (s *EKSServiceImpl) addonBucket(ctx context.Context, accountID string) (jet
 	}
 	return GetOrCreateAccountBucket(ctx, js, accountID)
 }
+
+// resolveAddonVersion maps a client version, in AWS form or bare upstream form,
+// to the upstream form the record and the bundle lookup use. The bare form is
+// more lenient than AWS, so each use is logged.
+func resolveAddonVersion(ctx context.Context, spec addon.Spec, requested, cluster string) (string, error) {
+	upstream, ok := spec.ParseVersion(requested)
+	if !ok {
+		return "", awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
+			"addonVersion %q is not supported for add-on %s; supported versions: %s",
+			requested, spec.Name, strings.Join(spec.AWSVersions(), ", "))
+	}
+	if requested == upstream {
+		slog.InfoContext(ctx, "EKS add-on version given in bare upstream form",
+			"addon", spec.Name, "cluster", cluster, "version", requested)
+	}
+	return upstream, nil
+}
+
+// awsAddonVersion returns a stored record's version in AWS form; an add-on the
+// catalogue no longer lists keeps its stored string.
+func awsAddonVersion(rec *addon.Record) string {
+	spec, ok := addon.Lookup(rec.AddonName)
+	if !ok {
+		return rec.AddonVersion
+	}
+	return spec.AWSVersion(rec.AddonVersion)
+}

spinifex/domains/eks/cluster/addons_test.go

@@ -98,7 +98,9 @@ func TestCreateAddon_UnknownVersionRejected(t *testing.T) {
 		ClusterName: aws.String("c1"), AddonName: aws.String(albController),
 		AddonVersion: aws.String("9.9.9-nope"),
 	}, testAccountID)
-	require.EqualError(t, err, awserrors.ErrorInvalidParameterValue)
+	code, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorInvalidParameterValue, code)
 }
 
 func TestCreateAddon_CreatesStagingAndDefaultsVersion(t *testing.T) {
@@ -113,7 +115,7 @@ func TestCreateAddon_CreatesStagingAndDefaultsVersion(t *testing.T) {
 	require.NotNil(t, out.Addon)
 	// Version defaults to catalog default; status starts CREATING (honest until
 	// VM-side delivery confirms ACTIVE).
-	assert.Equal(t, spec.DefaultVersion, aws.StringValue(out.Addon.AddonVersion))
+	assert.Equal(t, spec.AWSVersion(spec.DefaultVersion), aws.StringValue(out.Addon.AddonVersion))
 	assert.Equal(t, string(addon.StatusCreating), aws.StringValue(out.Addon.Status))
 	assert.Contains(t, aws.StringValue(out.Addon.AddonArn), ":addon/c1/"+albController)
 
@@ -214,7 +216,7 @@ func TestUpdateAddon_ChangesVersionAndReinstalls(t *testing.T) {
 	}, testAccountID)
 	require.NoError(t, err)
 	assert.Equal(t, `{"replicaCount":2}`, aws.StringValue(desc.Addon.ConfigurationValues))
-	assert.Equal(t, spec.DefaultVersion, aws.StringValue(desc.Addon.AddonVersion))
+	assert.Equal(t, spec.AWSVersion(spec.DefaultVersion), aws.StringValue(desc.Addon.AddonVersion))
 	assert.Equal(t, string(addon.StatusUpdating), aws.StringValue(desc.Addon.Status))
 }
 
@@ -230,7 +232,9 @@ func TestUpdateAddon_UnknownVersionRejected(t *testing.T) {
 		ClusterName: aws.String("c1"), AddonName: aws.String("argocd"),
 		AddonVersion: aws.String("9.9.9-nope"),
 	}, testAccountID)
-	require.EqualError(t, err, awserrors.ErrorInvalidParameterValue)
+	code, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorInvalidParameterValue, code)
 }
 
 func TestUpdateAddon_MissingIsNotFound(t *testing.T) {

spinifex/domains/eks/cluster/nodegroup.go

@@ -886,7 +886,7 @@ func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string
 		return s.attachRoleToProfile(accountID, profileName, roleName,
 			aws.StringValue(out.InstanceProfile.Arn), len(out.InstanceProfile.Roles) > 0)
 	}
-	if err.Error() != awserrors.ErrorIAMNoSuchEntity {
+	if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMNoSuchEntity {
 		return "", fmt.Errorf("get instance profile %q: %w", profileName, err)
 	}
 
@@ -895,7 +895,7 @@ func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string
 	})
 	if err != nil {
 		// A racing worker launch may have created it first; re-read and converge.
-		if err.Error() != awserrors.ErrorIAMEntityAlreadyExists {
+		if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMEntityAlreadyExists {
 			return "", fmt.Errorf("create instance profile %q: %w", profileName, err)
 		}
 		got, gerr := s.iamEnsurer().GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{
@@ -922,7 +922,7 @@ func (s *EKSServiceImpl) attachRoleToProfile(accountID, profileName, roleName, p
 		InstanceProfileName: aws.String(profileName),
 		RoleName:            aws.String(roleName),
 	})
-	if err != nil && err.Error() != awserrors.ErrorIAMLimitExceeded {
+	if code, _ := awserrors.ResolveErrorCode(err); err != nil && code != awserrors.ErrorIAMLimitExceeded {
 		return "", fmt.Errorf("add role %q to instance profile %q: %w", roleName, profileName, err)
 	}
 	return profileARN, nil

spinifex/domains/elbv2/haproxy.go

@@ -46,7 +46,7 @@ backend {{.Name}}
 {{- if .Redirect}}
     http-request redirect {{if .Redirect.Scheme}}scheme {{.Redirect.Scheme}}{{else}}location {{.Redirect.Location}}{{end}} code {{.Redirect.Code}}
 {{- else if .FixedResponse}}
-    http-request return status {{.FixedResponse.StatusCode}}{{if .FixedResponse.ContentType}} content-type "{{.FixedResponse.ContentType}}"{{end}}{{if .FixedResponse.Body}} string "{{.FixedResponse.Body}}"{{end}}
+    http-request return status {{.FixedResponse.StatusCode}}{{if .FixedResponse.ContentType}} content-type {{squote .FixedResponse.ContentType}}{{end}}{{if .FixedResponse.Body}} string {{squote .FixedResponse.Body}}{{end}}
 {{- else}}
     balance roundrobin{{if .HealthCheckPath}}
     option httpchk
@@ -65,7 +65,14 @@ backend {{.Name}}
 {{end}}
 `
 
-var haproxyHTTPTmpl = template.Must(template.New("haproxy").Parse(haproxyConfigTemplate))
+var haproxyHTTPTmpl = template.Must(template.New("haproxy").Funcs(template.FuncMap{"squote": haproxySingleQuote}).Parse(haproxyConfigTemplate))
+
+// haproxySingleQuote renders s as one literal HAProxy word. HAProxy expands
+// environment variables inside double quotes but not single quotes, so tenant
+// text is single-quoted with each embedded quote closed, escaped and reopened.
+func haproxySingleQuote(s string) string {
+	return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
+}
 
 // HAProxyConfig holds the full configuration for rendering the HAProxy template.
 type HAProxyConfig struct {
@@ -566,17 +573,10 @@ func validHTTPStatusCode(s string) bool {
 	return true
 }
 
-// validFixedResponseBody rejects bytes that would terminate or inject into the HAProxy directive.
+// validFixedResponseBody rejects bytes that would end the single-line HAProxy
+// directive the single-quoted body is rendered into.
 func validFixedResponseBody(s string) bool {
-	if s == "" {
-		return false
-	}
-	for _, r := range s {
-		if r == '\n' || r == '\r' || r == '"' || r == '\\' || r == '#' {
-			return false
-		}
-	}
-	return true
+	return s != "" && !strings.ContainsAny(s, "\n\r\x00")
 }
 
 // buildRedirect translates a RedirectAction into an HAProxy redirect directive.

spinifex/domains/elbv2/haproxy_test.go

@@ -279,7 +279,7 @@ func TestGenerateHAProxyConfig_FixedResponseDefault(t *testing.T) {
 	// Synthetic default backend named off the listener ARN, returning the fixed response.
 	assert.Contains(t, config, "default_backend bkdefault_lst-616760baf6a3031c3")
 	assert.Contains(t, config, "backend bkdefault_lst-616760baf6a3031c3")
-	assert.Contains(t, config, `http-request return status 404 content-type "text/plain" string "no route"`)
+	assert.Contains(t, config, `http-request return status 404 content-type 'text/plain' string 'no route'`)
 	// Host-header rule still routes to the app backend.
 	assert.Contains(t, config, "use_backend bk_tg-app1 if")
 	assert.Contains(t, config, "10.42.0.9:8080")
@@ -309,7 +309,59 @@ func TestGenerateHAProxyConfig_FixedResponseRejectsUnsafeBody(t *testing.T) {
 
 	assert.Contains(t, config, "http-request return status 200")
 	assert.NotContains(t, config, "always_true")
-	assert.NotContains(t, config, `string "evil`)
+	assert.NotContains(t, config, `string 'evil`)
+}
+
+// TestGenerateHAProxyConfig_FixedResponseIsLiteral renders the content type and
+// body in single quotes, where HAProxy performs no environment expansion, so a
+// body referencing the lb-agent's environment is served verbatim.
+func TestGenerateHAProxyConfig_FixedResponseIsLiteral(t *testing.T) {
+	t.Parallel()
+	lb := &LoadBalancerRecord{LoadBalancerID: "lb-env"}
+	listeners := []*ListenerRecord{
+		{
+			ListenerArn: "arn:lst-env",
+			Port:        80,
+			DefaultActions: []ListenerAction{
+				{Type: ActionTypeFixedResponse, FixedResponse: &FixedResponseAction{
+					StatusCode:  "200",
+					ContentType: "text/plain",
+					MessageBody: `sk=${LB_SECRET_KEY} ak=$LB_ACCESS_KEY it's "q" \ #x`,
+				}},
+			},
+		},
+	}
+
+	config, _, err := GenerateHAProxyConfigWithCerts(lb, listeners, nil, nil, "0.0.0.0", nil)
+	require.NoError(t, err)
+
+	assert.Contains(t, config, `http-request return status 200 content-type 'text/plain' string 'sk=${LB_SECRET_KEY} ak=$LB_ACCESS_KEY it'\''s "q" \ #x'`+"\n")
+	assert.NotContains(t, config, `"${LB_SECRET_KEY}`)
+}
+
+// TestGenerateHAProxyConfig_FixedResponseKeepsStoredTab keeps a stored body with
+// a tab, which is literal inside single quotes and rendered before API checks existed.
+func TestGenerateHAProxyConfig_FixedResponseKeepsStoredTab(t *testing.T) {
+	t.Parallel()
+	lb := &LoadBalancerRecord{LoadBalancerID: "lb-tab"}
+	listeners := []*ListenerRecord{
+		{
+			ListenerArn: "arn:lst-tab",
+			Port:        80,
+			DefaultActions: []ListenerAction{
+				{Type: ActionTypeFixedResponse, FixedResponse: &FixedResponseAction{
+					StatusCode:  "503",
+					ContentType: "text/html",
+					MessageBody: "<pre>\tdown</pre>",
+				}},
+			},
+		},
+	}
+
+	config, _, err := GenerateHAProxyConfigWithCerts(lb, listeners, nil, nil, "0.0.0.0", nil)
+	require.NoError(t, err)
+
+	assert.Contains(t, config, "http-request return status 503 content-type 'text/html' string '<pre>\tdown</pre>'\n")
 }
 
 // TestGenerateHAProxyConfig_RedirectSchemeDefault renders the canonical

spinifex/domains/elbv2/service_impl.go

@@ -17,6 +17,8 @@ import (
 	"strings"
 	"sync"
 	"time"
+	"unicode"
+	"unicode/utf8"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -2701,8 +2703,9 @@ func listenerActionFromSDK(a *elbv2.Action) ListenerAction {
 	return action
 }
 
-// validateListenerAction enforces the per-type action contract. Redirect fields
-// are validated here so invalid input fails at the API, not at render time.
+// validateListenerAction enforces the per-type action contract. Redirect and
+// fixed-response fields are validated here so invalid input fails at the API,
+// not at render time.
 func validateListenerAction(a ListenerAction) error {
 	if a.Type == ActionTypeRedirect {
 		if a.Redirect == nil {
@@ -2710,6 +2713,39 @@ func validateListenerAction(a ListenerAction) error {
 		}
 		return validateRedirectAction(a.Redirect)
 	}
+	if a.Type == ActionTypeFixedResponse {
+		return validateFixedResponseAction(a.FixedResponse)
+	}
+	return nil
+}
+
+// fixedResponseContentTypes are the only ContentType values ELBv2 accepts.
+var fixedResponseContentTypes = map[string]bool{
+	"text/plain":             true,
+	"text/css":               true,
+	"text/html":              true,
+	"application/javascript": true,
+	"application/json":       true,
+}
+
+// maxFixedResponseBodyLen is ELBv2's MessageBody length limit in characters.
+const maxFixedResponseBodyLen = 1024
+
+// validateFixedResponseAction rejects a ContentType ELBv2 does not allow, or a
+// MessageBody that is too long or holds control characters.
+func validateFixedResponseAction(fr *FixedResponseAction) error {
+	if fr == nil {
+		return errors.New(awserrors.ErrorMissingParameter)
+	}
+	if fr.ContentType != "" && !fixedResponseContentTypes[fr.ContentType] {
+		return errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	if utf8.RuneCountInString(fr.MessageBody) > maxFixedResponseBodyLen {
+		return errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	if strings.ContainsFunc(fr.MessageBody, unicode.IsControl) {
+		return errors.New(awserrors.ErrorInvalidParameterValue)
+	}
 	return nil
 }
 

spinifex/domains/elbv2/service_impl_rules.go

@@ -591,8 +591,8 @@ func (s *ELBv2ServiceImpl) validateAndConvertRuleActions(ctx context.Context, in
 				return nil, err
 			}
 		case ActionTypeFixedResponse:
-			if action.FixedResponse == nil {
-				return nil, errors.New(awserrors.ErrorMissingParameter)
+			if err := validateFixedResponseAction(action.FixedResponse); err != nil {
+				return nil, err
 			}
 		default:
 			return nil, errors.New(awserrors.ErrorELBv2InvalidConfigurationRequest)

spinifex/domains/elbv2/service_impl_rules_test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/elbv2"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -660,3 +661,53 @@ func sortByPriority(rs []*RuleRecord) []*RuleRecord {
 	}
 	return out
 }
+
+func TestValidateListenerAction_FixedResponse(t *testing.T) {
+	t.Parallel()
+	fixed := func(ct, body string) ListenerAction {
+		return ListenerAction{Type: ActionTypeFixedResponse, FixedResponse: &FixedResponseAction{
+			StatusCode: "200", ContentType: ct, MessageBody: body,
+		}}
+	}
+
+	for _, ct := range []string{"", "text/plain", "text/css", "text/html", "application/javascript", "application/json"} {
+		assert.NoError(t, validateListenerAction(fixed(ct, "ok")), ct)
+	}
+	assert.NoError(t, validateListenerAction(fixed("text/plain", "")))
+	assert.NoError(t, validateListenerAction(fixed("text/plain", "sk=${LB_SECRET_KEY} it's \"q\" #x")))
+	assert.NoError(t, validateListenerAction(fixed("text/plain", strings.Repeat("é", 1024))))
+
+	bad := []ListenerAction{
+		fixed("text/$LB_SECRET_KEY", "ok"),
+		fixed("text/xml", "ok"),
+		fixed("TEXT/PLAIN", "ok"),
+		fixed("text/plain", strings.Repeat("a", 1025)),
+		fixed("text/plain", "line\nacl x always_true"),
+		fixed("text/plain", "cr\r"),
+		fixed("text/plain", "nul\x00"),
+	}
+	for _, a := range bad {
+		assert.EqualError(t, validateListenerAction(a), awserrors.ErrorInvalidParameterValue, "%+v", *a.FixedResponse)
+	}
+	assert.EqualError(t, validateListenerAction(ListenerAction{Type: ActionTypeFixedResponse}), awserrors.ErrorMissingParameter)
+}
+
+func TestCreateRule_RejectsFixedResponseContentType(t *testing.T) {
+	t.Parallel()
+	env := newRuleTestEnv(t, "cr-badfixed")
+
+	_, err := env.svc.CreateRule(context.Background(), &elbv2.CreateRuleInput{
+		ListenerArn: aws.String(env.listenerArn),
+		Priority:    aws.Int64(1),
+		Conditions:  []*elbv2.RuleCondition{{Field: aws.String("path-pattern"), Values: aws.StringSlice([]string{"/x"})}},
+		Actions: []*elbv2.Action{{
+			Type: aws.String("fixed-response"),
+			FixedResponseConfig: &elbv2.FixedResponseActionConfig{
+				StatusCode:  aws.String("200"),
+				ContentType: aws.String("text/$LB_SECRET_KEY"),
+				MessageBody: aws.String("ok"),
+			},
+		}},
+	}, testAccountID)
+	require.EqualError(t, err, awserrors.ErrorInvalidParameterValue)
+}

spinifex/domains/iam/awsapi/authz.go

@@ -48,6 +48,7 @@ var iamScopes = map[string]resourceScope{
 	"CreateUser":               createScope(arn.IAMUser, "UserName"),
 	"GetUser":                  existingScope(arn.IAMUser, "UserName"),
 	"DeleteUser":               existingScope(arn.IAMUser, "UserName"),
+	"UpdateUser":               existingScope(arn.IAMUser, "UserName"),
 	"CreateAccessKey":          existingScope(arn.IAMUser, "UserName"),
 	"ListAccessKeys":           existingScope(arn.IAMUser, "UserName"),
 	"DeleteAccessKey":          existingScope(arn.IAMUser, "UserName"),
@@ -284,7 +285,7 @@ func lookupResult(t lookupTarget, resource string, err error) (string, error) {
 		}
 		return resource, nil
 	}
-	if err.Error() == awserrors.ErrorIAMNoSuchEntity {
+	if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
 		// Authorization widens to "*", so a Deny scoped to the real target
 		// will not fire. Without this line that widening leaves no trace.
 		slog.Warn("IAM authz: target not found, authorizing account-wide",

spinifex/domains/iam/awsapi/authz_test.go

@@ -33,7 +33,7 @@ func (s *resolverService) CanonicalResourceARN(_ string, kind arn.IAMResourceTyp
 			return resource, nil
 		}
 	}
-	return "", errors.New(awserrors.ErrorIAMNoSuchEntity)
+	return "", awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "The %s with name %s cannot be found.", kind, name)
 }
 
 func (s *resolverService) LookupAccessKey(_ string) (*iamdomain.AccessKey, error) {

spinifex/domains/iam/awsapi/registration.go

@@ -96,6 +96,9 @@ var actions = map[string]action{
 	"DeleteUser": handler(func(accountID string, input *iam.DeleteUserInput, d Deps) (any, error) {
 		return DeleteUser(accountID, input, d.IAM)
 	}),
+	"UpdateUser": handler(func(accountID string, input *iam.UpdateUserInput, d Deps) (any, error) {
+		return UpdateUser(accountID, input, d.IAM)
+	}),
 	"CreateAccessKey": handler(func(accountID string, input *iam.CreateAccessKeyInput, d Deps) (any, error) {
 		return CreateAccessKey(accountID, input, d.IAM)
 	}),

spinifex/domains/iam/awsapi/registration_test.go

@@ -26,6 +26,7 @@ type regService struct {
 
 	getUserName   string
 	getUserCalls  int
+	updateUser    *iam.UpdateUserInput
 	addRoleCalls  int
 	canonicalARN  string
 	canonicalErr  error
@@ -41,6 +42,11 @@ func (s *regService) GetUser(_ string, in *iam.GetUserInput) (*iam.GetUserOutput
 	return &iam.GetUserOutput{}, nil
 }
 
+func (s *regService) UpdateUser(_ string, in *iam.UpdateUserInput) (*iam.UpdateUserOutput, error) {
+	s.updateUser = in
+	return &iam.UpdateUserOutput{}, nil
+}
+
 func (s *regService) AddRoleToInstanceProfile(_ string, _ *iam.AddRoleToInstanceProfileInput) (*iam.AddRoleToInstanceProfileOutput, error) {
 	s.addRoleCalls++
 	return &iam.AddRoleToInstanceProfileOutput{}, nil
@@ -268,6 +274,23 @@ func TestRegistration_AddRoleToInstanceProfilePassRole(t *testing.T) {
 	})
 }
 
+func TestRegistration_UpdateUser(t *testing.T) {
+	svc := &regService{}
+	w, calls, err := invoke(t, Deps{IAM: svc}, testAccountID, dispatch.Actor{}, nil, url.Values{
+		"Action": {"UpdateUser"}, "UserName": {"alice"}, "NewUserName": {"alicia"}, "NewPath": {"/eng/"},
+	})
+	require.NoError(t, err)
+	require.Len(t, calls, 1)
+	assert.Equal(t, []string{"arn:aws:iam::" + testAccountID + ":user/alice"}, calls[0].resources)
+	assert.Contains(t, w.Body.String(), "UpdateUserResponse")
+	require.NotNil(t, svc.updateUser)
+	assert.Equal(t, "alicia", *svc.updateUser.NewUserName)
+	assert.Equal(t, "/eng/", *svc.updateUser.NewPath)
+
+	_, _, err = invoke(t, Deps{IAM: svc}, testAccountID, dispatch.Actor{}, nil, url.Values{"Action": {"UpdateUser"}})
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorMissingParameter), "got %v", err)
+}
+
 func TestActionNames_SortedAndInventory(t *testing.T) {
 	names := ActionNames()
 	require.NotEmpty(t, names)

spinifex/domains/iam/awsapi/user.go

@@ -78,6 +78,15 @@ func DeleteUser(accountID string, input *iam.DeleteUserInput, svc iamdomain.IAMS
 	return svc.DeleteUser(accountID, input)
 }
 
+// UpdateUser implements the IAM UpdateUser action, renaming a user or changing its path. It
+// returns MissingParameter when UserName is absent, then calls the IAM service for accountID.
+func UpdateUser(accountID string, input *iam.UpdateUserInput, svc iamdomain.IAMService) (*iam.UpdateUserOutput, error) {
+	if input.UserName == nil || *input.UserName == "" {
+		return nil, errors.New(awserrors.ErrorMissingParameter)
+	}
+	return svc.UpdateUser(accountID, input)
+}
+
 // PutUserPolicy implements the IAM PutUserPolicy action, adding or replacing a user inline
 // policy. It returns MissingParameter when UserName, PolicyName or PolicyDocument is absent, then
 // calls the IAM service for accountID.

spinifex/domains/iam/error_messages_test.go

@@ -0,0 +1,186 @@
+package iam
+
+import (
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestEntityErrorMessages pins the per-case text AWS returns for NoSuchEntity,
+// EntityAlreadyExists, DeleteConflict and LimitExceeded, as recorded against
+// real IAM, so a client sees which entity is missing or in conflict.
+func TestEntityErrorMessages(t *testing.T) {
+	svc := setupTestIAMService(t)
+	acct := testAccountID
+
+	createTestUser(t, svc, "u1")
+	createTestRole(t, svc, "r1")
+	createTestRole(t, svc, "r2")
+	createTestGroup(t, svc, "g1")
+	createTestInstanceProfile(t, svc, "ip1")
+	policy := createTestPolicy(t, svc, "p1")
+	policyARN := aws.StringValue(policy.Arn)
+	missingPolicyARN := "arn:aws:iam::" + acct + ":policy/p1-nope"
+	_, err := svc.CreateOpenIDConnectProvider(acct, &iam.CreateOpenIDConnectProviderInput{
+		Url: aws.String(testIssuer), ClientIDList: aws.StringSlice([]string{"sts.amazonaws.com"}),
+	})
+	require.NoError(t, err)
+	missingOIDCARN := "arn:aws:iam::" + acct + ":oidc-provider/oidc.nope.example.com"
+
+	// Conflict fixtures: a user holding a key, a role with a managed policy, a
+	// role and profile linked to each other, and a group with a member.
+	createTestUser(t, svc, "u-key")
+	_, err = svc.CreateAccessKey(acct, &iam.CreateAccessKeyInput{UserName: aws.String("u-key")})
+	require.NoError(t, err)
+	createTestRole(t, svc, "r-pol")
+	_, err = svc.AttachRolePolicy(acct, &iam.AttachRolePolicyInput{RoleName: aws.String("r-pol"), PolicyArn: aws.String(policyARN)})
+	require.NoError(t, err)
+	createTestRole(t, svc, "r-ip")
+	createTestInstanceProfile(t, svc, "ip-r")
+	_, err = svc.AddRoleToInstanceProfile(acct, &iam.AddRoleToInstanceProfileInput{InstanceProfileName: aws.String("ip-r"), RoleName: aws.String("r-ip")})
+	require.NoError(t, err)
+	createTestGroup(t, svc, "g-mem")
+	_, err = svc.AddUserToGroup(acct, &iam.AddUserToGroupInput{GroupName: aws.String("g-mem"), UserName: aws.String("u1")})
+	require.NoError(t, err)
+
+	cases := []struct {
+		name     string
+		call     func() error
+		wantCode string
+		wantMsg  string
+	}{
+		{"GetUser missing", func() error {
+			_, err := svc.GetUser(acct, &iam.GetUserInput{UserName: aws.String("nope")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The user with name nope cannot be found."},
+		{"TagUser missing", func() error {
+			_, err := svc.TagUser(acct, &iam.TagUserInput{UserName: aws.String("nope"), Tags: []*iam.Tag{{Key: aws.String("k"), Value: aws.String("v")}}})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The user with name nope cannot be found."},
+		{"AddUserToGroup missing user", func() error {
+			_, err := svc.AddUserToGroup(acct, &iam.AddUserToGroupInput{GroupName: aws.String("g1"), UserName: aws.String("nope")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The user with name nope cannot be found."},
+		{"GetRole missing", func() error {
+			_, err := svc.GetRole(acct, &iam.GetRoleInput{RoleName: aws.String("nope")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The role with name nope cannot be found."},
+		{"TagRole missing", func() error {
+			_, err := svc.TagRole(acct, &iam.TagRoleInput{RoleName: aws.String("nope"), Tags: []*iam.Tag{{Key: aws.String("k"), Value: aws.String("v")}}})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The role with name nope cannot be found."},
+		{"DeleteGroup missing", func() error {
+			_, err := svc.DeleteGroup(acct, &iam.DeleteGroupInput{GroupName: aws.String("nope")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The group with name nope cannot be found."},
+		{"GetPolicy missing", func() error {
+			_, err := svc.GetPolicy(acct, &iam.GetPolicyInput{PolicyArn: aws.String(missingPolicyARN)})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "Policy " + missingPolicyARN + " was not found."},
+		{"DeletePolicy missing", func() error {
+			_, err := svc.DeletePolicy(acct, &iam.DeletePolicyInput{PolicyArn: aws.String(missingPolicyARN)})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "Policy " + missingPolicyARN + " was not found."},
+		{"DetachUserPolicy not attached", func() error {
+			_, err := svc.DetachUserPolicy(acct, &iam.DetachUserPolicyInput{UserName: aws.String("u1"), PolicyArn: aws.String(policyARN)})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "Policy " + policyARN + " was not found."},
+		{"AttachRolePolicy missing policy", func() error {
+			_, err := svc.AttachRolePolicy(acct, &iam.AttachRolePolicyInput{RoleName: aws.String("r1"), PolicyArn: aws.String(missingPolicyARN)})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "Policy " + missingPolicyARN + " does not exist or is not attachable."},
+		{"GetInstanceProfile missing", func() error {
+			_, err := svc.GetInstanceProfile(acct, &iam.GetInstanceProfileInput{InstanceProfileName: aws.String("nope")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "Instance Profile nope cannot be found."},
+		{"RemoveRoleFromInstanceProfile not in profile", func() error {
+			_, err := svc.RemoveRoleFromInstanceProfile(acct, &iam.RemoveRoleFromInstanceProfileInput{InstanceProfileName: aws.String("ip1"), RoleName: aws.String("r2")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "Role r2 in Instance Profile ip1 cannot be found."},
+		{"GetOpenIDConnectProvider missing", func() error {
+			_, err := svc.GetOpenIDConnectProvider(acct, &iam.GetOpenIDConnectProviderInput{OpenIDConnectProviderArn: aws.String(missingOIDCARN)})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "OpenIDConnect Provider not found for arn " + missingOIDCARN},
+		{"GetUserPolicy missing", func() error {
+			_, err := svc.GetUserPolicy(acct, &iam.GetUserPolicyInput{UserName: aws.String("u1"), PolicyName: aws.String("inl")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The user policy with name inl cannot be found."},
+		{"DeleteRolePolicy missing", func() error {
+			_, err := svc.DeleteRolePolicy(acct, &iam.DeleteRolePolicyInput{RoleName: aws.String("r1"), PolicyName: aws.String("rinl")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The role policy with name rinl cannot be found."},
+		{"DeleteAccessKey missing", func() error {
+			_, err := svc.DeleteAccessKey(acct, &iam.DeleteAccessKeyInput{UserName: aws.String("u1"), AccessKeyId: aws.String("AKIANOPE")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The Access Key with id AKIANOPE cannot be found."},
+
+		{"CreateUser duplicate", func() error {
+			_, err := svc.CreateUser(acct, &iam.CreateUserInput{UserName: aws.String("u1")})
+			return err
+		}, awserrors.ErrorIAMEntityAlreadyExists, "User with name u1 already exists."},
+		{"CreateRole duplicate", func() error {
+			_, err := svc.CreateRole(acct, &iam.CreateRoleInput{RoleName: aws.String("r1"), AssumeRolePolicyDocument: aws.String(validTrustPolicy())})
+			return err
+		}, awserrors.ErrorIAMEntityAlreadyExists, "Role with name r1 already exists."},
+		{"CreateGroup duplicate", func() error {
+			_, err := svc.CreateGroup(acct, &iam.CreateGroupInput{GroupName: aws.String("g1")})
+			return err
+		}, awserrors.ErrorIAMEntityAlreadyExists, "Group with name g1 already exists."},
+		{"CreatePolicy duplicate", func() error {
+			_, err := svc.CreatePolicy(acct, &iam.CreatePolicyInput{PolicyName: aws.String("p1"), PolicyDocument: aws.String(validPolicyDocument())})
+			return err
+		}, awserrors.ErrorIAMEntityAlreadyExists, "A policy called p1 already exists. Duplicate names are not allowed."},
+		{"CreateInstanceProfile duplicate", func() error {
+			_, err := svc.CreateInstanceProfile(acct, &iam.CreateInstanceProfileInput{InstanceProfileName: aws.String("ip1")})
+			return err
+		}, awserrors.ErrorIAMEntityAlreadyExists, "Instance Profile ip1 already exists."},
+		{"CreateOpenIDConnectProvider duplicate", func() error {
+			_, err := svc.CreateOpenIDConnectProvider(acct, &iam.CreateOpenIDConnectProviderInput{
+				Url: aws.String(testIssuer), ClientIDList: aws.StringSlice([]string{"sts.amazonaws.com"}),
+			})
+			return err
+		}, awserrors.ErrorIAMEntityAlreadyExists, "Provider with url " + testIssuer + " already exists."},
+
+		{"DeleteUser with access key", func() error {
+			_, err := svc.DeleteUser(acct, &iam.DeleteUserInput{UserName: aws.String("u-key")})
+			return err
+		}, awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must delete access keys first."},
+		{"DeleteRole with attached policy", func() error {
+			_, err := svc.DeleteRole(acct, &iam.DeleteRoleInput{RoleName: aws.String("r-pol")})
+			return err
+		}, awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must detach all policies first."},
+		{"DeleteRole in instance profile", func() error {
+			_, err := svc.DeleteRole(acct, &iam.DeleteRoleInput{RoleName: aws.String("r-ip")})
+			return err
+		}, awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must remove roles from instance profile first."},
+		{"DeleteInstanceProfile with role", func() error {
+			_, err := svc.DeleteInstanceProfile(acct, &iam.DeleteInstanceProfileInput{InstanceProfileName: aws.String("ip-r")})
+			return err
+		}, awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must remove roles from instance profile first."},
+		{"DeleteGroup with member", func() error {
+			_, err := svc.DeleteGroup(acct, &iam.DeleteGroupInput{GroupName: aws.String("g-mem")})
+			return err
+		}, awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must remove users from group first."},
+		{"DeletePolicy attached", func() error {
+			_, err := svc.DeletePolicy(acct, &iam.DeletePolicyInput{PolicyArn: aws.String(policyARN)})
+			return err
+		}, awserrors.ErrorIAMDeleteConflict, "Cannot delete a policy attached to entities."},
+
+		{"AddRoleToInstanceProfile second role", func() error {
+			_, err := svc.AddRoleToInstanceProfile(acct, &iam.AddRoleToInstanceProfileInput{InstanceProfileName: aws.String("ip-r"), RoleName: aws.String("r2")})
+			return err
+		}, awserrors.ErrorIAMLimitExceeded, "Cannot exceed quota for InstanceSessionsPerInstanceProfile: 1"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			code, msg, ok := awserrors.ResolveErrorDetail(tc.call())
+			require.True(t, ok, "error carries no AWS code")
+			assert.Equal(t, tc.wantCode, code)
+			assert.Equal(t, tc.wantMsg, msg)
+		})
+	}
+}

spinifex/domains/iam/groups.go

@@ -65,7 +65,7 @@ func (s *IAMServiceImpl) CreateGroup(accountID string, input *iam.CreateGroupInp
 	kvKey := accountID + "." + groupName
 	if _, err := s.groupsBucket.Create(ctx, kvKey, data); err != nil {
 		if errors.Is(err, jetstream.ErrKeyExists) {
-			return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists)
+			return nil, awserrors.Errorf(awserrors.ErrorIAMEntityAlreadyExists, "Group with name %s already exists.", groupName)
 		}
 		return nil, fmt.Errorf("store group: %w", err)
 	}
@@ -175,11 +175,11 @@ func (s *IAMServiceImpl) DeleteGroup(accountID string, input *iam.DeleteGroupInp
 	}
 
 	if len(group.AttachedPolicies) > 0 {
-		return nil, errors.New(awserrors.ErrorIAMDeleteConflict)
+		return nil, errDetachPoliciesFirst
 	}
 
 	if len(group.InlinePolicies) > 0 {
-		return nil, errors.New(awserrors.ErrorIAMDeleteConflict)
+		return nil, errDeletePoliciesFirst
 	}
 
 	members, err := s.findGroupMembers(ctx, accountID, groupName)
@@ -187,7 +187,7 @@ func (s *IAMServiceImpl) DeleteGroup(accountID string, input *iam.DeleteGroupInp
 		return nil, fmt.Errorf("check group members: %w", err)
 	}
 	if len(members) > 0 {
-		return nil, errors.New(awserrors.ErrorIAMDeleteConflict)
+		return nil, awserrors.Errorf(awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must remove users from group first.")
 	}
 
 	if err := s.groupsBucket.Delete(ctx, accountID+"."+groupName); err != nil {
@@ -234,19 +234,28 @@ func (s *IAMServiceImpl) RemoveUserFromGroup(accountID string, input *iam.Remove
 	groupName := *input.GroupName
 	userName := *input.UserName
 
-	// Operate purely on the membership reference so a dangling pointer to an
-	// already-deleted group is still cleanable; never fetch the group here.
+	// Remove the membership reference without fetching the group, so a dangling
+	// pointer to an already-deleted group is still cleanable.
+	removed := false
 	err := s.updateUserCAS(ctx, accountID, userName, func(u *User) (bool, error) {
 		idx := slices.Index(u.Groups, groupName)
 		if idx < 0 {
-			return false, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return false, nil
 		}
 		u.Groups = slices.Delete(u.Groups, idx, idx+1)
+		removed = true
 		return true, nil
 	})
 	if err != nil {
 		return nil, err
 	}
+	// AWS answers success for a non-member when the group exists.
+	if !removed {
+		if _, err := s.getGroup(ctx, accountID, groupName); err != nil {
+			return nil, err
+		}
+		return &iam.RemoveUserFromGroupOutput{}, nil
+	}
 
 	slog.Info("IAM user removed from group", "accountID", accountID, "userName", userName, "groupName", groupName)
 	return &iam.RemoveUserFromGroupOutput{}, nil
@@ -265,7 +274,7 @@ func (s *IAMServiceImpl) ListGroupsForUser(accountID string, input *iam.ListGrou
 	for _, name := range user.Groups {
 		group, err := s.getGroup(ctx, accountID, name)
 		if err != nil {
-			if err.Error() == awserrors.ErrorIAMNoSuchEntity {
+			if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
 				// A dangling pointer to a vanished group is inert; skip it.
 				// Mirrors GetUserPolicies — transient/corrupt errors fail closed.
 				slog.Warn("ListGroupsForUser: member references missing group; skipping",
@@ -294,7 +303,7 @@ func (s *IAMServiceImpl) AttachGroupPolicy(accountID string, input *iam.AttachGr
 
 	// AWS-managed ARNs are stored opaquely (like roles); customer-managed ARNs must exist.
 	if !iamarn.IsAWSManagedPolicyARN(policyARN) {
-		if _, err := s.getPolicyByARN(ctx, accountID, policyARN); err != nil {
+		if err := s.attachablePolicy(ctx, accountID, policyARN); err != nil {
 			return nil, err
 		}
 	}
@@ -322,7 +331,7 @@ func (s *IAMServiceImpl) DetachGroupPolicy(accountID string, input *iam.DetachGr
 	err := s.updateGroupCAS(ctx, accountID, groupName, func(g *Group) (bool, error) {
 		idx := slices.Index(g.AttachedPolicies, policyARN)
 		if idx < 0 {
-			return false, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return false, policyNotFound(policyARN)
 		}
 		g.AttachedPolicies = slices.Delete(g.AttachedPolicies, idx, idx+1)
 		return true, nil
@@ -408,7 +417,7 @@ func (s *IAMServiceImpl) GetGroupPolicy(accountID string, input *iam.GetGroupPol
 
 	doc, ok := group.InlinePolicies[policyName]
 	if !ok {
-		return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+		return nil, noSuchEntity("group policy", policyName)
 	}
 
 	return &iam.GetGroupPolicyOutput{
@@ -427,7 +436,7 @@ func (s *IAMServiceImpl) DeleteGroupPolicy(accountID string, input *iam.DeleteGr
 
 	err := s.updateGroupCAS(ctx, accountID, groupName, func(g *Group) (bool, error) {
 		if _, ok := g.InlinePolicies[policyName]; !ok {
-			return false, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return false, noSuchEntity("group policy", policyName)
 		}
 		delete(g.InlinePolicies, policyName)
 		return true, nil
@@ -470,7 +479,7 @@ func (s *IAMServiceImpl) getGroup(ctx context.Context, accountID, groupName stri
 	entry, err := s.groupsBucket.Get(ctx, accountID+"."+groupName)
 	if err != nil {
 		if errors.Is(err, jetstream.ErrKeyNotFound) {
-			return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return nil, noSuchEntity("group", groupName)
 		}
 		return nil, fmt.Errorf("get group: %w", err)
 	}

spinifex/domains/iam/groups_test.go

@@ -409,6 +409,18 @@ func TestRemoveUserFromGroup_NotMember(t *testing.T) {
 		GroupName: aws.String("team"),
 		UserName:  aws.String("erin"),
 	})
+	require.NoError(t, err)
+}
+
+func TestRemoveUserFromGroup_GroupNotFound(t *testing.T) {
+	t.Parallel()
+	svc := setupTestIAMService(t)
+	createTestUser(t, svc, "erin")
+
+	_, err := svc.RemoveUserFromGroup(testAccountID, &iam.RemoveUserFromGroupInput{
+		GroupName: aws.String("ghost"),
+		UserName:  aws.String("erin"),
+	})
 	assert.Error(t, err)
 	assert.Contains(t, err.Error(), awserrors.ErrorIAMNoSuchEntity)
 }

spinifex/domains/iam/instance_profiles.go

@@ -66,7 +66,7 @@ func (s *IAMServiceImpl) CreateInstanceProfile(accountID string, input *iam.Crea
 
 	if _, err := s.instanceProfilesBucket.Create(ctx, accountID+"."+profileName, data); err != nil {
 		if errors.Is(err, jetstream.ErrKeyExists) {
-			return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists)
+			return nil, awserrors.Errorf(awserrors.ErrorIAMEntityAlreadyExists, "Instance Profile %s already exists.", profileName)
 		}
 		return nil, fmt.Errorf("store instance profile: %w", err)
 	}
@@ -172,7 +172,7 @@ func (s *IAMServiceImpl) DeleteInstanceProfile(accountID string, input *iam.Dele
 	_, err := kvutil.DeleteIf(ctx, s.instanceProfilesBucket, accountID+"."+profileName, s.instanceProfileCASConfig(accountID, profileName),
 		func(p *InstanceProfile) error {
 			if p.RoleName != "" {
-				return errors.New(awserrors.ErrorIAMDeleteConflict)
+				return awserrors.Errorf(awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must remove roles from instance profile first.")
 			}
 			return nil
 		})
@@ -195,7 +195,7 @@ func (s *IAMServiceImpl) AddRoleToInstanceProfile(accountID string, input *iam.A
 
 	err := s.updateInstanceProfileCAS(ctx, accountID, profileName, func(p *InstanceProfile) (bool, error) {
 		if p.RoleName != "" {
-			return false, errors.New(awserrors.ErrorIAMLimitExceeded)
+			return false, awserrors.Errorf(awserrors.ErrorIAMLimitExceeded, "Cannot exceed quota for InstanceSessionsPerInstanceProfile: 1")
 		}
 		p.RoleName = roleName
 		return true, nil
@@ -216,7 +216,7 @@ func (s *IAMServiceImpl) RemoveRoleFromInstanceProfile(accountID string, input *
 
 	err := s.updateInstanceProfileCAS(ctx, accountID, profileName, func(p *InstanceProfile) (bool, error) {
 		if p.RoleName == "" || p.RoleName != roleName {
-			return false, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return false, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Role %s in Instance Profile %s cannot be found.", roleName, profileName)
 		}
 		p.RoleName = ""
 		return true, nil
@@ -357,11 +357,16 @@ func (s *IAMServiceImpl) ListInstanceProfileTags(accountID string, input *iam.Li
 	}, nil
 }
 
+// instanceProfileNotFound is NoSuchEntity with AWS's message for a missing instance profile.
+func instanceProfileNotFound(profileName string) error {
+	return awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", profileName)
+}
+
 func (s *IAMServiceImpl) getInstanceProfile(ctx context.Context, accountID, profileName string) (*InstanceProfile, error) {
 	entry, err := s.instanceProfilesBucket.Get(ctx, accountID+"."+profileName)
 	if err != nil {
 		if errors.Is(err, jetstream.ErrKeyNotFound) {
-			return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return nil, instanceProfileNotFound(profileName)
 		}
 		return nil, fmt.Errorf("get instance profile: %w", err)
 	}
@@ -385,7 +390,7 @@ func (s *IAMServiceImpl) updateInstanceProfileCAS(ctx context.Context, accountID
 func (s *IAMServiceImpl) instanceProfileCASConfig(accountID, profileName string) kvutil.CASConfig {
 	return kvutil.CASConfig{
 		Attempts: instanceProfileCASMaxRetries,
-		NotFound: errors.New(awserrors.ErrorIAMNoSuchEntity),
+		NotFound: instanceProfileNotFound(profileName),
 		Exhausted: func(string, int) error {
 			slog.Warn("IAM instance profile CAS exhausted retries",
 				"accountID", accountID, "instanceProfileName", profileName, "attempts", instanceProfileCASMaxRetries)

spinifex/domains/iam/instance_profiles_test.go

@@ -778,7 +778,7 @@ func TestResolveInstanceProfile_NonCanonicalARNRejected(t *testing.T) {
 		_, err := svc.ResolveInstanceProfile(testAccountID, arn)
 		require.Error(t, err, arn)
 		// The EC2 gateway maps this to InvalidIamInstanceProfile.NotFound by exact match.
-		assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, err.Error(), arn)
+		assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, awserrors.ValidErrorCodeFromError(err), arn)
 	}
 }
 

spinifex/domains/iam/mock/mock.go

@@ -90,7 +90,7 @@ func (e *SystemInstanceRoleEnsurer) GetRole(_ string, in *iam.GetRoleInput) (*ia
 
 	r, ok := e.Roles[aws.StringValue(in.RoleName)]
 	if !ok {
-		return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "The role with name %s cannot be found.", aws.StringValue(in.RoleName))
 	}
 	return &iam.GetRoleOutput{Role: copyRole(r)}, nil
 }
@@ -104,7 +104,7 @@ func (e *SystemInstanceRoleEnsurer) CreateRole(accountID string, in *iam.CreateR
 	e.LastRoleAcct = accountID
 	name := aws.StringValue(in.RoleName)
 	if _, ok := e.Roles[name]; ok {
-		return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists)
+		return nil, awserrors.Errorf(awserrors.ErrorIAMEntityAlreadyExists, "Role with name %s already exists.", name)
 	}
 	r := &iam.Role{
 		RoleName: aws.String(name),
@@ -132,7 +132,7 @@ func (e *SystemInstanceRoleEnsurer) GetInstanceProfile(_ string, in *iam.GetInst
 
 	p, ok := e.Profiles[aws.StringValue(in.InstanceProfileName)]
 	if !ok {
-		return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", aws.StringValue(in.InstanceProfileName))
 	}
 	return &iam.GetInstanceProfileOutput{InstanceProfile: copyProfile(p)}, nil
 }
@@ -148,7 +148,7 @@ func (e *SystemInstanceRoleEnsurer) CreateInstanceProfile(accountID string, in *
 	}
 	name := aws.StringValue(in.InstanceProfileName)
 	if _, ok := e.Profiles[name]; ok {
-		return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists)
+		return nil, awserrors.Errorf(awserrors.ErrorIAMEntityAlreadyExists, "Instance Profile %s already exists.", name)
 	}
 	arn := "arn:aws:iam::" + accountID + ":instance-profile/" + name
 	if e.EmptyInstanceProfileARN {
@@ -176,7 +176,7 @@ func (e *SystemInstanceRoleEnsurer) AddRoleToInstanceProfile(_ string, in *iam.A
 		return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
 	}
 	if len(p.Roles) > 0 {
-		return nil, errors.New(awserrors.ErrorIAMLimitExceeded)
+		return nil, awserrors.Errorf(awserrors.ErrorIAMLimitExceeded, "Cannot exceed quota for InstanceSessionsPerInstanceProfile: 1")
 	}
 	p.Roles = []*iam.Role{{RoleName: in.RoleName}}
 	return &iam.AddRoleToInstanceProfileOutput{}, nil

spinifex/domains/iam/mock/mock_test.go

@@ -13,7 +13,7 @@ import (
 func TestGetRole_MissingReturnsNoSuchEntity(t *testing.T) {
 	e := mock.New()
 	_, err := e.GetRole("acct", &iam.GetRoleInput{RoleName: aws.String("r1")})
-	if err == nil || err.Error() != awserrors.ErrorIAMNoSuchEntity {
+	if err == nil || awserrors.ValidErrorCodeFromError(err) != awserrors.ErrorIAMNoSuchEntity {
 		t.Fatalf("err = %v, want ErrorIAMNoSuchEntity", err)
 	}
 }
@@ -24,7 +24,7 @@ func TestCreateRole_DuplicateReturnsEntityAlreadyExists(t *testing.T) {
 	if _, err := e.CreateRole("acct", in); err != nil {
 		t.Fatalf("first CreateRole: %v", err)
 	}
-	if _, err := e.CreateRole("acct", in); err == nil || err.Error() != awserrors.ErrorIAMEntityAlreadyExists {
+	if _, err := e.CreateRole("acct", in); err == nil || awserrors.ValidErrorCodeFromError(err) != awserrors.ErrorIAMEntityAlreadyExists {
 		t.Fatalf("second CreateRole err = %v, want EntityAlreadyExists", err)
 	}
 	if e.CreateRoleCalls != 2 {
@@ -48,7 +48,7 @@ func TestAddRoleToInstanceProfile_SecondAttachIsLimitExceeded(t *testing.T) {
 	if _, err := e.AddRoleToInstanceProfile("acct", in); err != nil {
 		t.Fatalf("first AddRoleToInstanceProfile: %v", err)
 	}
-	if _, err := e.AddRoleToInstanceProfile("acct", in); err == nil || err.Error() != awserrors.ErrorIAMLimitExceeded {
+	if _, err := e.AddRoleToInstanceProfile("acct", in); err == nil || awserrors.ValidErrorCodeFromError(err) != awserrors.ErrorIAMLimitExceeded {
 		t.Fatalf("second AddRoleToInstanceProfile err = %v, want LimitExceeded", err)
 	}
 }

spinifex/domains/iam/oidc_providers.go

@@ -178,7 +178,7 @@ func (s *IAMServiceImpl) CreateOpenIDConnectProvider(accountID string, input *ia
 
 	if _, err := kv.Create(ctx, OIDCProviderKey(issuer), data); err != nil {
 		if errors.Is(err, jetstream.ErrKeyExists) {
-			return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists)
+			return nil, awserrors.Errorf(awserrors.ErrorIAMEntityAlreadyExists, "Provider with url %s already exists.", issuer)
 		}
 		return nil, fmt.Errorf("store OIDC provider: %w", err)
 	}
@@ -268,7 +268,7 @@ func (s *IAMServiceImpl) DeleteOpenIDConnectProvider(accountID string, input *ia
 	kv, err := s.js.KeyValue(ctx, IAMAccountBucketName(accountID))
 	if err != nil {
 		if errors.Is(err, jetstream.ErrBucketNotFound) {
-			return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return nil, oidcProviderNotFound(arn)
 		}
 		return nil, fmt.Errorf("open IAM account bucket: %w", err)
 	}
@@ -276,7 +276,7 @@ func (s *IAMServiceImpl) DeleteOpenIDConnectProvider(accountID string, input *ia
 	key := OIDCProviderKey(issuer)
 	if _, err := kv.Get(ctx, key); err != nil {
 		if errors.Is(err, jetstream.ErrKeyNotFound) {
-			return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return nil, oidcProviderNotFound(arn)
 		}
 		return nil, fmt.Errorf("get OIDC provider: %w", err)
 	}
@@ -356,7 +356,7 @@ func (s *IAMServiceImpl) updateOIDCProviderCAS(ctx context.Context, accountID, a
 	kv, err := s.js.KeyValue(ctx, IAMAccountBucketName(accountID))
 	if err != nil {
 		if errors.Is(err, jetstream.ErrBucketNotFound) {
-			return errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return oidcProviderNotFound(arn)
 		}
 		return fmt.Errorf("open IAM account bucket: %w", err)
 	}
@@ -373,14 +373,14 @@ func (s *IAMServiceImpl) getOIDCProvider(ctx context.Context, accountID, arn str
 	kv, err := s.js.KeyValue(ctx, IAMAccountBucketName(accountID))
 	if err != nil {
 		if errors.Is(err, jetstream.ErrBucketNotFound) {
-			return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return nil, oidcProviderNotFound(arn)
 		}
 		return nil, fmt.Errorf("open IAM account bucket: %w", err)
 	}
 	entry, err := kv.Get(ctx, OIDCProviderKey(issuer))
 	if err != nil {
 		if errors.Is(err, jetstream.ErrKeyNotFound) {
-			return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return nil, oidcProviderNotFound(arn)
 		}
 		return nil, fmt.Errorf("get OIDC provider: %w", err)
 	}

spinifex/domains/iam/oidc_providers_test.go

@@ -50,7 +50,7 @@ func TestCreateOpenIDConnectProvider_Idempotent(t *testing.T) {
 		t.Fatalf("first create: %v", err)
 	}
 	_, err := svc.CreateOpenIDConnectProvider(acct, in)
-	if err == nil || err.Error() != awserrors.ErrorIAMEntityAlreadyExists {
+	if err == nil || awserrors.ValidErrorCodeFromError(err) != awserrors.ErrorIAMEntityAlreadyExists {
 		t.Fatalf("duplicate create err = %v, want EntityAlreadyExists", err)
 	}
 }
@@ -132,7 +132,7 @@ func TestGetOpenIDConnectProvider_NotFound(t *testing.T) {
 	_, err := svc.GetOpenIDConnectProvider("000000000001", &iam.GetOpenIDConnectProviderInput{
 		OpenIDConnectProviderArn: aws.String(OIDCProviderARN("000000000001", "10.0.0.1:9999/oidc/eks/x/y/z")),
 	})
-	if err == nil || err.Error() != awserrors.ErrorIAMNoSuchEntity {
+	if err == nil || awserrors.ValidErrorCodeFromError(err) != awserrors.ErrorIAMNoSuchEntity {
 		t.Fatalf("err = %v, want NoSuchEntity", err)
 	}
 }
@@ -195,7 +195,7 @@ func TestDeleteOpenIDConnectProvider(t *testing.T) {
 	_, err = svc.DeleteOpenIDConnectProvider(acct, &iam.DeleteOpenIDConnectProviderInput{
 		OpenIDConnectProviderArn: created.OpenIDConnectProviderArn,
 	})
-	if err == nil || err.Error() != awserrors.ErrorIAMNoSuchEntity {
+	if err == nil || awserrors.ValidErrorCodeFromError(err) != awserrors.ErrorIAMNoSuchEntity {
 		t.Fatalf("second delete err = %v, want NoSuchEntity", err)
 	}
 }

spinifex/domains/iam/policy_versions.go

@@ -111,11 +111,11 @@ func policyCASTarget(accountID, policyARN string) (string, kvutil.CASConfig, err
 	if err != nil {
 		slog.Debug("policyCASTarget: unparseable policy ARN",
 			"accountID", accountID, "policyArn", policyARN, "err", err)
-		return "", kvutil.CASConfig{}, errors.New(awserrors.ErrorIAMNoSuchEntity)
+		return "", kvutil.CASConfig{}, policyNotFound(policyARN)
 	}
 	return accountID + "." + policyName, kvutil.CASConfig{
 		Attempts: policyCASMaxRetries,
-		NotFound: errors.New(awserrors.ErrorIAMNoSuchEntity),
+		NotFound: policyNotFound(policyARN),
 		Exhausted: func(string, int) error {
 			slog.Error("IAM policy CAS retries exhausted under contention",
 				"accountID", accountID, "policyName", policyName, "attempts", policyCASMaxRetries)

spinifex/domains/iam/records_cas_test.go

@@ -224,11 +224,11 @@ func TestTagUser_DoesNotRecreateConcurrentlyDeletedUser(t *testing.T) {
 	close(gate.release)
 	err = <-done
 	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, err.Error())
+	assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, awserrors.ValidErrorCodeFromError(err))
 
 	_, err = svc.getUser(t.Context(), testAccountID, *user.UserName)
 	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, err.Error(), "the deleted user must stay deleted")
+	assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, awserrors.ValidErrorCodeFromError(err), "the deleted user must stay deleted")
 }
 
 // Every group writer that read the record before a DetachGroupPolicy and writes
@@ -376,7 +376,7 @@ func TestOIDCProviderTagWriters_DoNotRecreateDeletedProvider(t *testing.T) {
 			close(gate.release)
 			err = <-done
 			require.Error(t, err)
-			assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, err.Error())
+			assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, awserrors.ValidErrorCodeFromError(err))
 
 			_, err = kv.Get(t.Context(), OIDCProviderKey(testIssuer))
 			assert.ErrorIs(t, err, jetstream.ErrKeyNotFound, "the deleted provider must stay deleted")

spinifex/domains/iam/role_resolver.go

@@ -33,7 +33,7 @@ func (r *RoleResolver) ResolveRole(accountID, roleARN string) (passrole.Role, er
 	}
 	role, err := r.svc.getRole(context.Background(), accountID, name)
 	if err != nil {
-		if err.Error() == awserrors.ErrorIAMNoSuchEntity {
+		if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
 			return passrole.Role{}, passrole.ErrNotFound
 		}
 		return passrole.Role{}, err

spinifex/domains/iam/roles.go

@@ -106,7 +106,7 @@ func (s *IAMServiceImpl) CreateRole(accountID string, input *iam.CreateRoleInput
 	kvKey := accountID + "." + roleName
 	if _, err := s.rolesBucket.Create(ctx, kvKey, data); err != nil {
 		if errors.Is(err, jetstream.ErrKeyExists) {
-			return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists)
+			return nil, awserrors.Errorf(awserrors.ErrorIAMEntityAlreadyExists, "Role with name %s already exists.", roleName)
 		}
 		return nil, fmt.Errorf("store role: %w", err)
 	}
@@ -195,11 +195,11 @@ func (s *IAMServiceImpl) DeleteRole(accountID string, input *iam.DeleteRoleInput
 	}
 
 	if len(role.AttachedPolicies) > 0 {
-		return nil, errors.New(awserrors.ErrorIAMDeleteConflict)
+		return nil, errDetachPoliciesFirst
 	}
 
 	if len(role.InlinePolicies) > 0 {
-		return nil, errors.New(awserrors.ErrorIAMDeleteConflict)
+		return nil, errDeletePoliciesFirst
 	}
 
 	profiles, err := s.findInstanceProfilesForRole(ctx, accountID, roleName)
@@ -207,7 +207,7 @@ func (s *IAMServiceImpl) DeleteRole(accountID string, input *iam.DeleteRoleInput
 		return nil, fmt.Errorf("check role instance profiles: %w", err)
 	}
 	if len(profiles) > 0 {
-		return nil, errors.New(awserrors.ErrorIAMDeleteConflict)
+		return nil, awserrors.Errorf(awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must remove roles from instance profile first.")
 	}
 
 	if err := s.rolesBucket.Delete(ctx, accountID+"."+roleName); err != nil {
@@ -291,7 +291,7 @@ func (s *IAMServiceImpl) UpdateRoleDescription(accountID string, input *iam.Upda
 	}
 
 	slog.Info("IAM role description updated", "accountID", accountID, "roleName", roleName)
-	return &iam.UpdateRoleDescriptionOutput{Role: gotRoleToSDK(role)}, nil
+	return &iam.UpdateRoleDescriptionOutput{Role: describedRoleToSDK(role)}, nil
 }
 
 func (s *IAMServiceImpl) UpdateAssumeRolePolicy(accountID string, input *iam.UpdateAssumeRolePolicyInput) (*iam.UpdateAssumeRolePolicyOutput, error) {
@@ -326,7 +326,7 @@ func (s *IAMServiceImpl) AttachRolePolicy(accountID string, input *iam.AttachRol
 	// so ListAttachedRolePolicies / DescribeNodegroup round-trip instead of
 	// failing NoSuchEntity. Customer-managed ARNs must still exist.
 	if !iamarn.IsAWSManagedPolicyARN(policyARN) {
-		if _, err := s.getPolicyByARN(ctx, accountID, policyARN); err != nil {
+		if err := s.attachablePolicy(ctx, accountID, policyARN); err != nil {
 			return nil, err
 		}
 	}
@@ -354,7 +354,7 @@ func (s *IAMServiceImpl) DetachRolePolicy(accountID string, input *iam.DetachRol
 	err := s.updateRoleCAS(ctx, accountID, roleName, func(role *Role) (bool, error) {
 		idx := slices.Index(role.AttachedPolicies, policyARN)
 		if idx < 0 {
-			return false, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return false, policyNotFound(policyARN)
 		}
 		role.AttachedPolicies = slices.Delete(role.AttachedPolicies, idx, idx+1)
 		return true, nil
@@ -444,7 +444,7 @@ func (s *IAMServiceImpl) GetRolePolicy(accountID string, input *iam.GetRolePolic
 
 	doc, ok := role.InlinePolicies[policyName]
 	if !ok {
-		return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+		return nil, noSuchEntity("role policy", policyName)
 	}
 
 	return &iam.GetRolePolicyOutput{
@@ -463,7 +463,7 @@ func (s *IAMServiceImpl) DeleteRolePolicy(accountID string, input *iam.DeleteRol
 
 	err := s.updateRoleCAS(ctx, accountID, roleName, func(role *Role) (bool, error) {
 		if _, ok := role.InlinePolicies[policyName]; !ok {
-			return false, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return false, noSuchEntity("role policy", policyName)
 		}
 		delete(role.InlinePolicies, policyName)
 		return true, nil
@@ -600,7 +600,7 @@ func (s *IAMServiceImpl) getRole(ctx context.Context, accountID, roleName string
 	entry, err := s.rolesBucket.Get(ctx, accountID+"."+roleName)
 	if err != nil {
 		if errors.Is(err, jetstream.ErrKeyNotFound) {
-			return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return nil, noSuchEntity("role", roleName)
 		}
 		return nil, fmt.Errorf("get role: %w", err)
 	}
@@ -622,7 +622,7 @@ func (s *IAMServiceImpl) updateRoleCAS(ctx context.Context, accountID, roleName
 	defer s.recordLocks.lock("role/" + accountID + "." + roleName)()
 	_, err := kvutil.Update(ctx, s.rolesBucket, accountID+"."+roleName, kvutil.CASConfig{
 		Attempts: roleCASMaxRetries,
-		NotFound: errors.New(awserrors.ErrorIAMNoSuchEntity),
+		NotFound: noSuchEntity("role", roleName),
 		Exhausted: func(string, int) error {
 			// Only a CAS conflict reaches here, so the role is contended rather than
 			// broken; say so, or the caller's InternalError has no cause anywhere.
@@ -698,6 +698,16 @@ func createdRoleToSDK(r *Role) *iam.Role {
 	return out
 }
 
+// describedRoleToSDK is UpdateRoleDescription's shape, which carries no
+// MaxSessionDuration, Tags or RoleLastUsed.
+func describedRoleToSDK(r *Role) *iam.Role {
+	out := roleToSDK(r)
+	if r.Description != "" {
+		out.Description = aws.String(r.Description)
+	}
+	return out
+}
+
 // listedRoleToSDK is ListRoles' shape, which carries no Tags or RoleLastUsed.
 func listedRoleToSDK(r *Role) *iam.Role {
 	out := roleToSDK(r)

spinifex/domains/iam/roles_test.go

@@ -561,7 +561,13 @@ func TestUpdateRole_NotFound(t *testing.T) {
 func TestUpdateRoleDescription(t *testing.T) {
 	t.Parallel()
 	svc := setupTestIAMService(t)
-	createTestRole(t, svc, "desc-role")
+	_, err := svc.CreateRole(testAccountID, &iam.CreateRoleInput{
+		RoleName:                 aws.String("desc-role"),
+		AssumeRolePolicyDocument: aws.String(validTrustPolicy()),
+		MaxSessionDuration:       aws.Int64(7200),
+		Tags:                     []*iam.Tag{{Key: aws.String("team"), Value: aws.String("infra")}},
+	})
+	require.NoError(t, err)
 
 	out, err := svc.UpdateRoleDescription(testAccountID, &iam.UpdateRoleDescriptionInput{
 		RoleName:    aws.String("desc-role"),
@@ -571,7 +577,15 @@ func TestUpdateRoleDescription(t *testing.T) {
 	got, err := svc.GetRole(testAccountID, &iam.GetRoleInput{RoleName: aws.String("desc-role")})
 	require.NoError(t, err)
 	assert.Equal(t, "new description", *got.Role.Description)
-	assert.Equal(t, got.Role, out.Role, "output carries the role in the GetRole shape")
+	assert.Equal(t, &iam.Role{
+		Path:                     got.Role.Path,
+		RoleName:                 got.Role.RoleName,
+		RoleId:                   got.Role.RoleId,
+		Arn:                      got.Role.Arn,
+		CreateDate:               got.Role.CreateDate,
+		AssumeRolePolicyDocument: got.Role.AssumeRolePolicyDocument,
+		Description:              aws.String("new description"),
+	}, out.Role, "AWS omits MaxSessionDuration, Tags and RoleLastUsed from this output")
 
 	_, err = svc.UpdateRoleDescription(testAccountID, &iam.UpdateRoleDescriptionInput{
 		RoleName:    aws.String("desc-role"),

spinifex/domains/iam/service.go

@@ -15,6 +15,7 @@ type IAMService interface {
 	GetUser(accountID string, input *iam.GetUserInput) (*iam.GetUserOutput, error)
 	ListUsers(accountID string, input *iam.ListUsersInput) (*iam.ListUsersOutput, error)
 	DeleteUser(accountID string, input *iam.DeleteUserInput) (*iam.DeleteUserOutput, error)
+	UpdateUser(accountID string, input *iam.UpdateUserInput) (*iam.UpdateUserOutput, error)
 
 	// Access key lifecycle — account-scoped
 	CreateAccessKey(accountID string, input *iam.CreateAccessKeyInput) (*iam.CreateAccessKeyOutput, error)

spinifex/domains/iam/service_impl.go

@@ -467,7 +467,7 @@ func (s *IAMServiceImpl) CreateUser(accountID string, input *iam.CreateUserInput
 	// Atomic create — fails if key already exists (race-safe)
 	if _, err := s.usersBucket.Create(ctx, kvKey, data); err != nil {
 		if errors.Is(err, jetstream.ErrKeyExists) {
-			return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists)
+			return nil, awserrors.Errorf(awserrors.ErrorIAMEntityAlreadyExists, "User with name %s already exists.", userName)
 		}
 		return nil, fmt.Errorf("store user: %w", err)
 	}
@@ -587,7 +587,7 @@ func (s *IAMServiceImpl) DeleteUser(accountID string, input *iam.DeleteUserInput
 	// block the delete either; removing the user record drops it.
 	for _, keyID := range user.AccessKeys {
 		if _, err := s.accessKeysBucket.Get(ctx, keyID); err == nil {
-			return nil, errors.New(awserrors.ErrorIAMDeleteConflict)
+			return nil, awserrors.Errorf(awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must delete access keys first.")
 		} else if !errors.Is(err, jetstream.ErrKeyNotFound) {
 			return nil, fmt.Errorf("get access key: %w", err)
 		}
@@ -595,13 +595,13 @@ func (s *IAMServiceImpl) DeleteUser(accountID string, input *iam.DeleteUserInput
 			"accountID", accountID, "userName", userName, "accessKeyID", keyID)
 	}
 	if len(user.AttachedPolicies) > 0 {
-		return nil, errors.New(awserrors.ErrorIAMDeleteConflict)
+		return nil, errDetachPoliciesFirst
 	}
 	if len(user.Groups) > 0 {
 		return nil, errors.New(awserrors.ErrorIAMDeleteConflict)
 	}
 	if len(user.InlinePolicies) > 0 {
-		return nil, errors.New(awserrors.ErrorIAMDeleteConflict)
+		return nil, errDeletePoliciesFirst
 	}
 
 	if err := s.usersBucket.Delete(ctx, kvKey); err != nil {
@@ -625,6 +625,133 @@ func (s *IAMServiceImpl) DeleteUser(accountID string, input *iam.DeleteUserInput
 	return &iam.DeleteUserOutput{}, nil
 }
 
+// UpdateUser applies NewPath and NewUserName to a user. A rename moves the
+// record to its new key, keeping its ID, access keys, groups and policies.
+func (s *IAMServiceImpl) UpdateUser(accountID string, input *iam.UpdateUserInput) (*iam.UpdateUserOutput, error) {
+	ctx := context.Background()
+	userName := *input.UserName
+	if input.NewUserName != nil {
+		if err := validateIAMName("newUserName", *input.NewUserName, 64); err != nil {
+			return nil, err
+		}
+	}
+	if input.NewPath != nil {
+		if err := validatePath(*input.NewPath); err != nil {
+			return nil, err
+		}
+	}
+
+	newName := aws.StringValue(input.NewUserName)
+	var err error
+	if newName == "" || newName == userName {
+		err = s.updateUserCAS(ctx, accountID, userName, func(u *User) (bool, error) {
+			if input.NewPath == nil || *input.NewPath == u.Path {
+				return false, nil
+			}
+			u.Path = *input.NewPath
+			u.ARN = arn.FormatIAMPath(arn.IAMUser, accountID, u.Path, u.UserName)
+			return true, nil
+		})
+	} else {
+		err = s.renameUser(ctx, accountID, userName, newName, input.NewPath)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	slog.Info("IAM user updated", "accountID", accountID, "userName", userName, "newUserName", newName)
+	return &iam.UpdateUserOutput{}, nil
+}
+
+// renameUser writes the user under newName and repoints its access keys before
+// deleting the old record at the revision it read. A racing writer of the old
+// record makes that delete fail, so the rename is undone and retried.
+func (s *IAMServiceImpl) renameUser(ctx context.Context, accountID, oldName, newName string, newPath *string) error {
+	oldKey, newKey := accountID+"."+oldName, accountID+"."+newName
+	first, second := oldKey, newKey
+	if second < first {
+		first, second = second, first
+	}
+	defer s.recordLocks.lock("user/" + first)()
+	defer s.recordLocks.lock("user/" + second)()
+
+	for range roleCASMaxRetries {
+		entry, err := s.usersBucket.Get(ctx, oldKey)
+		if err != nil {
+			if errors.Is(err, jetstream.ErrKeyNotFound) {
+				return errors.New(awserrors.ErrorIAMNoSuchEntity)
+			}
+			return fmt.Errorf("get user: %w", err)
+		}
+		var user User
+		if err := json.Unmarshal(entry.Value(), &user); err != nil {
+			return fmt.Errorf("unmarshal user: %w", err)
+		}
+		user.UserName = newName
+		if newPath != nil {
+			user.Path = *newPath
+		}
+		user.ARN = arn.FormatIAMPath(arn.IAMUser, accountID, user.Path, newName)
+		data, err := json.Marshal(user)
+		if err != nil {
+			return fmt.Errorf("marshal user: %w", err)
+		}
+
+		newRev, err := s.usersBucket.Create(ctx, newKey, data)
+		if err != nil {
+			if errors.Is(err, jetstream.ErrKeyExists) {
+				return errors.New(awserrors.ErrorIAMEntityAlreadyExists)
+			}
+			return fmt.Errorf("store renamed user: %w", err)
+		}
+		err = s.setAccessKeysOwner(ctx, accountID, user.AccessKeys, newName)
+		if err == nil {
+			err = s.usersBucket.Delete(ctx, oldKey, jetstream.LastRevision(entry.Revision()))
+			if err == nil {
+				return nil
+			}
+		}
+		s.undoRenameUser(ctx, accountID, user.AccessKeys, oldName, newKey, newRev)
+		if !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
+			return fmt.Errorf("rename user: %w", err)
+		}
+	}
+	slog.Error("IAM user rename lost every revision race on the old record",
+		"accountID", accountID, "userName", oldName, "attempts", roleCASMaxRetries)
+	return errors.New(awserrors.ErrorServerInternal)
+}
+
+// setAccessKeysOwner points each access key at userName. A listed key with no
+// record is skipped, as ListAccessKeys skips it.
+func (s *IAMServiceImpl) setAccessKeysOwner(ctx context.Context, accountID string, keyIDs []string, userName string) error {
+	for _, keyID := range keyIDs {
+		cfg := iamRecordCASConfig("access key", accountID, keyID)
+		cfg.NotFound = nil
+		_, err := kvutil.Update(ctx, s.accessKeysBucket, keyID, cfg, func(ak *AccessKey) (bool, error) {
+			if ak.UserName == userName {
+				return false, nil
+			}
+			ak.UserName = userName
+			return true, nil
+		})
+		if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
+			return fmt.Errorf("repoint access key %s: %w", keyID, err)
+		}
+	}
+	return nil
+}
+
+// undoRenameUser returns the access keys to oldName and drops the record
+// written under the new name, unless a writer has changed it since.
+func (s *IAMServiceImpl) undoRenameUser(ctx context.Context, accountID string, keyIDs []string, oldName, newKey string, newRev uint64) {
+	if err := s.setAccessKeysOwner(ctx, accountID, keyIDs, oldName); err != nil {
+		slog.Error("Rollback failed: access keys left on renamed user", "accountID", accountID, "userName", oldName, "err", err)
+	}
+	if err := s.usersBucket.Delete(ctx, newKey, jetstream.LastRevision(newRev)); err != nil {
+		slog.Error("Rollback failed: renamed user record left behind", "accountID", accountID, "userName", oldName, "key", newKey, "err", err)
+	}
+}
+
 // ---------------------------------------------------------------------------
 // Access Key Lifecycle
 // ---------------------------------------------------------------------------
@@ -753,7 +880,7 @@ func (s *IAMServiceImpl) DeleteAccessKey(accountID string, input *iam.DeleteAcce
 	err := s.updateUserCAS(ctx, accountID, userName, func(u *User) (bool, error) {
 		idx := slices.Index(u.AccessKeys, accessKeyID)
 		if idx < 0 {
-			return false, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return false, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "The Access Key with id %s cannot be found.", accessKeyID)
 		}
 		u.AccessKeys = slices.Delete(u.AccessKeys, idx, idx+1)
 		return true, nil
@@ -1355,7 +1482,8 @@ func (s *IAMServiceImpl) CreatePolicy(accountID string, input *iam.CreatePolicyI
 
 	if _, err := s.policiesBucket.Create(ctx, kvKey, data); err != nil {
 		if errors.Is(err, jetstream.ErrKeyExists) {
-			return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists)
+			return nil, awserrors.Errorf(awserrors.ErrorIAMEntityAlreadyExists,
+				"A policy called %s already exists. Duplicate names are not allowed.", policyName)
 		}
 		return nil, fmt.Errorf("store policy: %w", err)
 	}
@@ -1502,7 +1630,7 @@ func (s *IAMServiceImpl) DeletePolicy(accountID string, input *iam.DeletePolicyI
 	// Revision-guarded, so a version created after the checks cannot be deleted unseen.
 	policy, err := kvutil.DeleteIf(ctx, s.policiesBucket, key, cfg, func(p *Policy) error {
 		if p.ARN != policyARN {
-			return errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return policyNotFound(policyARN)
 		}
 		if len(p.OtherVersions) > 0 {
 			return awserrors.Errorf(awserrors.ErrorIAMDeleteConflict,
@@ -1513,7 +1641,7 @@ func (s *IAMServiceImpl) DeletePolicy(accountID string, input *iam.DeletePolicyI
 			return fmt.Errorf("check policy attachments: %w", err)
 		}
 		if counts[p.ARN] > 0 {
-			return errors.New(awserrors.ErrorIAMDeleteConflict)
+			return awserrors.Errorf(awserrors.ErrorIAMDeleteConflict, "Cannot delete a policy attached to entities.")
 		}
 		return nil
 	})
@@ -1612,7 +1740,7 @@ func (s *IAMServiceImpl) AttachUserPolicy(accountID string, input *iam.AttachUse
 	// grant document is modeled in builtinManagedPolicyDoc and resolved at
 	// evaluation time. Customer-managed ARNs must still exist.
 	if !iamarn.IsAWSManagedPolicyARN(policyARN) {
-		if _, err := s.getPolicyByARN(ctx, accountID, policyARN); err != nil {
+		if err := s.attachablePolicy(ctx, accountID, policyARN); err != nil {
 			return nil, err
 		}
 	}
@@ -1640,7 +1768,7 @@ func (s *IAMServiceImpl) DetachUserPolicy(accountID string, input *iam.DetachUse
 	err := s.updateUserCAS(ctx, accountID, userName, func(u *User) (bool, error) {
 		idx := slices.Index(u.AttachedPolicies, policyARN)
 		if idx < 0 {
-			return false, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return false, policyNotFound(policyARN)
 		}
 		u.AttachedPolicies = slices.Delete(u.AttachedPolicies, idx, idx+1)
 		return true, nil
@@ -1774,7 +1902,7 @@ func (s *IAMServiceImpl) GetUserPolicy(accountID string, input *iam.GetUserPolic
 
 	doc, ok := user.InlinePolicies[policyName]
 	if !ok {
-		return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+		return nil, noSuchEntity("user policy", policyName)
 	}
 
 	return &iam.GetUserPolicyOutput{
@@ -1793,7 +1921,7 @@ func (s *IAMServiceImpl) DeleteUserPolicy(accountID string, input *iam.DeleteUse
 
 	err := s.updateUserCAS(ctx, accountID, userName, func(u *User) (bool, error) {
 		if _, ok := u.InlinePolicies[policyName]; !ok {
-			return false, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return false, noSuchEntity("user policy", policyName)
 		}
 		delete(u.InlinePolicies, policyName)
 		return true, nil
@@ -1984,7 +2112,7 @@ func (s *IAMServiceImpl) GetUserPolicies(accountID, userName string) ([]PolicyDo
 	for _, groupName := range user.Groups {
 		group, err := s.getGroup(ctx, accountID, groupName)
 		if err != nil {
-			if err.Error() == awserrors.ErrorIAMNoSuchEntity {
+			if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
 				slog.Warn("GetUserPolicies: member references missing group; skipping",
 					"accountID", accountID, "user", userName, "group", groupName)
 				continue
@@ -2025,7 +2153,7 @@ func (s *IAMServiceImpl) getPolicyByARN(ctx context.Context, accountID, policyAR
 		entry, err := s.policiesBucket.Get(ctx, kvKey)
 		if err != nil {
 			if errors.Is(err, jetstream.ErrKeyNotFound) {
-				return "", errors.New(awserrors.ErrorIAMNoSuchEntity)
+				return "", policyNotFound(policyARN)
 			}
 			return "", fmt.Errorf("get policy: %w", err)
 		}
@@ -2042,9 +2170,9 @@ func (s *IAMServiceImpl) getPolicyByARN(ctx context.Context, accountID, policyAR
 		// otherwise a malformed ARN is indistinguishable from a missing policy.
 		slog.Debug("getPolicyByARN: unparseable policy ARN",
 			"accountID", accountID, "policyArn", policyARN, "err", err)
-		return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+		return nil, policyNotFound(policyARN)
 	case errors.Is(err, iamarn.ErrPolicyARNMismatch):
-		return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+		return nil, policyNotFound(policyARN)
 	case err != nil:
 		return nil, err
 	}
@@ -2166,12 +2294,45 @@ func (s *IAMServiceImpl) buildAttachmentCounts(ctx context.Context, accountID st
 	return counts, nil
 }
 
+// noSuchEntity is NoSuchEntity with AWS's message naming the missing user,
+// role, group or inline policy, e.g. "The role with name X cannot be found.".
+func noSuchEntity(kind, name string) error {
+	return awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "The %s with name %s cannot be found.", kind, name)
+}
+
+// policyNotFound is NoSuchEntity with AWS's message for a managed policy ARN
+// that names no stored policy, or none attached to the entity.
+func policyNotFound(policyARN string) error {
+	return awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Policy %s was not found.", policyARN)
+}
+
+// oidcProviderNotFound is NoSuchEntity with AWS's message for a missing OIDC provider.
+func oidcProviderNotFound(providerARN string) error {
+	return awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "OpenIDConnect Provider not found for arn %s", providerARN)
+}
+
+// The DeleteConflict messages AWS shares across users, roles and groups.
+var (
+	errDetachPoliciesFirst = awserrors.Errorf(awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must detach all policies first.")
+	errDeletePoliciesFirst = awserrors.Errorf(awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must delete policies first.")
+)
+
+// attachablePolicy refuses a customer-managed policy ARN that names no stored
+// policy, with the message AWS gives on attach.
+func (s *IAMServiceImpl) attachablePolicy(ctx context.Context, accountID, policyARN string) error {
+	_, err := s.getPolicyByARN(ctx, accountID, policyARN)
+	if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
+		return awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Policy %s does not exist or is not attachable.", policyARN)
+	}
+	return err
+}
+
 func (s *IAMServiceImpl) getUser(ctx context.Context, accountID, userName string) (*User, error) {
 	kvKey := accountID + "." + userName
 	entry, err := s.usersBucket.Get(ctx, kvKey)
 	if err != nil {
 		if errors.Is(err, jetstream.ErrKeyNotFound) {
-			return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return nil, noSuchEntity("user", userName)
 		}
 		return nil, fmt.Errorf("get user: %w", err)
 	}
@@ -2195,9 +2356,13 @@ func (s *IAMServiceImpl) updateUserCAS(ctx context.Context, accountID, userName
 // iamRecordCASConfig is the CAS policy for an IAM user, group or OIDC provider
 // record: a missing record is NoSuchEntity and exhausted retries are logged.
 func iamRecordCASConfig(kind, accountID, name string) kvutil.CASConfig {
+	notFound := noSuchEntity(kind, name)
+	if kind == "oidc-provider" {
+		notFound = oidcProviderNotFound(name)
+	}
 	return kvutil.CASConfig{
 		Attempts: roleCASMaxRetries,
-		NotFound: errors.New(awserrors.ErrorIAMNoSuchEntity),
+		NotFound: notFound,
 		Exhausted: func(string, int) error {
 			slog.Error("IAM record CAS retries exhausted under contention",
 				"kind", kind, "accountID", accountID, "name", name, "attempts", roleCASMaxRetries)

spinifex/domains/iam/system_role.go

@@ -48,7 +48,7 @@ func EnsureSystemInstanceProfile(e SystemInstanceRoleEnsurer, accountID, roleNam
 // there draws credentials from it, so provisioning one would be unasked-for.
 func ConvergeSystemRolePolicy(e SystemInstanceRoleEnsurer, accountID, roleName, policyName, policyDoc string) error {
 	if _, err := e.GetRole(accountID, &iam.GetRoleInput{RoleName: aws.String(roleName)}); err != nil {
-		if err.Error() == awserrors.ErrorIAMNoSuchEntity {
+		if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
 			return nil
 		}
 		return fmt.Errorf("get role %q: %w", roleName, err)
@@ -68,7 +68,7 @@ func ConvergeSystemRolePolicy(e SystemInstanceRoleEnsurer, accountID, roleName,
 func ensureSystemRole(e SystemInstanceRoleEnsurer, accountID, roleName string) error {
 	if _, err := e.GetRole(accountID, &iam.GetRoleInput{RoleName: aws.String(roleName)}); err == nil {
 		return nil
-	} else if err.Error() != awserrors.ErrorIAMNoSuchEntity {
+	} else if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMNoSuchEntity {
 		return fmt.Errorf("get role %q: %w", roleName, err)
 	}
 
@@ -77,7 +77,7 @@ func ensureSystemRole(e SystemInstanceRoleEnsurer, accountID, roleName string) e
 		AssumeRolePolicyDocument: aws.String(EC2InstanceTrustPolicy),
 		Description:              aws.String("Spinifex-managed system instance role"),
 	})
-	if err == nil || err.Error() == awserrors.ErrorIAMEntityAlreadyExists {
+	if code, _ := awserrors.ResolveErrorCode(err); err == nil || code == awserrors.ErrorIAMEntityAlreadyExists {
 		return nil
 	}
 	return fmt.Errorf("create role %q: %w", roleName, err)
@@ -94,7 +94,7 @@ func ensureSystemInstanceProfile(e SystemInstanceRoleEnsurer, accountID, roleNam
 		return attachSystemRole(e, accountID, roleName,
 			aws.StringValue(out.InstanceProfile.Arn), len(out.InstanceProfile.Roles) > 0)
 	}
-	if err.Error() != awserrors.ErrorIAMNoSuchEntity {
+	if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMNoSuchEntity {
 		return "", fmt.Errorf("get instance profile %q: %w", roleName, err)
 	}
 
@@ -102,7 +102,7 @@ func ensureSystemInstanceProfile(e SystemInstanceRoleEnsurer, accountID, roleNam
 		InstanceProfileName: aws.String(roleName),
 	})
 	if err != nil {
-		if err.Error() != awserrors.ErrorIAMEntityAlreadyExists {
+		if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMEntityAlreadyExists {
 			return "", fmt.Errorf("create instance profile %q: %w", roleName, err)
 		}
 		got, gerr := e.GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{
@@ -128,7 +128,7 @@ func attachSystemRole(e SystemInstanceRoleEnsurer, accountID, roleName, profileA
 		InstanceProfileName: aws.String(roleName),
 		RoleName:            aws.String(roleName),
 	})
-	if err != nil && err.Error() != awserrors.ErrorIAMLimitExceeded {
+	if code, _ := awserrors.ResolveErrorCode(err); err != nil && code != awserrors.ErrorIAMLimitExceeded {
 		return "", fmt.Errorf("add role %q to instance profile %q: %w", roleName, roleName, err)
 	}
 	return profileARN, nil

spinifex/domains/iam/tags_test.go

@@ -460,7 +460,7 @@ func TestCreate_InvalidTagsRefusedAndNotStored(t *testing.T) {
 
 				_, err = ops.list(svc, id)
 				require.Error(t, err)
-				assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, err.Error())
+				assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, awserrors.ValidErrorCodeFromError(err))
 			})
 		}
 	}
@@ -577,15 +577,15 @@ func TestResourceTagging_MissingEntity(t *testing.T) {
 
 			err := ops.tag(svc, ops.missingID, []*iam.Tag{sdkTag("env", "prod")})
 			require.Error(t, err)
-			assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, err.Error())
+			assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, awserrors.ValidErrorCodeFromError(err))
 
 			err = ops.untag(svc, ops.missingID, []*string{aws.String("env")})
 			require.Error(t, err)
-			assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, err.Error())
+			assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, awserrors.ValidErrorCodeFromError(err))
 
 			_, err = ops.list(svc, ops.missingID)
 			require.Error(t, err)
-			assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, err.Error())
+			assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, awserrors.ValidErrorCodeFromError(err))
 		})
 	}
 }
@@ -608,7 +608,7 @@ func TestTagUser_MergedLimitExceeded(t *testing.T) {
 		Tags:     []*iam.Tag{sdkTag("overflow", "v")},
 	})
 	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorIAMLimitExceeded, err.Error())
+	assert.Equal(t, awserrors.ErrorIAMLimitExceeded, awserrors.ValidErrorCodeFromError(err))
 
 	// Upserting an existing key does not grow the set and stays allowed.
 	_, err = svc.TagUser(testAccountID, &iam.TagUserInput{

spinifex/domains/iam/user_update_test.go

@@ -0,0 +1,203 @@
+package iam
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestUpdateUser_RenameKeepsRelationships(t *testing.T) {
+	t.Parallel()
+	svc := setupTestIAMService(t)
+	before := createTestUser(t, svc, "bob")
+	policy := createTestPolicy(t, svc, "bob-managed")
+	createTestGroup(t, svc, "devs")
+
+	_, err := svc.AttachUserPolicy(testAccountID, &iam.AttachUserPolicyInput{UserName: aws.String("bob"), PolicyArn: policy.Arn})
+	require.NoError(t, err)
+	_, err = svc.PutUserPolicy(testAccountID, &iam.PutUserPolicyInput{
+		UserName: aws.String("bob"), PolicyName: aws.String("inline"), PolicyDocument: aws.String(validPolicyDocument()),
+	})
+	require.NoError(t, err)
+	_, err = svc.AddUserToGroup(testAccountID, &iam.AddUserToGroupInput{UserName: aws.String("bob"), GroupName: aws.String("devs")})
+	require.NoError(t, err)
+	key, err := svc.CreateAccessKey(testAccountID, &iam.CreateAccessKeyInput{UserName: aws.String("bob")})
+	require.NoError(t, err)
+
+	out, err := svc.UpdateUser(testAccountID, &iam.UpdateUserInput{
+		UserName: aws.String("bob"), NewUserName: aws.String("robert"), NewPath: aws.String("/eng/"),
+	})
+	require.NoError(t, err)
+	assert.Equal(t, &iam.UpdateUserOutput{}, out)
+
+	_, err = svc.GetUser(testAccountID, &iam.GetUserInput{UserName: aws.String("bob")})
+	require.Error(t, err)
+	code, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok, "no AWS code on %v", err)
+	assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, code)
+
+	got, err := svc.GetUser(testAccountID, &iam.GetUserInput{UserName: aws.String("robert")})
+	require.NoError(t, err)
+	assert.Equal(t, *before.UserId, *got.User.UserId)
+	assert.Equal(t, "/eng/", *got.User.Path)
+	assert.Equal(t, "arn:aws:iam::"+testAccountID+":user/eng/robert", *got.User.Arn)
+
+	ak, err := svc.LookupAccessKey(*key.AccessKey.AccessKeyId)
+	require.NoError(t, err)
+	assert.Equal(t, "robert", ak.UserName)
+	keys, err := svc.ListAccessKeys(testAccountID, &iam.ListAccessKeysInput{UserName: aws.String("robert")})
+	require.NoError(t, err)
+	require.Len(t, keys.AccessKeyMetadata, 1)
+	assert.Equal(t, *key.AccessKey.AccessKeyId, *keys.AccessKeyMetadata[0].AccessKeyId)
+
+	attached, err := svc.ListAttachedUserPolicies(testAccountID, &iam.ListAttachedUserPoliciesInput{UserName: aws.String("robert")})
+	require.NoError(t, err)
+	require.Len(t, attached.AttachedPolicies, 1)
+	assert.Equal(t, *policy.Arn, *attached.AttachedPolicies[0].PolicyArn)
+
+	inline, err := svc.ListUserPolicies(testAccountID, &iam.ListUserPoliciesInput{UserName: aws.String("robert")})
+	require.NoError(t, err)
+	assert.Equal(t, []*string{aws.String("inline")}, inline.PolicyNames)
+
+	groups, err := svc.ListGroupsForUser(testAccountID, &iam.ListGroupsForUserInput{UserName: aws.String("robert")})
+	require.NoError(t, err)
+	require.Len(t, groups.Groups, 1)
+	assert.Equal(t, "devs", *groups.Groups[0].GroupName)
+
+	members, err := svc.GetGroup(testAccountID, &iam.GetGroupInput{GroupName: aws.String("devs")})
+	require.NoError(t, err)
+	require.Len(t, members.Users, 1)
+	assert.Equal(t, "robert", *members.Users[0].UserName)
+}
+
+func TestUpdateUser_PathOnly(t *testing.T) {
+	t.Parallel()
+	svc := setupTestIAMService(t)
+	createTestUser(t, svc, "carol")
+
+	_, err := svc.UpdateUser(testAccountID, &iam.UpdateUserInput{UserName: aws.String("carol"), NewPath: aws.String("/ops/")})
+	require.NoError(t, err)
+
+	got, err := svc.GetUser(testAccountID, &iam.GetUserInput{UserName: aws.String("carol")})
+	require.NoError(t, err)
+	assert.Equal(t, "/ops/", *got.User.Path)
+	assert.Equal(t, "arn:aws:iam::"+testAccountID+":user/ops/carol", *got.User.Arn)
+}
+
+func TestUpdateUser_Errors(t *testing.T) {
+	t.Parallel()
+	svc := setupTestIAMService(t)
+	createTestUser(t, svc, "dave")
+	createTestUser(t, svc, "erin")
+
+	tests := []struct {
+		name  string
+		input *iam.UpdateUserInput
+		want  string
+	}{
+		{"missing user", &iam.UpdateUserInput{UserName: aws.String("ghost"), NewPath: aws.String("/x/")}, awserrors.ErrorIAMNoSuchEntity},
+		{"missing user rename", &iam.UpdateUserInput{UserName: aws.String("ghost"), NewUserName: aws.String("ghost2")}, awserrors.ErrorIAMNoSuchEntity},
+		{"taken name", &iam.UpdateUserInput{UserName: aws.String("dave"), NewUserName: aws.String("erin")}, awserrors.ErrorIAMEntityAlreadyExists},
+		{"bad name", &iam.UpdateUserInput{UserName: aws.String("dave"), NewUserName: aws.String("no spaces")}, awserrors.ErrorValidationError},
+		{"bad path", &iam.UpdateUserInput{UserName: aws.String("dave"), NewPath: aws.String("noslash")}, awserrors.ErrorValidationError},
+	}
+	for _, tc := range tests {
+		_, err := svc.UpdateUser(testAccountID, tc.input)
+		require.Error(t, err, tc.name)
+		assert.True(t, awserrors.IsErrorCode(err, tc.want), "%s: got %v want %s", tc.name, err, tc.want)
+	}
+
+	got, err := svc.GetUser(testAccountID, &iam.GetUserInput{UserName: aws.String("dave")})
+	require.NoError(t, err)
+	assert.Equal(t, "/", *got.User.Path, "a refused update leaves the user unchanged")
+}
+
+// racingUsersKV writes the old user record once, right after the renamed
+// record is created, as a concurrent AttachUserPolicy on another node would.
+type racingUsersKV struct {
+	jetstream.KeyValue
+
+	oldKey string
+	raced  bool
+}
+
+func (k *racingUsersKV) Create(ctx context.Context, key string, value []byte, opts ...jetstream.KVCreateOpt) (uint64, error) {
+	rev, err := k.KeyValue.Create(ctx, key, value, opts...)
+	if err != nil || k.raced {
+		return rev, err
+	}
+	k.raced = true
+	entry, gerr := k.Get(ctx, k.oldKey)
+	if gerr != nil {
+		return rev, gerr
+	}
+	var u User
+	if uerr := json.Unmarshal(entry.Value(), &u); uerr != nil {
+		return rev, uerr
+	}
+	u.AttachedPolicies = append(u.AttachedPolicies, "arn:aws:iam::"+testAccountID+":policy/raced")
+	data, _ := json.Marshal(u)
+	if _, perr := k.Update(ctx, k.oldKey, data, entry.Revision()); perr != nil {
+		return rev, perr
+	}
+	return rev, nil
+}
+
+func TestUpdateUser_RenameRetriesAfterRacingWrite(t *testing.T) {
+	t.Parallel()
+	svc := setupTestIAMService(t)
+	createTestUser(t, svc, "frank")
+	key, err := svc.CreateAccessKey(testAccountID, &iam.CreateAccessKeyInput{UserName: aws.String("frank")})
+	require.NoError(t, err)
+	svc.usersBucket = &racingUsersKV{KeyValue: svc.usersBucket, oldKey: testAccountID + ".frank"}
+
+	_, err = svc.UpdateUser(testAccountID, &iam.UpdateUserInput{UserName: aws.String("frank"), NewUserName: aws.String("franklin")})
+	require.NoError(t, err)
+
+	user, err := svc.getUser(t.Context(), testAccountID, "franklin")
+	require.NoError(t, err)
+	assert.Equal(t, []string{"arn:aws:iam::" + testAccountID + ":policy/raced"}, user.AttachedPolicies,
+		"the racing write survives the rename")
+	_, err = svc.getUser(t.Context(), testAccountID, "frank")
+	code, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok, "no AWS code on %v", err)
+	assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, code)
+	ak, err := svc.LookupAccessKey(*key.AccessKey.AccessKeyId)
+	require.NoError(t, err)
+	assert.Equal(t, "franklin", ak.UserName)
+}
+
+// UpdateUser names its user by bare name, so the caller's account is the only
+// scope: a rename must never reach a same-named user in another account.
+func TestUpdateUser_ScopedToCallerAccount(t *testing.T) {
+	t.Parallel()
+	svc := setupTestIAMService(t)
+	const other = "999988887777"
+	mine := createTestUser(t, svc, "bob")
+	theirs, err := svc.CreateUser(other, &iam.CreateUserInput{UserName: aws.String("bob")})
+	require.NoError(t, err)
+	_, err = svc.CreateUser(other, &iam.CreateUserInput{UserName: aws.String("carol")})
+	require.NoError(t, err)
+
+	_, err = svc.UpdateUser(testAccountID, &iam.UpdateUserInput{UserName: aws.String("bob"), NewUserName: aws.String("robert")})
+	require.NoError(t, err)
+	got, err := svc.GetUser(other, &iam.GetUserInput{UserName: aws.String("bob")})
+	require.NoError(t, err)
+	assert.Equal(t, *theirs.User.UserId, *got.User.UserId)
+	assert.NotEqual(t, *mine.UserId, *got.User.UserId)
+
+	_, err = svc.UpdateUser(testAccountID, &iam.UpdateUserInput{UserName: aws.String("carol"), NewUserName: aws.String("caroline")})
+	require.Error(t, err)
+	code, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok, "no AWS code on %v", err)
+	assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, code)
+	_, err = svc.GetUser(other, &iam.GetUserInput{UserName: aws.String("carol")})
+	require.NoError(t, err)
+}

spinifex/domains/lambda/awsapi/create.go

@@ -0,0 +1,168 @@
+package awsapi
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/base64"
+	"encoding/hex"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/lambda"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/codestore"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+const (
+	rollbackDeadline = 10 * time.Second
+	lastModifiedFmt  = "2006-01-02T15:04:05.000-0700"
+)
+
+func randomID() (string, error) {
+	b := make([]byte, 16)
+	if _, err := rand.Read(b); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(b), nil
+}
+
+func serviceError(msg string) error {
+	return awserrors.Errorf(awserrors.ErrorServiceException, "%s", msg)
+}
+
+// createFunction implements CreateFunction. The name is claimed first, the code
+// copy written second, and the claim completed last, so a failure at any point
+// leaves either a free name or a claim that expires on its own.
+func (d Deps) createFunction(ctx context.Context, c *call) (reply, error) {
+	p, err := validateCreate(c.Create)
+	if err != nil {
+		return reply{}, err
+	}
+	functionID, err := randomID()
+	if err != nil {
+		return reply{}, serviceError("could not generate a function identifier")
+	}
+	revisionID, err := randomID()
+	if err != nil {
+		return reply{}, serviceError("could not generate a revision identifier")
+	}
+	var envCipher string
+	if len(p.Env) > 0 {
+		if d.AEAD == nil {
+			return reply{}, serviceError("environment encryption is unavailable")
+		}
+		if envCipher, err = function.SealEnv(d.AEAD, c.AccountID, functionID, p.Env); err != nil {
+			slog.ErrorContext(ctx, "Lambda: seal environment failed", "function_id", functionID, "err", err)
+			return reply{}, serviceError("could not protect the function environment")
+		}
+	}
+
+	rec := function.Record{
+		FormatVersion: function.FormatVersion,
+		FunctionID:    functionID,
+		AccountID:     c.AccountID,
+		Region:        c.Region,
+		FunctionName:  p.Name,
+		Role:          p.Role,
+		Runtime:       supportedRuntime,
+		Handler:       p.Handler,
+		Description:   p.Description,
+		Timeout:       p.TimeoutSec,
+		MemorySize:    p.MemorySize,
+		EnvCipher:     envCipher,
+		State:         function.StatePending,
+		Code:          function.Code{RevisionID: revisionID, State: function.CodeStoring},
+	}
+	createRev, err := d.Meta.Create(ctx, c.AccountID, c.Region, rec)
+	if err != nil {
+		return reply{}, err
+	}
+
+	ref := codestore.Ref{AccountID: c.AccountID, FunctionID: functionID, RevisionID: revisionID}
+	store, err := d.Codes(ctx)
+	if err != nil {
+		d.rollback(ctx, c, p.Name, createRev, ref, "code store unavailable", err)
+		return reply{}, serviceError("the function code could not be stored")
+	}
+	sha, err := store.Put(ctx, ref, p.Code)
+	if err != nil {
+		d.rollback(ctx, c, p.Name, createRev, ref, "code write failed", err)
+		return reply{}, serviceError("the function code could not be stored")
+	}
+	stored, storedRev, err := d.Meta.CodeStored(ctx, c.AccountID, c.Region, p.Name, createRev, sha, int64(len(p.Code)))
+	if err != nil {
+		d.rollback(ctx, c, p.Name, createRev, ref, "code-stored was not recorded", err)
+		return reply{}, serviceError("the function could not be completed")
+	}
+
+	cfg, err := configuration(stored, storedRev, p.Env)
+	if err != nil {
+		return reply{}, serviceError("could not build the function configuration")
+	}
+	return reply{Status: http.StatusCreated, Body: cfg}, nil
+}
+
+// rollback makes one compare-and-swap attempt to free the name. It cannot undo
+// a create that completed, because the revision it holds has moved on. The code
+// object, if one landed, is left in place and recorded as a cleanup obligation.
+func (d Deps) rollback(ctx context.Context, c *call, name string, createRev uint64, ref codestore.Ref, why string, cause error) {
+	slog.ErrorContext(ctx, "Lambda: create did not complete", "reason", why, "function_name", name,
+		"function_id", ref.FunctionID, "err", cause)
+	rctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), rollbackDeadline)
+	defer cancel()
+	if err := d.Meta.RollbackCreate(rctx, c.AccountID, c.Region, name, createRev); err != nil {
+		slog.WarnContext(ctx, "Lambda: rollback of the name claim failed; it expires on its own",
+			"function_name", name, "function_id", ref.FunctionID, "err", err)
+	}
+	slog.WarnContext(ctx, "Lambda: code object cleanup is owed",
+		"account_id", ref.AccountID, "function_id", ref.FunctionID, "revision_id", ref.RevisionID)
+}
+
+// configuration renders the record as a FunctionConfiguration.
+func configuration(rec function.Record, revision uint64, env map[string]string) (*lambda.FunctionConfiguration, error) {
+	cfg := &lambda.FunctionConfiguration{
+		FunctionName:     aws.String(rec.FunctionName),
+		FunctionArn:      aws.String(arn.FormatLambdaFunction(rec.Region, rec.AccountID, rec.FunctionName)),
+		Runtime:          aws.String(rec.Runtime),
+		Role:             aws.String(rec.Role),
+		Handler:          aws.String(rec.Handler),
+		Description:      aws.String(rec.Description),
+		Timeout:          aws.Int64(rec.Timeout),
+		MemorySize:       aws.Int64(rec.MemorySize),
+		LastModified:     aws.String(rec.LastModified.UTC().Format(lastModifiedFmt)),
+		Version:          aws.String("$LATEST"),
+		State:            aws.String(rec.State),
+		PackageType:      aws.String(lambda.PackageTypeZip),
+		Architectures:    aws.StringSlice([]string{lambda.ArchitectureX8664}),
+		RevisionId:       aws.String(strconv.FormatUint(revision, 10)),
+		LastUpdateStatus: aws.String(lambda.LastUpdateStatusSuccessful),
+		EphemeralStorage: &lambda.EphemeralStorage{Size: aws.Int64(512)},
+		LoggingConfig:    &lambda.LoggingConfig{LogFormat: aws.String(lambda.LogFormatText), LogGroup: aws.String(logGroup(rec.FunctionName))},
+	}
+	if rec.StateReason != "" {
+		cfg.StateReason = aws.String(rec.StateReason)
+		cfg.StateReasonCode = aws.String(rec.StateReasonCode)
+	}
+	if rec.Code.State == function.CodeStored {
+		raw, err := hex.DecodeString(rec.Code.SHA256)
+		if err != nil {
+			return nil, fmt.Errorf("stored digest is not hex: %w", err)
+		}
+		cfg.CodeSha256 = aws.String(base64.StdEncoding.EncodeToString(raw))
+		cfg.CodeSize = aws.Int64(rec.Code.Size)
+	}
+	if len(env) > 0 {
+		vars := make(map[string]*string, len(env))
+		for k, v := range env {
+			vars[k] = aws.String(v)
+		}
+		cfg.Environment = &lambda.EnvironmentResponse{Variables: vars}
+	}
+	return cfg, nil
+}

spinifex/domains/lambda/awsapi/get.go

@@ -0,0 +1,82 @@
+package awsapi
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+	"net/http"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/lambda"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/codestore"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// codeURLTTL is how long the presigned code location stays valid.
+const codeURLTTL = 10 * time.Minute
+
+func notFound(region, accountID, name string) error {
+	return awserrors.Errorf(awserrors.ErrorResourceNotFoundException, "Function not found: %s",
+		arn.FormatLambdaFunction(region, accountID, name))
+}
+
+// getFunction implements GetFunction for the $LATEST version.
+func (d Deps) getFunction(ctx context.Context, c *call) (reply, error) {
+	name, q, err := resolveFunctionName(c.Region, c.AccountID, c.Params[0])
+	if err != nil {
+		return reply{}, err
+	}
+	if query := c.Query.Get("Qualifier"); query != "" {
+		if q != "" && q != query {
+			return reply{}, invalid("The derived qualifier from the function name does not match the specified qualifier.")
+		}
+		q = query
+	}
+	if q != "" && q != "$LATEST" {
+		return reply{}, notFound(c.Region, c.AccountID, name+":"+q)
+	}
+	rec, revision, err := d.Meta.Get(ctx, c.AccountID, c.Region, name)
+	if errors.Is(err, function.ErrNotFound) {
+		return reply{}, notFound(c.Region, c.AccountID, name)
+	}
+	if err != nil {
+		return reply{}, err
+	}
+
+	var env map[string]string
+	if rec.EnvCipher != "" {
+		if d.AEAD == nil {
+			return reply{}, serviceError("environment decryption is unavailable")
+		}
+		if env, err = function.OpenEnv(d.AEAD, rec.AccountID, rec.FunctionID, rec.EnvCipher); err != nil {
+			slog.ErrorContext(ctx, "Lambda: open environment failed", "function_id", rec.FunctionID, "err", err)
+			return reply{}, serviceError("could not read the function environment")
+		}
+	}
+	cfg, err := configuration(rec, revision, env)
+	if err != nil {
+		return reply{}, serviceError("could not build the function configuration")
+	}
+	out := &lambda.GetFunctionOutput{
+		Configuration: cfg,
+		Code:          &lambda.FunctionCodeLocation{RepositoryType: aws.String("S3")},
+	}
+	if rec.Code.State == function.CodeStored {
+		store, err := d.Codes(ctx)
+		if err != nil {
+			slog.ErrorContext(ctx, "Lambda: code store unavailable for GetFunction", "function_id", rec.FunctionID, "err", err)
+			return reply{}, serviceError("the function code location is unavailable")
+		}
+		ref := codestore.Ref{AccountID: rec.AccountID, FunctionID: rec.FunctionID, RevisionID: rec.Code.RevisionID}
+		location, err := store.PresignGet(ctx, ref, rec.Code.SHA256, codeURLTTL)
+		if err != nil {
+			slog.ErrorContext(ctx, "Lambda: presign code location failed", "function_id", rec.FunctionID, "err", err)
+			return reply{}, serviceError("the function code location is unavailable")
+		}
+		out.Code.Location = aws.String(location)
+	}
+	return reply{Status: http.StatusOK, Body: out}, nil
+}

spinifex/domains/lambda/awsapi/handler_test.go

@@ -0,0 +1,669 @@
+package awsapi_test
+
+import (
+	"archive/zip"
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"runtime"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/lambda"
+	"github.com/mulgadc/bluebottle/pkg/auth"
+	"github.com/mulgadc/bluebottle/pkg/masterkey"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/internal/testkit/gwtest"
+	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/codestore"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/passrole"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	roleARN        = "arn:aws:iam::" + gwtest.AccountID + ":role/lambda-exec"
+	foreignRoleARN = "arn:aws:iam::999999999999:role/lambda-exec"
+)
+
+// localClient reaches a real MetaService without NATS.
+type localClient struct {
+	svc *function.MetaService
+
+	mu             sync.Mutex
+	failCodeStored bool
+	passOwnerClock bool
+	rollbacks      int
+}
+
+var _ function.Client = (*localClient)(nil)
+
+func (c *localClient) Create(ctx context.Context, account, region string, rec function.Record) (uint64, error) {
+	r, err := c.svc.Create(ctx, &function.CreateRequest{Region: region, Record: rec}, account)
+	if err != nil {
+		return 0, err
+	}
+	return r.Revision, nil
+}
+
+func (c *localClient) Get(ctx context.Context, account, region, name string) (function.Record, uint64, error) {
+	r, err := c.svc.Get(ctx, &function.GetRequest{Region: region, FunctionName: name}, account)
+	if err != nil {
+		return function.Record{}, 0, err
+	}
+	if !r.Found {
+		return function.Record{}, 0, function.ErrNotFound
+	}
+	return r.Record, r.Revision, nil
+}
+
+func (c *localClient) CodeStored(ctx context.Context, account, region, name string, rev uint64, sha string, size int64) (function.Record, uint64, error) {
+	c.mu.Lock()
+	fail, passClock := c.failCodeStored, c.passOwnerClock
+	c.mu.Unlock()
+	if fail {
+		return function.Record{}, 0, errors.New("nats: timeout")
+	}
+	r, err := c.svc.CodeStored(ctx, &function.CodeStoredRequest{Region: region, FunctionName: name, Revision: rev, SHA256: sha, Size: size}, account)
+	if err != nil {
+		return function.Record{}, 0, err
+	}
+	// Let the wall clock pass the owner's millisecond, so a timestamp the
+	// gateway reads for itself would differ from the stored one.
+	for passClock && !time.Now().Truncate(time.Millisecond).After(r.Record.LastModified.Truncate(time.Millisecond)) {
+		runtime.Gosched()
+	}
+	return r.Record, r.Revision, nil
+}
+
+func (c *localClient) RollbackCreate(ctx context.Context, account, region, name string, rev uint64) error {
+	c.mu.Lock()
+	c.rollbacks++
+	c.mu.Unlock()
+	_, err := c.svc.RollbackCreate(ctx, &function.RollbackCreateRequest{Region: region, FunctionName: name, Revision: rev}, account)
+	return err
+}
+
+// memStore is an in-memory code store; objects is keyed by Ref.
+type memStore struct {
+	mu      sync.Mutex
+	objects map[codestore.Ref][]byte
+	putErr  error
+	urlErr  error
+}
+
+var _ awsapi.CodeStore = (*memStore)(nil)
+
+func (m *memStore) Put(_ context.Context, ref codestore.Ref, code []byte) (string, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.putErr != nil {
+		return "", m.putErr
+	}
+	m.objects[ref] = append([]byte(nil), code...)
+	sum := sha256.Sum256(code)
+	return hex.EncodeToString(sum[:]), nil
+}
+
+func (m *memStore) PresignGet(_ context.Context, ref codestore.Ref, sha string, ttl time.Duration) (string, error) {
+	if m.urlErr != nil {
+		return "", m.urlErr
+	}
+	return "https://code.test/" + ref.FunctionID + "/" + sha + "?ttl=" + ttl.String(), nil
+}
+
+func (m *memStore) count() int {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	return len(m.objects)
+}
+
+type roleSet map[string]bool
+
+func (r roleSet) ResolveRole(accountID, arn string) (passrole.Role, error) {
+	acct, name, err := auth.ParseRoleARN(arn)
+	if err != nil {
+		return passrole.Role{}, passrole.ErrMalformed
+	}
+	if acct != accountID {
+		return passrole.Role{}, passrole.ErrForeignAccount
+	}
+	if !r[arn] {
+		return passrole.Role{}, passrole.ErrNotFound
+	}
+	return passrole.Role{ARN: arn, Name: name}, nil
+}
+
+func newHarness(t *testing.T, statements ...iamdomain.Statement) (*gwtest.Harness, *localClient, *memStore, func(*testing.T) jetstream.KeyValue, *masterkey.Key) {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	key, err := masterkey.New(make([]byte, 32))
+	require.NoError(t, err)
+	meta := &localClient{svc: function.NewMetaService(js)}
+	store := &memStore{objects: map[codestore.Ref][]byte{}}
+	kv := func(t *testing.T) jetstream.KeyValue {
+		t.Helper()
+		b, err := js.KeyValue(t.Context(), function.KVAccountBucket(gwtest.AccountID))
+		require.NoError(t, err)
+		return b
+	}
+	return gwtest.Scoped(statements...), meta, store, kv, key
+}
+
+type env struct {
+	h     *gwtest.Harness
+	meta  *localClient
+	store *memStore
+	kv    func(*testing.T) jetstream.KeyValue
+	deps  awsapi.Deps
+}
+
+func newEnv(t *testing.T, statements ...iamdomain.Statement) *env {
+	t.Helper()
+	if len(statements) == 0 {
+		statements = []iamdomain.Statement{gwtest.Statement("Allow", "*", "*")}
+	}
+	h, meta, store, kv, key := newHarness(t, statements...)
+	e := &env{h: h, meta: meta, store: store, kv: kv}
+	e.deps = awsapi.Deps{
+		Meta: meta,
+		Codes: func(context.Context) (awsapi.CodeStore, error) {
+			return store, nil
+		},
+		Roles:  roleSet{roleARN: true},
+		Trusts: func(string, string) (bool, error) { return true, nil },
+		AEAD:   key.AEAD,
+	}
+	return e
+}
+
+func (e *env) do(method, path string, body any) (*httptest.ResponseRecorder, error) {
+	var raw []byte
+	if body != nil {
+		raw, _ = json.Marshal(body)
+	}
+	rec := httptest.NewRecorder()
+	req := httptest.NewRequest(method, path, strings.NewReader(string(raw)))
+	err := e.h.Dispatch(awsapi.NewRegistration(e.deps), awsapi.ServiceName, rec, req)
+	return rec, err
+}
+
+var testZip = buildZip("hello")
+
+func buildZip(content string) []byte {
+	var buf bytes.Buffer
+	w := zip.NewWriter(&buf)
+	f, err := w.Create("bootstrap")
+	if err == nil {
+		_, err = f.Write([]byte(content))
+	}
+	if err == nil {
+		err = w.Close()
+	}
+	if err != nil {
+		panic(err)
+	}
+	return buf.Bytes()
+}
+
+func mustZip(t *testing.T, content string) []byte {
+	t.Helper()
+	return buildZip(content)
+}
+
+func createBody(name string, mutate func(map[string]any)) map[string]any {
+	b := map[string]any{
+		"FunctionName": name,
+		"Runtime":      "provided.al2023",
+		"Role":         roleARN,
+		"Handler":      "bootstrap",
+		"Code":         map[string]any{"ZipFile": testZip},
+	}
+	if mutate != nil {
+		mutate(b)
+	}
+	return b
+}
+
+func requireCode(t *testing.T, err error, code string) {
+	t.Helper()
+	require.Error(t, err)
+	got, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok, "no AWS code on %v", err)
+	assert.Equal(t, code, got)
+}
+
+func TestCreateFunction_Success(t *testing.T) {
+	e := newEnv(t)
+	code := testZip
+	rec, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) {
+		b["Environment"] = map[string]any{"Variables": map[string]string{"DB_PASSWORD": "hunter2"}}
+		b["Description"] = "demo"
+		b["Timeout"] = 30
+		b["MemorySize"] = 256
+	}))
+	require.NoError(t, err)
+	require.Equal(t, http.StatusCreated, rec.Code)
+
+	var cfg lambda.FunctionConfiguration
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &cfg))
+	sum := sha256.Sum256(code)
+	assert.Equal(t, base64.StdEncoding.EncodeToString(sum[:]), *cfg.CodeSha256, "AWS reports base64 of the raw digest")
+	assert.Equal(t, int64(len(code)), *cfg.CodeSize)
+	assert.Equal(t, "Pending", *cfg.State)
+	assert.Equal(t, "Creating", *cfg.StateReasonCode)
+	assert.Equal(t, "arn:aws:lambda:"+gwtest.Region+":"+gwtest.AccountID+":function:hello", *cfg.FunctionArn)
+	assert.Equal(t, "$LATEST", *cfg.Version)
+	assert.Equal(t, "Zip", *cfg.PackageType)
+	assert.Equal(t, int64(30), *cfg.Timeout)
+	assert.Equal(t, int64(256), *cfg.MemorySize)
+	assert.Equal(t, "hunter2", *cfg.Environment.Variables["DB_PASSWORD"])
+	assert.Regexp(t, `^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}\+0000$`, *cfg.LastModified)
+
+	require.Equal(t, 1, e.store.count())
+	for _, stored := range e.store.objects {
+		assert.Equal(t, code, stored, "the code copy is retrievable")
+	}
+
+	entry, err := e.kv(t).Get(t.Context(), function.KVFunctionKey(gwtest.Region, "hello"))
+	require.NoError(t, err)
+	assert.NotContains(t, string(entry.Value()), "hunter2", "the environment is never stored in plaintext")
+	assert.Contains(t, string(entry.Value()), "envCipher")
+}
+
+func TestCreateFunction_RendersTheStoredTimestamps(t *testing.T) {
+	e := newEnv(t)
+	e.meta.passOwnerClock = true
+	rec, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+	var created lambda.FunctionConfiguration
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &created))
+
+	rec, err = e.do(http.MethodGet, "/2015-03-31/functions/hello", nil)
+	require.NoError(t, err)
+	var got lambda.GetFunctionOutput
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got))
+	assert.Equal(t, *got.Configuration.LastModified, *created.LastModified)
+	assert.Equal(t, *got.Configuration.RevisionId, *created.RevisionId)
+}
+
+func TestCreateFunction_DuplicateNameLeavesFirstUntouched(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+	first, _, err := e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
+	require.NoError(t, err)
+
+	_, err = e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) {
+		b["Code"] = map[string]any{"ZipFile": mustZip(t, "different")}
+	}))
+	requireCode(t, err, awserrors.ErrorResourceConflictException)
+	assert.Contains(t, err.Error(), "Function already exist: hello")
+
+	assert.Equal(t, 1, e.store.count(), "the loser wrote no code")
+	after, _, err := e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
+	require.NoError(t, err)
+	assert.Equal(t, first, after)
+}
+
+func TestCreateFunction_CodePutFailureFreesTheName(t *testing.T) {
+	e := newEnv(t)
+	e.store.putErr = errors.New("predastore down")
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	requireCode(t, err, awserrors.ErrorServiceException)
+	assert.NotContains(t, err.Error(), "predastore down")
+	assert.Equal(t, 1, e.meta.rollbacks)
+	_, _, err = e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
+	require.ErrorIs(t, err, function.ErrNotFound)
+
+	e.store.putErr = nil
+	rec, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusCreated, rec.Code)
+}
+
+func TestCreateFunction_CodeStoredFailureRollsBackOnce(t *testing.T) {
+	e := newEnv(t)
+	e.meta.failCodeStored = true
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	requireCode(t, err, awserrors.ErrorServiceException)
+	assert.Equal(t, 1, e.meta.rollbacks)
+	assert.Equal(t, 1, e.store.count(), "the code object is left in place, never deleted here")
+	_, _, err = e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
+	require.ErrorIs(t, err, function.ErrNotFound, "the still-storing claim was withdrawn")
+}
+
+func TestCreateFunction_RefusesUnsupportedInput(t *testing.T) {
+	cases := map[string]func(map[string]any){
+		"Code.S3Bucket":        func(b map[string]any) { b["Code"] = map[string]any{"S3Bucket": "b", "S3Key": "k"} },
+		"Code.S3Key":           func(b map[string]any) { b["Code"] = map[string]any{"ZipFile": []byte("x"), "S3Key": "k"} },
+		"Code.S3ObjectVersion": func(b map[string]any) { b["Code"] = map[string]any{"ZipFile": []byte("x"), "S3ObjectVersion": "v"} },
+		"Code.ImageUri":        func(b map[string]any) { b["Code"] = map[string]any{"ImageUri": "x"} },
+		"Code.SourceKMSKeyArn": func(b map[string]any) { b["Code"] = map[string]any{"ZipFile": []byte("x"), "SourceKMSKeyArn": "k"} },
+		"Layers":               func(b map[string]any) { b["Layers"] = []string{"arn:aws:lambda:r:1:layer:l:1"} },
+		"VpcConfig":            func(b map[string]any) { b["VpcConfig"] = map[string]any{"SubnetIds": []string{"subnet-1"}} },
+		"KMSKeyArn":            func(b map[string]any) { b["KMSKeyArn"] = "arn:aws:kms:r:1:key/k" },
+		"DeadLetterConfig":     func(b map[string]any) { b["DeadLetterConfig"] = map[string]any{"TargetArn": "arn:aws:sqs:r:1:q"} },
+		"TracingConfig":        func(b map[string]any) { b["TracingConfig"] = map[string]any{"Mode": "Active"} },
+		"FileSystemConfigs": func(b map[string]any) {
+			b["FileSystemConfigs"] = []map[string]any{{"Arn": "a", "LocalMountPath": "/mnt/x"}}
+		},
+		"ImageConfig":          func(b map[string]any) { b["ImageConfig"] = map[string]any{"Command": []string{"x"}} },
+		"Architectures":        func(b map[string]any) { b["Architectures"] = []string{"arm64"} },
+		"Publish":              func(b map[string]any) { b["Publish"] = true },
+		"SnapStart":            func(b map[string]any) { b["SnapStart"] = map[string]any{"ApplyOn": "PublishedVersions"} },
+		"EphemeralStorage":     func(b map[string]any) { b["EphemeralStorage"] = map[string]any{"Size": 1024} },
+		"CodeSigningConfigArn": func(b map[string]any) { b["CodeSigningConfigArn"] = "arn:aws:lambda:r:1:code-signing-config:c" },
+		"LoggingConfig":        func(b map[string]any) { b["LoggingConfig"] = map[string]any{"LogFormat": "JSON"} },
+		"Tags":                 func(b map[string]any) { b["Tags"] = map[string]string{"a": "b"} },
+		"TenancyConfig":        func(b map[string]any) { b["TenancyConfig"] = map[string]any{"TenantIsolationMode": "PER_TENANT"} },
+	}
+	for field, mutate := range cases {
+		t.Run(field, func(t *testing.T) {
+			e := newEnv(t)
+			_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", mutate))
+			requireCode(t, err, awserrors.ErrorInvalidParameterValueException)
+			assert.Zero(t, e.store.count())
+			_, _, getErr := e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
+			require.ErrorIs(t, getErr, function.ErrNotFound, "a refused request claims nothing")
+		})
+	}
+}
+
+func TestCreateFunction_AcceptsDefaultsEquivalentToAbsent(t *testing.T) {
+	e := newEnv(t)
+	rec, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) {
+		b["Architectures"] = []string{"x86_64"}
+		b["Publish"] = false
+		b["EphemeralStorage"] = map[string]any{"Size": 512}
+		b["TracingConfig"] = map[string]any{"Mode": "PassThrough"}
+		b["PackageType"] = "Zip"
+		b["Layers"] = []string{}
+		b["Tags"] = map[string]string{}
+		b["LoggingConfig"] = map[string]any{"LogFormat": "Text", "LogGroup": "/aws/lambda/hello"}
+	}))
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusCreated, rec.Code)
+}
+
+func TestCreateFunction_RefusesNonDefaultLoggingConfig(t *testing.T) {
+	for _, cfg := range []map[string]any{
+		{"LogGroup": "/custom/group"},
+		{"LogFormat": "Text", "LogGroup": "/aws/lambda/other"},
+		{"LogFormat": "Text", "ApplicationLogLevel": "DEBUG"},
+		{"SystemLogLevel": "WARN"},
+	} {
+		e := newEnv(t)
+		_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) { b["LoggingConfig"] = cfg }))
+		requireCode(t, err, awserrors.ErrorInvalidParameterValueException)
+		_, _, getErr := e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
+		require.ErrorIs(t, getErr, function.ErrNotFound, "%v claims nothing", cfg)
+	}
+}
+
+func TestFunctionResponses_CarryTheDefaultLoggingConfig(t *testing.T) {
+	e := newEnv(t)
+	rec, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+	var created lambda.FunctionConfiguration
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &created))
+	want := &lambda.LoggingConfig{LogFormat: aws.String("Text"), LogGroup: aws.String("/aws/lambda/hello")}
+	assert.Equal(t, want, created.LoggingConfig)
+
+	rec, err = e.do(http.MethodGet, "/2015-03-31/functions/hello", nil)
+	require.NoError(t, err)
+	var got lambda.GetFunctionOutput
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &got))
+	assert.Equal(t, want, got.Configuration.LoggingConfig)
+}
+
+func TestCreateFunction_ParameterValidation(t *testing.T) {
+	cases := map[string]func(map[string]any){
+		"bad name":        func(b map[string]any) { b["FunctionName"] = "has space" },
+		"arn name":        func(b map[string]any) { b["FunctionName"] = "arn:aws:lambda:r:1:function:x" },
+		"long name":       func(b map[string]any) { b["FunctionName"] = strings.Repeat("a", 65) },
+		"runtime":         func(b map[string]any) { b["Runtime"] = "python3.12" },
+		"no runtime":      func(b map[string]any) { delete(b, "Runtime") },
+		"package type":    func(b map[string]any) { b["PackageType"] = "Image" },
+		"no code":         func(b map[string]any) { delete(b, "Code") },
+		"empty zip":       func(b map[string]any) { b["Code"] = map[string]any{"ZipFile": []byte{}} },
+		"no handler":      func(b map[string]any) { delete(b, "Handler") },
+		"no role":         func(b map[string]any) { delete(b, "Role") },
+		"timeout low":     func(b map[string]any) { b["Timeout"] = 0 },
+		"timeout high":    func(b map[string]any) { b["Timeout"] = 901 },
+		"memory low":      func(b map[string]any) { b["MemorySize"] = 64 },
+		"memory high":     func(b map[string]any) { b["MemorySize"] = 10241 },
+		"cross account":   func(b map[string]any) { b["Role"] = foreignRoleARN },
+		"missing role":    func(b map[string]any) { b["Role"] = "arn:aws:iam::" + gwtest.AccountID + ":role/ghost" },
+		"not a role arn":  func(b map[string]any) { b["Role"] = "arn:aws:iam::" + gwtest.AccountID + ":user/alice" },
+		"malformed body":  func(b map[string]any) { b["Timeout"] = "soon" },
+		"empty env name":  func(b map[string]any) { b["Environment"] = map[string]any{"Variables": map[string]string{"": "x"}} },
+		"description len": func(b map[string]any) { b["Description"] = strings.Repeat("d", 257) },
+	}
+	for name, mutate := range cases {
+		t.Run(name, func(t *testing.T) {
+			e := newEnv(t)
+			_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", mutate))
+			requireCode(t, err, awserrors.ErrorInvalidParameterValueException)
+			assert.Zero(t, e.store.count())
+		})
+	}
+}
+
+func TestCreateFunction_RoleErrorsDoNotLeakAcrossAccounts(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) { b["Role"] = foreignRoleARN }))
+	requireCode(t, err, awserrors.ErrorInvalidParameterValueException)
+	assert.Contains(t, err.Error(), "cannot be assumed by Lambda")
+}
+
+func TestCreateFunction_RefusesBytesThatAreNotAZip(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) {
+		b["Code"] = map[string]any{"ZipFile": []byte("PK\x03\x04 not really a zip")}
+	}))
+	requireCode(t, err, awserrors.ErrorInvalidParameterValueException)
+	assert.Contains(t, err.Error(), "Could not unzip uploaded file")
+	_, _, err = e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
+	require.ErrorIs(t, err, function.ErrNotFound)
+	assert.Empty(t, e.store.objects)
+}
+
+func TestCreateFunction_RoleLambdaCannotAssumeRecordsNothing(t *testing.T) {
+	e := newEnv(t)
+	e.deps.Trusts = func(string, string) (bool, error) { return false, nil }
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	requireCode(t, err, awserrors.ErrorInvalidParameterValueException)
+	assert.Contains(t, err.Error(), "cannot be assumed by Lambda")
+	_, _, err = e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
+	require.ErrorIs(t, err, function.ErrNotFound)
+	assert.Empty(t, e.store.objects)
+}
+
+func TestCreateFunction_MissingRoleCheckFailsClosed(t *testing.T) {
+	for name, mutate := range map[string]func(*awsapi.Deps){
+		"no resolver":      func(d *awsapi.Deps) { d.Roles = nil },
+		"no trust checker": func(d *awsapi.Deps) { d.Trusts = nil },
+	} {
+		t.Run(name, func(t *testing.T) {
+			e := newEnv(t)
+			mutate(&e.deps)
+			_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+			requireCode(t, err, awserrors.ErrorServiceException)
+			_, _, err = e.meta.Get(t.Context(), gwtest.AccountID, gwtest.Region, "hello")
+			require.ErrorIs(t, err, function.ErrNotFound)
+		})
+	}
+}
+
+func TestGetFunction_DecryptsEnvAndLocatesStoredCode(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) {
+		b["Environment"] = map[string]any{"Variables": map[string]string{"MODE": "prod"}}
+	}))
+	require.NoError(t, err)
+
+	rec, err := e.do(http.MethodGet, "/2015-03-31/functions/hello", nil)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, rec.Code)
+	var out lambda.GetFunctionOutput
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out))
+	assert.Equal(t, "prod", *out.Configuration.Environment.Variables["MODE"])
+	assert.Equal(t, "S3", *out.Code.RepositoryType)
+	require.NotNil(t, out.Code.Location)
+	assert.Contains(t, *out.Code.Location, "https://code.test/")
+	assert.Contains(t, *out.Code.Location, "ttl=10m0s")
+
+	rec, err = e.do(http.MethodGet, "/2015-03-31/functions/hello?Qualifier=%24LATEST", nil)
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, rec.Code)
+	_, err = e.do(http.MethodGet, "/2015-03-31/functions/hello?Qualifier=3", nil)
+	requireCode(t, err, awserrors.ErrorResourceNotFoundException)
+}
+
+func TestGetFunction_AcceptsSameAccountARNForms(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+	full := "arn:aws:lambda:" + gwtest.Region + ":" + gwtest.AccountID + ":function:hello"
+	esc := func(s string) string { return strings.ReplaceAll(strings.ReplaceAll(s, ":", "%3A"), "$", "%24") }
+
+	for _, name := range []string{full, full + ":$LATEST", gwtest.AccountID + ":function:hello", "hello:$LATEST"} {
+		t.Run(name, func(t *testing.T) {
+			rec, err := e.do(http.MethodGet, "/2015-03-31/functions/"+esc(name), nil)
+			require.NoError(t, err)
+			require.Equal(t, http.StatusOK, rec.Code)
+			var out lambda.GetFunctionOutput
+			require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out))
+			assert.Equal(t, full, *out.Configuration.FunctionArn)
+		})
+	}
+
+	notFound := map[string]string{
+		"other account":   "arn:aws:lambda:" + gwtest.Region + ":999999999999:function:hello",
+		"other region":    "arn:aws:lambda:us-west-2:" + gwtest.AccountID + ":function:hello",
+		"partial foreign": "999999999999:function:hello",
+		"version":         full + ":3",
+	}
+	for label, name := range notFound {
+		t.Run(label, func(t *testing.T) {
+			_, err := e.do(http.MethodGet, "/2015-03-31/functions/"+esc(name), nil)
+			requireCode(t, err, awserrors.ErrorResourceNotFoundException)
+		})
+	}
+
+	_, err = e.do(http.MethodGet, "/2015-03-31/functions/"+esc(full+":$LATEST")+"?Qualifier=3", nil)
+	requireCode(t, err, awserrors.ErrorInvalidParameterValueException)
+	_, err = e.do(http.MethodGet, "/2015-03-31/functions/"+esc("arn:aws:s3:::bucket"), nil)
+	requireCode(t, err, awserrors.ErrorInvalidParameterValueException)
+}
+
+func TestGetFunction_NoLocationUntilCodeIsStored(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.meta.Create(t.Context(), gwtest.AccountID, gwtest.Region, function.Record{
+		FunctionID: "f1", FunctionName: "pending", Role: roleARN, Runtime: "provided.al2023", Handler: "h",
+		Timeout: 3, MemorySize: 128, Code: function.Code{RevisionID: "r1"},
+	})
+	require.NoError(t, err)
+
+	rec, err := e.do(http.MethodGet, "/2015-03-31/functions/pending", nil)
+	require.NoError(t, err)
+	var out lambda.GetFunctionOutput
+	require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out))
+	assert.Nil(t, out.Code.Location)
+	assert.Nil(t, out.Configuration.CodeSha256)
+	assert.Equal(t, "Pending", *out.Configuration.State)
+}
+
+func TestGetFunction_FailsWithoutAnOffHostCodeURL(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+	e.store.urlErr = codestore.ErrNoPresignEndpoint
+
+	rec, err := e.do(http.MethodGet, "/2015-03-31/functions/hello", nil)
+	requireCode(t, err, awserrors.ErrorServiceException)
+	assert.NotContains(t, rec.Body.String(), "Location")
+}
+
+func TestGetFunction_NotFoundNamesTheARN(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodGet, "/2015-03-31/functions/ghost", nil)
+	requireCode(t, err, awserrors.ErrorResourceNotFoundException)
+	assert.Contains(t, err.Error(), "Function not found: arn:aws:lambda:"+gwtest.Region+":"+gwtest.AccountID+":function:ghost")
+}
+
+func TestGetFunction_FunctionsAreRegional(t *testing.T) {
+	e := newEnv(t)
+	_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+	require.NoError(t, err)
+	e.h.Region = "us-west-2"
+	_, err = e.do(http.MethodGet, "/2015-03-31/functions/hello", nil)
+	requireCode(t, err, awserrors.ErrorResourceNotFoundException)
+}
+
+func TestAuthorization(t *testing.T) {
+	create := func(e *env) error {
+		_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", nil))
+		return err
+	}
+	get := func(e *env) error {
+		_, err := e.do(http.MethodGet, "/2015-03-31/functions/hello", nil)
+		return err
+	}
+
+	t.Run("create needs lambda:CreateFunction", func(t *testing.T) {
+		e := newEnv(t, gwtest.Statement("Allow", "lambda:GetFunction", "*"), gwtest.Statement("Allow", "iam:PassRole", "*"))
+		gwtest.AssertDenied(t, create(e))
+		assert.Zero(t, e.store.count())
+	})
+	t.Run("create needs iam:PassRole", func(t *testing.T) {
+		e := newEnv(t, gwtest.Statement("Allow", "lambda:*", "*"))
+		gwtest.AssertDenied(t, create(e))
+		assert.Zero(t, e.store.count())
+	})
+	t.Run("create is scoped to the function ARN", func(t *testing.T) {
+		arn := "arn:aws:lambda:" + gwtest.Region + ":" + gwtest.AccountID + ":function:hello"
+		e := newEnv(t,
+			gwtest.Statement("Allow", "lambda:*", "*"),
+			gwtest.Statement("Allow", "iam:PassRole", "*"),
+			gwtest.Statement("Deny", "lambda:CreateFunction", arn))
+		gwtest.AssertDenied(t, create(e))
+	})
+	t.Run("create passes with both grants", func(t *testing.T) {
+		e := newEnv(t, gwtest.Statement("Allow", "lambda:CreateFunction", "*"), gwtest.Statement("Allow", "iam:PassRole", "*"))
+		require.NoError(t, create(e))
+	})
+	t.Run("get needs lambda:GetFunction", func(t *testing.T) {
+		e := newEnv(t, gwtest.Statement("Allow", "lambda:CreateFunction", "*"))
+		gwtest.AssertDenied(t, get(e))
+	})
+	t.Run("a denied caller does not learn a role is absent", func(t *testing.T) {
+		e := newEnv(t, gwtest.Statement("Allow", "lambda:*", "*"))
+		_, err := e.do(http.MethodPost, "/2015-03-31/functions", createBody("hello", func(b map[string]any) {
+			b["Role"] = "arn:aws:iam::" + gwtest.AccountID + ":role/ghost"
+		}))
+		gwtest.AssertDenied(t, err)
+	})
+}
+
+func TestRegistration_ResolvesActionsAndInventory(t *testing.T) {
+	reg := awsapi.NewRegistration(awsapi.Deps{})
+	assert.Equal(t, []string{"CreateFunction", "GetFunction"}, reg.Inventory.Registered)
+	assert.Equal(t, "CreateFunction", reg.ResolveAction(httptest.NewRequest(http.MethodPost, "/2015-03-31/functions", nil)))
+	assert.Equal(t, "GetFunction", reg.ResolveAction(httptest.NewRequest(http.MethodGet, "/2015-03-31/functions/x", nil)))
+	assert.Empty(t, reg.ResolveAction(httptest.NewRequest(http.MethodDelete, "/2015-03-31/functions/x", nil)))
+}

spinifex/domains/lambda/awsapi/registration.go

@@ -0,0 +1,196 @@
+// Package awsapi serves the Lambda REST-JSON control plane (API version
+// 2015-03-31) for the actions this build supports.
+package awsapi
+
+import (
+	"context"
+	"crypto/cipher"
+	"errors"
+	"io"
+	"log/slog"
+	"net/http"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/private/protocol/json/jsonutil"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/codestore"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/function"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/passrole"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/envelope"
+)
+
+// ServiceName is the SigV4 credential-scope service Lambda clients sign for.
+const ServiceName = "lambda"
+
+// servicePrincipal is the service a function execution role is passed to.
+const servicePrincipal = "lambda.amazonaws.com"
+
+// CodeStore is what the handlers need of the Regional code store.
+type CodeStore interface {
+	Put(ctx context.Context, ref codestore.Ref, code []byte) (string, error)
+	PresignGet(ctx context.Context, ref codestore.Ref, sha256Hex string, ttl time.Duration) (string, error)
+}
+
+var _ CodeStore = (*codestore.Store)(nil)
+
+// CodeStoreFunc opens the code store on first use; it fails while the store is
+// not ready.
+type CodeStoreFunc func(ctx context.Context) (CodeStore, error)
+
+// Deps are the composed capabilities the Lambda control plane dispatches to.
+type Deps struct {
+	Meta  function.Client
+	Codes CodeStoreFunc
+	Roles passrole.Resolver
+	// Trusts reports whether the role's trust policy lets Lambda assume it for
+	// the account; false is a refusal, an error an outage.
+	Trusts func(accountID, roleARN string) (bool, error)
+	// AEAD seals function environment variables; it is the master key's cipher.
+	AEAD cipher.AEAD
+}
+
+// NewRegistration returns the Lambda REST-JSON registration.
+func NewRegistration(deps Deps) dispatch.Registration {
+	return dispatch.Registration{
+		Service:       ServiceName,
+		ResolveAction: resolveAction,
+		Dispatch:      deps.dispatch,
+		Errors:        dispatch.ErrorEnvelopeRESTJSON,
+		Inventory:     dispatch.Inventory{Registered: actionNames()},
+	}
+}
+
+func resolveAction(r *http.Request) string {
+	action, _, _, _ := router.Lookup(r.Method, r.URL.EscapedPath())
+	return action
+}
+
+func readBody(r *http.Request) ([]byte, error) {
+	body, err := io.ReadAll(io.LimitReader(r.Body, maxBodyBytes+1))
+	if err != nil {
+		return nil, errors.New(awserrors.ErrorInvalidParameterValueException)
+	}
+	if len(body) > maxBodyBytes {
+		return nil, errors.New(awserrors.ErrorRequestEntityTooLarge)
+	}
+	return body, nil
+}
+
+// functionARN names the function the request is about, or "*" when the request
+// does not yet name a legal one, so a malformed request stays the handler's
+// validation fault rather than becoming an authorization failure.
+func functionARN(region, accountID, name string) string {
+	if region == "" || accountID == "" || !nameRE.MatchString(name) {
+		return "*"
+	}
+	return arn.FormatLambdaFunction(region, accountID, name)
+}
+
+// serve resolves the route, authorizes the request, then runs the action.
+func (d Deps) serve(w http.ResponseWriter, inv dispatch.Invocation) error {
+	r := inv.Request
+	action, params, handler, ok := router.Lookup(r.Method, r.URL.EscapedPath())
+	if !ok {
+		return errors.New(awserrors.ErrorInvalidAction)
+	}
+	if inv.AccountID == "" {
+		slog.ErrorContext(r.Context(), "Lambda dispatch: no account ID in auth context")
+		return errors.New(awserrors.ErrorInternalError)
+	}
+	body, err := readBody(r)
+	if err != nil {
+		return err
+	}
+	c := &call{AccountID: inv.AccountID, Region: inv.Region, Params: params, Query: r.URL.Query(), Body: body}
+
+	// The region is the gateway's own, never the caller-supplied credential
+	// scope, so a caller cannot sign for another region to slide out from
+	// under a Deny.
+	resource := "*"
+	var decodeErr error
+	switch action {
+	case "CreateFunction":
+		c.Create, decodeErr = decodeCreate(body)
+		if decodeErr == nil {
+			resource = functionARN(inv.Region, inv.AccountID, aws.StringValue(c.Create.FunctionName))
+		}
+	case "GetFunction":
+		if name, _, err := resolveFunctionName(inv.Region, inv.AccountID, params[0]); err == nil {
+			resource = functionARN(inv.Region, inv.AccountID, name)
+		}
+	}
+	if err := inv.Authorize(ServiceName, action, []string{resource}, nil); err != nil {
+		return err
+	}
+	if decodeErr != nil {
+		return decodeErr
+	}
+	if action == "CreateFunction" {
+		if err := d.checkPassRole(inv, aws.StringValue(c.Create.Role)); err != nil {
+			return err
+		}
+	}
+	if d.Meta == nil || d.Codes == nil {
+		return errors.New(awserrors.ErrorServiceException)
+	}
+
+	out, err := handler(r.Context(), d, c)
+	if err != nil {
+		return err
+	}
+	writeJSON(w, out)
+	return nil
+}
+
+func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
+	return d.serve(w, inv)
+}
+
+// checkPassRole enforces iam:PassRole on the execution role and that its trust
+// policy admits Lambda. Only the exact stored ARN resolves, and a missing
+// resolver fails closed.
+func (d Deps) checkPassRole(inv dispatch.Invocation, roleARN string) error {
+	if roleARN == "" {
+		return nil
+	}
+	if d.Roles == nil || d.Trusts == nil {
+		return errors.New(awserrors.ErrorServiceException)
+	}
+	role, err := passrole.Check(d.Roles, passrole.AuthorizeFunc(inv.Authorize), inv.AccountID, roleARN, servicePrincipal)
+	switch {
+	case err == nil:
+		trusted, terr := d.Trusts(inv.AccountID, role.ARN)
+		if terr != nil {
+			return terr
+		}
+		if !trusted {
+			return invalid("The role defined for the function cannot be assumed by Lambda.")
+		}
+		return nil
+	case errors.Is(err, passrole.ErrMalformed), errors.Is(err, passrole.ErrForeignAccount):
+		return invalid("The role defined for the function cannot be assumed by Lambda.")
+	case errors.Is(err, passrole.ErrNotFound):
+		return invalid("The role defined for the function cannot be assumed by Lambda.")
+	default:
+		return err
+	}
+}
+
+func writeJSON(w http.ResponseWriter, out reply) {
+	body, err := jsonutil.BuildJSON(out.Body)
+	if err != nil {
+		slog.Error("Failed to marshal Lambda response", "err", err)
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusInternalServerError)
+		_, _ = w.Write(envelope.JSONBody(awserrors.ErrorServiceException, "failed to marshal response"))
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(out.Status)
+	if _, err := w.Write(body); err != nil {
+		slog.Error("Failed to write Lambda response", "err", err)
+	}
+}

spinifex/domains/lambda/awsapi/routes.go

@@ -0,0 +1,56 @@
+package awsapi
+
+import (
+	"context"
+	"net/url"
+	"slices"
+
+	"github.com/aws/aws-sdk-go/service/lambda"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/rest"
+)
+
+// call is one authorized request as a handler sees it.
+type call struct {
+	AccountID string
+	Region    string
+	Params    []string
+	Query     url.Values
+	Body      []byte
+	// Create is the decoded CreateFunction body, decoded once ahead of the
+	// authorization gate and the PassRole check that both read it.
+	Create *lambda.CreateFunctionInput
+}
+
+// reply is a handler's success: the HTTP status and the SDK shape to encode.
+type reply struct {
+	Status int
+	Body   any
+}
+
+type route = rest.Route[routeHandler]
+
+type routeHandler func(ctx context.Context, d Deps, c *call) (reply, error)
+
+// routes is the dispatch table. Lambda API version 2015-03-31.
+var routes = []route{
+	{Method: "POST", Pattern: "/2015-03-31/functions", Action: "CreateFunction",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.createFunction(ctx, c)
+		}},
+	{Method: "GET", Pattern: "/2015-03-31/functions/{FunctionName}", Action: "GetFunction",
+		Handler: func(ctx context.Context, d Deps, c *call) (reply, error) {
+			return d.getFunction(ctx, c)
+		}},
+}
+
+// actionNames returns the distinct actions in routes in stable order.
+func actionNames() []string {
+	names := make([]string, 0, len(routes))
+	for _, r := range routes {
+		names = append(names, r.Action)
+	}
+	slices.Sort(names)
+	return slices.Compact(names)
+}
+
+var router = rest.NewRouter("lambda", routes)

spinifex/domains/lambda/awsapi/validate.go

@@ -0,0 +1,259 @@
+package awsapi
+
+import (
+	"archive/zip"
+	"bytes"
+	"encoding/json"
+	"reflect"
+	"regexp"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/lambda"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/codestore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+const (
+	supportedRuntime  = "provided.al2023"
+	defaultTimeout    = 3
+	maxTimeout        = 900
+	defaultMemory     = 128
+	maxMemory         = 10240
+	maxDescription    = 256
+	maxEnvironmentLen = 4096
+	// maxBodyBytes bounds a CreateFunction body: the base64 of the largest
+	// code copy plus room for the rest of the document.
+	maxBodyBytes = (codestore.MaxCodeBytes/3+1)*4 + 1<<20
+)
+
+var nameRE = regexp.MustCompile(`^[a-zA-Z0-9-_]{1,64}$`)
+
+func invalid(format string, args ...any) error {
+	return awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, format, args...)
+}
+
+func unsupported(field string) error {
+	return invalid("%s is not supported", field)
+}
+
+// decodeCreate reads a CreateFunction body strictly: a member this build does
+// not model is refused rather than silently dropped.
+func decodeCreate(body []byte) (*lambda.CreateFunctionInput, error) {
+	in := new(lambda.CreateFunctionInput)
+	dec := json.NewDecoder(bytes.NewReader(body))
+	dec.DisallowUnknownFields()
+	if err := dec.Decode(in); err != nil {
+		return nil, invalid("the request body is not a valid CreateFunction document: %v", err)
+	}
+	return in, nil
+}
+
+// createParams is a CreateFunction request after validation, defaults applied.
+type createParams struct {
+	Name        string
+	Role        string
+	Handler     string
+	Description string
+	TimeoutSec  int64
+	MemorySize  int64
+	Code        []byte
+	Env         map[string]string
+}
+
+// isSet reports whether a modelled optional member carries anything: a non-nil
+// pointer to a non-zero value, a non-empty list or map, or a struct with any
+// such member.
+func isSet(v any) bool {
+	return valueSet(reflect.ValueOf(v))
+}
+
+func valueSet(v reflect.Value) bool {
+	switch v.Kind() {
+	case reflect.Pointer, reflect.Interface:
+		return !v.IsNil() && valueSet(v.Elem())
+	case reflect.Slice, reflect.Map:
+		return v.Len() > 0
+	case reflect.Struct:
+		for i := range v.NumField() {
+			if v.Type().Field(i).IsExported() && valueSet(v.Field(i)) {
+				return true
+			}
+		}
+		return false
+	case reflect.Invalid:
+		return false
+	default:
+		return !v.IsZero()
+	}
+}
+
+// refuseUnsupported names the first supplied member this build cannot honour.
+func refuseUnsupported(in *lambda.CreateFunctionInput) error {
+	checks := []struct {
+		field string
+		set   bool
+	}{
+		{"Code.S3Bucket", in.Code != nil && isSet(in.Code.S3Bucket)},
+		{"Code.S3Key", in.Code != nil && isSet(in.Code.S3Key)},
+		{"Code.S3ObjectVersion", in.Code != nil && isSet(in.Code.S3ObjectVersion)},
+		{"Code.ImageUri", in.Code != nil && isSet(in.Code.ImageUri)},
+		{"Layers", isSet(in.Layers)},
+		{"VpcConfig", isSet(in.VpcConfig)},
+		{"KMSKeyArn", isSet(in.KMSKeyArn)},
+		{"DeadLetterConfig", isSet(in.DeadLetterConfig)},
+		{"TracingConfig", in.TracingConfig != nil && aws.StringValue(in.TracingConfig.Mode) == lambda.TracingModeActive},
+		{"FileSystemConfigs", isSet(in.FileSystemConfigs)},
+		{"ImageConfig", isSet(in.ImageConfig)},
+		{"Architectures", isSet(in.Architectures) && (len(in.Architectures) != 1 || aws.StringValue(in.Architectures[0]) != lambda.ArchitectureX8664)},
+		{"Publish", aws.BoolValue(in.Publish)},
+		{"SnapStart", in.SnapStart != nil && aws.StringValue(in.SnapStart.ApplyOn) != "" && aws.StringValue(in.SnapStart.ApplyOn) != lambda.SnapStartApplyOnNone},
+		{"EphemeralStorage", in.EphemeralStorage != nil && in.EphemeralStorage.Size != nil && aws.Int64Value(in.EphemeralStorage.Size) != 512},
+		{"CodeSigningConfigArn", isSet(in.CodeSigningConfigArn)},
+		{"Tags", isSet(in.Tags)},
+	}
+	for _, c := range checks {
+		if c.set {
+			return unsupported(c.field)
+		}
+	}
+	return nil
+}
+
+// validateCreate checks a decoded CreateFunction and applies the defaults.
+// resolveFunctionName accepts the GetFunction forms AWS does: a name, a partial
+// ARN (account:function:name) or a full ARN, each with an optional qualifier.
+// Another account or Region is reported not found under the ARN it named.
+func resolveFunctionName(region, accountID, raw string) (name, qualifier string, err error) {
+	parts := strings.Split(raw, ":")
+	qualified := false
+	switch {
+	case len(parts) <= 2:
+		name = parts[0]
+		if len(parts) == 2 {
+			qualifier, qualified = parts[1], true
+		}
+	case len(parts) <= 4 && parts[1] == "function":
+		if parts[0] != accountID {
+			return "", "", awserrors.Errorf(awserrors.ErrorResourceNotFoundException, "Function not found: %s",
+				arn.FormatLambdaFunction(region, parts[0], strings.Join(parts[2:], ":")))
+		}
+		name = parts[2]
+		if len(parts) == 4 {
+			qualifier, qualified = parts[3], true
+		}
+	case len(parts) >= 7 && len(parts) <= 8 && parts[0] == "arn" && parts[2] == "lambda" && parts[5] == "function":
+		if parts[1] != "aws" || parts[3] != region || parts[4] != accountID {
+			return "", "", awserrors.Errorf(awserrors.ErrorResourceNotFoundException, "Function not found: %s", raw)
+		}
+		name = parts[6]
+		if len(parts) == 8 {
+			qualifier, qualified = parts[7], true
+		}
+	default:
+		return "", "", invalidFunctionName()
+	}
+	if !nameRE.MatchString(name) || (qualified && qualifier == "") {
+		return "", "", invalidFunctionName()
+	}
+	return name, qualifier, nil
+}
+
+func invalidFunctionName() error {
+	return invalid("1 validation error detected: Value at 'functionName' failed to satisfy constraint: Member must satisfy regular expression pattern: (arn:(aws[a-zA-Z-]*)?:lambda:)?([a-z]{2}(-gov)?-[a-z]+-\\d{1}:)?(\\d{12}:)?(function:)?([a-zA-Z0-9-_]+)(:(\\$LATEST|[a-zA-Z0-9-_]+))?")
+}
+
+// logGroup is the log group AWS names for a function by default.
+func logGroup(name string) string { return "/aws/lambda/" + name }
+
+// defaultLogging reports whether c is absent or spells AWS's default exactly;
+// without a logs capability any other value would be inert.
+func defaultLogging(c *lambda.LoggingConfig, name string) bool {
+	if c == nil {
+		return true
+	}
+	format, group := aws.StringValue(c.LogFormat), aws.StringValue(c.LogGroup)
+	return (format == "" || format == lambda.LogFormatText) && (group == "" || group == logGroup(name)) &&
+		c.ApplicationLogLevel == nil && c.SystemLogLevel == nil
+}
+
+func validateCreate(in *lambda.CreateFunctionInput) (*createParams, error) {
+	if err := refuseUnsupported(in); err != nil {
+		return nil, err
+	}
+	name := aws.StringValue(in.FunctionName)
+	if strings.HasPrefix(name, "arn:") || !nameRE.MatchString(name) {
+		return nil, invalid("1 validation error detected: Value at 'functionName' failed to satisfy constraint: Member must satisfy regular expression pattern: [a-zA-Z0-9-_]+ and have length 1 to 64")
+	}
+	if !defaultLogging(in.LoggingConfig, name) {
+		return nil, unsupported("A LoggingConfig other than LogFormat Text with log group " + logGroup(name))
+	}
+	if aws.StringValue(in.Runtime) != supportedRuntime {
+		return nil, invalid("Runtime must be %s", supportedRuntime)
+	}
+	if pt := aws.StringValue(in.PackageType); pt != "" && pt != lambda.PackageTypeZip {
+		return nil, invalid("PackageType must be %s", lambda.PackageTypeZip)
+	}
+	if in.Code == nil || len(in.Code.ZipFile) == 0 {
+		return nil, invalid("Code.ZipFile is required")
+	}
+	if len(in.Code.ZipFile) > codestore.MaxCodeBytes {
+		return nil, awserrors.Errorf(awserrors.ErrorRequestEntityTooLarge, "Code.ZipFile exceeds %d bytes", codestore.MaxCodeBytes)
+	}
+	if _, err := zip.NewReader(bytes.NewReader(in.Code.ZipFile), int64(len(in.Code.ZipFile))); err != nil {
+		return nil, invalid("Could not unzip uploaded file. Please check your file, then try to upload again.")
+	}
+	handler := aws.StringValue(in.Handler)
+	if handler == "" {
+		return nil, invalid("Handler is required")
+	}
+	role := aws.StringValue(in.Role)
+	if role == "" {
+		return nil, invalid("Role is required")
+	}
+	limit, memory := int64(defaultTimeout), int64(defaultMemory)
+	if in.Timeout != nil {
+		limit = *in.Timeout
+	}
+	if limit < 1 || limit > maxTimeout {
+		return nil, invalid("Timeout must be between 1 and %d", maxTimeout)
+	}
+	if in.MemorySize != nil {
+		memory = *in.MemorySize
+	}
+	if memory < 128 || memory > maxMemory {
+		return nil, invalid("MemorySize must be between 128 and %d", maxMemory)
+	}
+	desc := aws.StringValue(in.Description)
+	if len(desc) > maxDescription {
+		return nil, invalid("Description must be at most %d characters", maxDescription)
+	}
+	env, err := envVars(in.Environment)
+	if err != nil {
+		return nil, err
+	}
+	return &createParams{
+		Name: name, Role: role, Handler: handler, Description: desc,
+		TimeoutSec: limit, MemorySize: memory, Code: in.Code.ZipFile, Env: env,
+	}, nil
+}
+
+func envVars(e *lambda.Environment) (map[string]string, error) {
+	if e == nil || len(e.Variables) == 0 {
+		return nil, nil
+	}
+	vars := make(map[string]string, len(e.Variables))
+	size := 0
+	for k, v := range e.Variables {
+		if k == "" {
+			return nil, invalid("Environment variable names must not be empty")
+		}
+		vars[k] = aws.StringValue(v)
+		size += len(k) + len(vars[k])
+	}
+	if size > maxEnvironmentLen {
+		return nil, invalid("Environment variables exceed %d bytes", maxEnvironmentLen)
+	}
+	return vars, nil
+}

spinifex/domains/lambda/codestore/credentials.go

@@ -0,0 +1,113 @@
+package codestore
+
+import (
+	"errors"
+	"fmt"
+	"log/slog"
+	"sync"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/service/sts"
+	telemetry "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+)
+
+const (
+	// refreshMargin re-assumes this long before a session expires.
+	refreshMargin = 5 * time.Minute
+	// retryInterval spaces re-assume attempts while an old session is still valid.
+	retryInterval = 30 * time.Second
+)
+
+// AssumeFunc mints a code-store session. The composition layer binds it to the
+// in-process STS entry point for one Region.
+type AssumeFunc func() (*sts.AssumeRoleOutput, error)
+
+// sessionProvider re-assumes the role ahead of expiry. A failed re-assume keeps
+// the current session in use until it expires, then fails closed.
+type sessionProvider struct {
+	assume AssumeFunc
+	now    func() time.Time
+
+	mu        sync.Mutex
+	value     credentials.Value
+	expires   time.Time
+	lastError time.Time
+	// minRemaining makes the next Retrieve re-assume unless the session has at
+	// least this long left; it is cleared by that Retrieve.
+	minRemaining time.Duration
+}
+
+var (
+	_ credentials.Provider = (*sessionProvider)(nil)
+	_ credentials.Expirer  = (*sessionProvider)(nil)
+)
+
+func newSessionProvider(assume AssumeFunc, now func() time.Time) *sessionProvider {
+	return &sessionProvider{assume: assume, now: now}
+}
+
+func (p *sessionProvider) Retrieve() (credentials.Value, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	now := p.now()
+	valid := p.value.AccessKeyID != "" && now.Before(p.expires)
+	need := p.minRemaining
+	p.minRemaining = 0
+	if need > 0 {
+		if valid && p.expires.Sub(now) >= need {
+			return p.value, nil
+		}
+	} else if valid && (now.Before(p.expires.Add(-refreshMargin)) || now.Before(p.lastError.Add(retryInterval))) {
+		return p.value, nil
+	}
+
+	out, err := p.assume()
+	if err == nil && (out == nil || out.Credentials == nil) {
+		err = errors.New("empty session")
+	}
+	if err != nil {
+		p.lastError = now
+		if valid {
+			slog.Warn("Lambda code store: re-assume failed; keeping current session",
+				"remaining_ms", telemetry.Millis(p.expires.Sub(now)), "err", err)
+			return p.value, nil
+		}
+		return credentials.Value{}, fmt.Errorf("assume code-store role: %w", err)
+	}
+
+	c := out.Credentials
+	p.value = credentials.Value{
+		AccessKeyID:     aws.StringValue(c.AccessKeyId),
+		SecretAccessKey: aws.StringValue(c.SecretAccessKey),
+		SessionToken:    aws.StringValue(c.SessionToken),
+		ProviderName:    "LambdaCodeStoreSession",
+	}
+	p.expires = aws.TimeValue(c.Expiration)
+	p.lastError = time.Time{}
+	return p.value, nil
+}
+
+// IsExpired reports true inside the refresh margin so the SDK calls Retrieve.
+func (p *sessionProvider) IsExpired() bool {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	return p.value.AccessKeyID == "" || !p.now().Before(p.expires.Add(-refreshMargin))
+}
+
+// ExpiresAt is when the current session stops working.
+func (p *sessionProvider) ExpiresAt() time.Time {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	return p.expires
+}
+
+// requireRemaining makes the next Retrieve draw a new session unless the
+// current one outlasts d.
+func (p *sessionProvider) requireRemaining(d time.Duration) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.minRemaining = d
+}

spinifex/domains/lambda/codestore/credentials_test.go

@@ -0,0 +1,94 @@
+package codestore
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/sts"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type fakeAssume struct {
+	calls   int
+	err     error
+	expires time.Time
+}
+
+func (f *fakeAssume) assume() (*sts.AssumeRoleOutput, error) {
+	f.calls++
+	if f.err != nil {
+		return nil, f.err
+	}
+	return &sts.AssumeRoleOutput{Credentials: &sts.Credentials{
+		AccessKeyId:     aws.String("ASIA" + string(rune('A'+f.calls))),
+		SecretAccessKey: aws.String("secret"),
+		SessionToken:    aws.String("token"),
+		Expiration:      aws.Time(f.expires),
+	}}, nil
+}
+
+type clock struct{ t time.Time }
+
+func (c *clock) now() time.Time { return c.t }
+
+func TestSessionProvider_ReusesUntilRefreshMargin(t *testing.T) {
+	c := &clock{t: time.Unix(1_000_000, 0)}
+	a := &fakeAssume{expires: c.t.Add(time.Hour)}
+	p := newSessionProvider(a.assume, c.now)
+
+	first, err := p.Retrieve()
+	require.NoError(t, err)
+	c.t = c.t.Add(54 * time.Minute)
+	assert.False(t, p.IsExpired())
+	again, err := p.Retrieve()
+	require.NoError(t, err)
+	assert.Equal(t, first.AccessKeyID, again.AccessKeyID)
+	assert.Equal(t, 1, a.calls)
+
+	c.t = c.t.Add(2 * time.Minute)
+	a.expires = c.t.Add(time.Hour)
+	assert.True(t, p.IsExpired())
+	fresh, err := p.Retrieve()
+	require.NoError(t, err)
+	assert.NotEqual(t, first.AccessKeyID, fresh.AccessKeyID)
+	assert.Equal(t, 2, a.calls)
+}
+
+// A failed re-assume keeps the still-valid session, spaces retries, and fails
+// closed once the session has expired.
+func TestSessionProvider_KeepsSessionOnFailureUntilExpiry(t *testing.T) {
+	c := &clock{t: time.Unix(1_000_000, 0)}
+	a := &fakeAssume{expires: c.t.Add(time.Hour)}
+	p := newSessionProvider(a.assume, c.now)
+	first, err := p.Retrieve()
+	require.NoError(t, err)
+
+	a.err = errors.New("sts unavailable")
+	c.t = c.t.Add(58 * time.Minute)
+	kept, err := p.Retrieve()
+	require.NoError(t, err)
+	assert.Equal(t, first.AccessKeyID, kept.AccessKeyID)
+	assert.Equal(t, 2, a.calls)
+
+	c.t = c.t.Add(10 * time.Second)
+	_, err = p.Retrieve()
+	require.NoError(t, err)
+	assert.Equal(t, 2, a.calls, "no re-assume inside the retry interval")
+
+	c.t = c.t.Add(2 * time.Minute)
+	_, err = p.Retrieve()
+	require.Error(t, err)
+	assert.Equal(t, 3, a.calls)
+}
+
+func TestSessionProvider_FirstFailureFails(t *testing.T) {
+	a := &fakeAssume{err: errors.New("AccessDenied")}
+	p := newSessionProvider(a.assume, time.Now)
+
+	_, err := p.Retrieve()
+	require.Error(t, err)
+	assert.True(t, p.IsExpired())
+}

spinifex/domains/lambda/codestore/manager.go

@@ -0,0 +1,74 @@
+package codestore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"sync"
+	"time"
+
+	telemetry "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+)
+
+// ErrNotReady means the code store is not provisioned or its role is not
+// assumable yet; callers report ServiceException.
+var ErrNotReady = errors.New("lambda code store not ready")
+
+const (
+	initialBackoff = time.Second
+	maxBackoff     = 5 * time.Minute
+)
+
+// Manager provisions and opens the store on first use, so a process that never
+// uses Lambda creates nothing. After a failure, calls inside the capped backoff
+// window fail fast with the last reason instead of retrying.
+type Manager struct {
+	provision func(context.Context) error
+	open      func() (*Store, error)
+	now       func() time.Time
+
+	mu      sync.Mutex
+	store   *Store
+	reason  error
+	backoff time.Duration
+	retryAt time.Time
+}
+
+// NewManager binds the bounded provisioning step and the scoped opener. It does
+// no I/O.
+func NewManager(provision func(context.Context) error, open func() (*Store, error)) *Manager {
+	return &Manager{provision: provision, open: open, now: time.Now}
+}
+
+// Store returns the opened store, initialising it on first use. An ErrNotOwned
+// refusal is retried after backoff, so an operator's fix takes effect without a
+// restart.
+func (m *Manager) Store(ctx context.Context) (*Store, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.store != nil {
+		return m.store, nil
+	}
+	if now := m.now(); now.Before(m.retryAt) {
+		return nil, fmt.Errorf("%w: %w", ErrNotReady, m.reason)
+	}
+
+	store, err := m.attempt(ctx)
+	if err == nil {
+		m.store, m.reason, m.backoff = store, nil, 0
+		slog.InfoContext(ctx, "Lambda code store ready")
+		return store, nil
+	}
+	m.backoff = min(max(m.backoff*2, initialBackoff), maxBackoff)
+	m.reason, m.retryAt = err, m.now().Add(m.backoff)
+	slog.WarnContext(ctx, "Lambda code store not ready", "err", err, "retry_ms", telemetry.Millis(m.backoff))
+	return nil, fmt.Errorf("%w: %w", ErrNotReady, err)
+}
+
+func (m *Manager) attempt(ctx context.Context) (*Store, error) {
+	if err := m.provision(ctx); err != nil {
+		return nil, err
+	}
+	return m.open()
+}

spinifex/domains/lambda/codestore/presign_test.go

@@ -0,0 +1,158 @@
+package codestore
+
+import (
+	"errors"
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	ststypes "github.com/aws/aws-sdk-go/service/sts"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestStore_PresignGet(t *testing.T) {
+	sess, err := session.NewSession(&aws.Config{
+		Endpoint:         aws.String("https://predastore.test:8443"),
+		Region:           aws.String("ap-southeast-2"),
+		Credentials:      credentials.NewStaticCredentials("AKIATEST", "secret", "session-token"),
+		S3ForcePathStyle: aws.Bool(true),
+	})
+	require.NoError(t, err)
+	s := newStore(testRegion, objectstore.NewMemoryObjectStore())
+	s.presign = presignWith(s3.New(sess))
+	sum := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+
+	raw, err := s.PresignGet(t.Context(), testRef, sum, 10*time.Minute)
+	require.NoError(t, err)
+
+	u, err := url.Parse(raw)
+	require.NoError(t, err)
+	assert.Equal(t, "predastore.test:8443", u.Host)
+	assert.Equal(t, "/"+BucketName(testRegion)+"/"+copyKey(sum), u.Path)
+	q := u.Query()
+	assert.Equal(t, "600", q.Get("X-Amz-Expires"))
+	assert.Equal(t, "session-token", q.Get("X-Amz-Security-Token"))
+	assert.NotEmpty(t, q.Get("X-Amz-Signature"))
+}
+
+func TestStore_PresignGetRefusals(t *testing.T) {
+	s := newStore(testRegion, objectstore.NewMemoryObjectStore())
+	sum := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+
+	_, err := s.PresignGet(t.Context(), testRef, sum, time.Minute)
+	require.Error(t, err, "a store without a signer cannot presign")
+
+	s.presign = func(*s3.GetObjectInput, time.Duration) (string, error) { return "unused", nil }
+	_, err = s.PresignGet(t.Context(), testRef, "not-a-digest", time.Minute)
+	require.ErrorIs(t, err, ErrInvalidRef)
+}
+
+func presignStoreOn(t *testing.T, c *clock, a *fakeAssume) *Store {
+	t.Helper()
+	provider := newSessionProvider(a.assume, c.now)
+	creds := credentials.NewCredentials(provider)
+	sess, err := session.NewSession(&aws.Config{
+		Endpoint: aws.String("https://predastore.test:8443"), Region: aws.String("ap-southeast-2"),
+		Credentials: creds, S3ForcePathStyle: aws.Bool(true),
+	})
+	require.NoError(t, err)
+	s := newStore(testRegion, objectstore.NewMemoryObjectStore())
+	s.presign = presignWith(s3.New(sess))
+	s.creds, s.provider, s.now = creds, provider, c.now
+	return s
+}
+
+func signedKeyID(t *testing.T, raw string) string {
+	t.Helper()
+	u, err := url.Parse(raw)
+	require.NoError(t, err)
+	return strings.SplitN(u.Query().Get("X-Amz-Credential"), "/", 2)[0]
+}
+
+func TestStore_PresignGetRefreshesAShortSession(t *testing.T) {
+	sum := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+	c := &clock{t: time.Unix(1_000_000, 0)}
+	a := &fakeAssume{expires: c.t.Add(7 * time.Minute)}
+	s := presignStoreOn(t, c, a)
+	_, err := s.creds.Get()
+	require.NoError(t, err)
+	first := "ASIAB"
+	require.Equal(t, 1, a.calls)
+
+	a.expires = c.t.Add(time.Hour)
+	raw, err := s.PresignGet(t.Context(), testRef, sum, 10*time.Minute)
+	require.NoError(t, err)
+	assert.Equal(t, 2, a.calls, "a 7 minute session is re-assumed before a 10 minute signature")
+	assert.NotEqual(t, first, signedKeyID(t, raw))
+	assert.Equal(t, "ASIAC", signedKeyID(t, raw), "the URL is signed with the new session")
+
+	_, err = s.PresignGet(t.Context(), testRef, sum, 10*time.Minute)
+	require.NoError(t, err)
+	assert.Equal(t, 2, a.calls, "a long session is reused")
+}
+
+func TestStore_PresignGetRefusesWhenSessionStaysShort(t *testing.T) {
+	sum := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+	c := &clock{t: time.Unix(1_000_000, 0)}
+	a := &fakeAssume{expires: c.t.Add(7 * time.Minute)}
+	s := presignStoreOn(t, c, a)
+
+	_, err := s.PresignGet(t.Context(), testRef, sum, 10*time.Minute)
+	require.Error(t, err, "a re-assume that is still short signs nothing")
+	assert.Equal(t, 2, a.calls)
+
+	a.err = errors.New("AccessDenied")
+	_, err = s.PresignGet(t.Context(), testRef, sum, 10*time.Minute)
+	require.Error(t, err, "a failed re-assume does not fall back to the short session")
+}
+
+// Open signs download URLs for the off-host endpoint while its own reads and
+// writes keep the local one, so a client outside the node can fetch the copy.
+func TestOpen_PresignsForTheOffHostEndpoint(t *testing.T) {
+	assume := func() (*ststypes.AssumeRoleOutput, error) {
+		return &ststypes.AssumeRoleOutput{Credentials: &ststypes.Credentials{
+			AccessKeyId:     aws.String("ASIATEST"),
+			SecretAccessKey: aws.String("secret"),
+			SessionToken:    aws.String("session-token"),
+			Expiration:      aws.Time(time.Now().Add(time.Hour)),
+		}}, nil
+	}
+	s, err := Open(OpenConfig{
+		Region:          testRegion,
+		Endpoint:        "https://127.0.0.1:8443",
+		SigningRegion:   "ap-southeast-2",
+		Assume:          assume,
+		PresignEndpoint: "https://192.0.2.15:8443",
+	})
+	require.NoError(t, err)
+
+	raw, err := s.PresignGet(t.Context(), testRef, strings.Repeat("a", 64), 10*time.Minute)
+	require.NoError(t, err)
+	u, err := url.Parse(raw)
+	require.NoError(t, err)
+	assert.Equal(t, "192.0.2.15:8443", u.Host)
+}
+
+func TestOpen_NoOffHostEndpointRefusesDownloadURLs(t *testing.T) {
+	assume := func() (*ststypes.AssumeRoleOutput, error) {
+		return &ststypes.AssumeRoleOutput{Credentials: &ststypes.Credentials{
+			AccessKeyId:     aws.String("ASIATEST"),
+			SecretAccessKey: aws.String("secret"),
+			SessionToken:    aws.String("session-token"),
+			Expiration:      aws.Time(time.Now().Add(time.Hour)),
+		}}, nil
+	}
+	s, err := Open(OpenConfig{Region: testRegion, Endpoint: "https://127.0.0.1:8443", SigningRegion: "ap-southeast-2", Assume: assume})
+	require.NoError(t, err)
+
+	raw, err := s.PresignGet(t.Context(), testRef, strings.Repeat("a", 64), 10*time.Minute)
+	require.ErrorIs(t, err, ErrNoPresignEndpoint)
+	assert.Empty(t, raw)
+}

spinifex/domains/lambda/codestore/provision.go

@@ -0,0 +1,178 @@
+// Package codestore keeps Lambda code copies in the Region's system-account
+// bucket, reached only through a short-lived session of the Region's scoped
+// code-store role. The composition layer owns every privileged client.
+package codestore
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"reflect"
+)
+
+const (
+	// OwnerTagKey marks the role and bucket Lambda intends to own. It names the
+	// intended owner; any principal able to tag could also set it.
+	OwnerTagKey = "spinifex:owner"
+	// OwnerTagValue is the owner tag's value.
+	OwnerTagValue = "lambda-code-store"
+	// PolicyName is the role's only inline policy.
+	PolicyName = "lambda-code-store"
+)
+
+// TrustPolicy lets only the in-process lambda.amazonaws.com path assume the role.
+const TrustPolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"lambda.amazonaws.com"},"Action":"sts:AssumeRole"}]}`
+
+var (
+	// ErrExists is returned by an admin Create when the resource already exists
+	// under the system account.
+	ErrExists = errors.New("already exists")
+	// ErrNotFound is returned by RoleAdmin.Describe for an absent role.
+	ErrNotFound = errors.New("not found")
+	// ErrOwnedElsewhere is returned by BucketAdmin.Create when another account
+	// owns the bucket name.
+	ErrOwnedElsewhere = errors.New("owned by another account")
+	// ErrNotOwned is a refusal that persists until an operator resolves it.
+	ErrNotOwned = errors.New("lambda code store: existing resource is not Lambda's")
+)
+
+// BucketName is the Region's code bucket in the system account.
+func BucketName(region string) string {
+	return "lambda-code-" + region
+}
+
+type policyStatement struct {
+	Effect   string   `json:"Effect"`
+	Action   []string `json:"Action"`
+	Resource string   `json:"Resource"`
+}
+
+type policyDocument struct {
+	Version   string            `json:"Version"`
+	Statement []policyStatement `json:"Statement"`
+}
+
+// RolePolicy grants object access to the Region's code bucket and nothing else.
+func RolePolicy(region string) (string, error) {
+	b := "arn:aws:s3:::" + BucketName(region)
+	doc, err := json.Marshal(policyDocument{
+		Version: "2012-10-17",
+		Statement: []policyStatement{
+			{Effect: "Allow", Action: []string{"s3:PutObject", "s3:GetObject", "s3:DeleteObject"}, Resource: b + "/*"},
+			{Effect: "Allow", Action: []string{"s3:ListBucket"}, Resource: b},
+		},
+	})
+	if err != nil {
+		return "", fmt.Errorf("lambda code store policy: %w", err)
+	}
+	return string(doc), nil
+}
+
+// RoleState is what provisioning reads back about an existing role.
+type RoleState struct {
+	TrustPolicy      string
+	Tags             map[string]string
+	InlinePolicies   []string
+	AttachedPolicies int
+}
+
+// RoleAdmin is bound by the composition layer to the one code-store role.
+type RoleAdmin interface {
+	Create(trustPolicy string, tags map[string]string) error
+	Describe() (*RoleState, error)
+	PutPolicy(name, document string) error
+}
+
+// BucketAdmin is bound by the composition layer to the one code bucket.
+// Create tags the bucket only when it created it, never an existing one.
+type BucketAdmin interface {
+	Create(ctx context.Context, tags map[string]string) error
+	Tags(ctx context.Context) (map[string]string, error)
+}
+
+func ownerTags() map[string]string {
+	return map[string]string{OwnerTagKey: OwnerTagValue}
+}
+
+// Provision creates or verifies the Region's role, policy and bucket. A retry
+// completes Lambda's own partial run; any state it cannot attribute to Lambda
+// is refused with ErrNotOwned and left untouched.
+func Provision(ctx context.Context, region string, role RoleAdmin, bucket BucketAdmin) error {
+	if err := ensureRole(role); err != nil {
+		return err
+	}
+	policy, err := RolePolicy(region)
+	if err != nil {
+		return err
+	}
+	if err := role.PutPolicy(PolicyName, policy); err != nil {
+		return fmt.Errorf("put code-store role policy: %w", err)
+	}
+	return ensureBucket(ctx, region, bucket)
+}
+
+func ensureRole(role RoleAdmin) error {
+	err := role.Create(TrustPolicy, ownerTags())
+	if err == nil {
+		return nil
+	}
+	if !errors.Is(err, ErrExists) {
+		return fmt.Errorf("create code-store role: %w", err)
+	}
+	st, err := role.Describe()
+	if err != nil {
+		return fmt.Errorf("describe code-store role: %w", err)
+	}
+	return checkRole(st)
+}
+
+// checkRole accepts only Lambda's own shape: owner tag, exact trust, no attached
+// policies, and no inline policy but Lambda's. No inline policy at all is the
+// state a crash between CreateRole and PutRolePolicy leaves, so it is retried.
+func checkRole(st *RoleState) error {
+	switch {
+	case st.Tags[OwnerTagKey] != OwnerTagValue:
+		return fmt.Errorf("%w: role has no %s=%s tag", ErrNotOwned, OwnerTagKey, OwnerTagValue)
+	case !sameJSON(st.TrustPolicy, TrustPolicy):
+		return fmt.Errorf("%w: role trust policy differs from the code-store trust policy", ErrNotOwned)
+	case st.AttachedPolicies > 0:
+		return fmt.Errorf("%w: role has %d attached managed policies", ErrNotOwned, st.AttachedPolicies)
+	}
+	for _, name := range st.InlinePolicies {
+		if name != PolicyName {
+			return fmt.Errorf("%w: role has inline policy %q", ErrNotOwned, name)
+		}
+	}
+	return nil
+}
+
+func ensureBucket(ctx context.Context, region string, bucket BucketAdmin) error {
+	err := bucket.Create(ctx, ownerTags())
+	if err == nil {
+		return nil
+	}
+	if errors.Is(err, ErrOwnedElsewhere) {
+		return fmt.Errorf("%w: bucket %s is owned by another account", ErrNotOwned, BucketName(region))
+	}
+	if !errors.Is(err, ErrExists) {
+		return fmt.Errorf("create code bucket: %w", err)
+	}
+	tags, err := bucket.Tags(ctx)
+	if err != nil {
+		return fmt.Errorf("read code bucket tags: %w", err)
+	}
+	if tags[OwnerTagKey] != OwnerTagValue {
+		return fmt.Errorf("%w: bucket %s exists without the %s=%s tag; an operator must tag or delete it",
+			ErrNotOwned, BucketName(region), OwnerTagKey, OwnerTagValue)
+	}
+	return nil
+}
+
+func sameJSON(a, b string) bool {
+	var x, y any
+	if json.Unmarshal([]byte(a), &x) != nil || json.Unmarshal([]byte(b), &y) != nil {
+		return false
+	}
+	return reflect.DeepEqual(x, y)
+}

spinifex/domains/lambda/codestore/provision_test.go

@@ -0,0 +1,165 @@
+package codestore
+
+import (
+	"context"
+	"errors"
+	"slices"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const testRegion = "ap-southeast-2"
+
+type fakeRole struct {
+	state    *RoleState
+	created  int
+	policies map[string]string
+}
+
+func (f *fakeRole) Create(trust string, tags map[string]string) error {
+	if f.state != nil {
+		return ErrExists
+	}
+	f.created++
+	f.state = &RoleState{TrustPolicy: trust, Tags: tags}
+	return nil
+}
+
+func (f *fakeRole) Describe() (*RoleState, error) {
+	if f.state == nil {
+		return nil, ErrNotFound
+	}
+	return f.state, nil
+}
+
+func (f *fakeRole) PutPolicy(name, doc string) error {
+	if f.policies == nil {
+		f.policies = map[string]string{}
+	}
+	f.policies[name] = doc
+	if !slices.Contains(f.state.InlinePolicies, name) {
+		f.state.InlinePolicies = append(f.state.InlinePolicies, name)
+	}
+	return nil
+}
+
+type fakeBucket struct {
+	exists, foreign bool
+	tags            map[string]string
+	tagErr          error
+	tagWrites       int
+}
+
+func (f *fakeBucket) Create(_ context.Context, tags map[string]string) error {
+	switch {
+	case f.foreign:
+		return ErrOwnedElsewhere
+	case f.exists:
+		return ErrExists
+	}
+	f.exists = true
+	if f.tagErr != nil {
+		return f.tagErr
+	}
+	f.tagWrites++
+	f.tags = tags
+	return nil
+}
+
+func (f *fakeBucket) Tags(context.Context) (map[string]string, error) { return f.tags, nil }
+
+func mustPolicy(t *testing.T) string {
+	t.Helper()
+	p, err := RolePolicy(testRegion)
+	require.NoError(t, err)
+	return p
+}
+
+func ownRole(inline ...string) *fakeRole {
+	return &fakeRole{state: &RoleState{TrustPolicy: TrustPolicy, Tags: ownerTags(), InlinePolicies: inline}}
+}
+
+func TestProvision_FreshCreatesTaggedRolePolicyAndBucket(t *testing.T) {
+	role, bucket := &fakeRole{}, &fakeBucket{}
+
+	require.NoError(t, Provision(t.Context(), testRegion, role, bucket))
+
+	assert.Equal(t, 1, role.created)
+	assert.Equal(t, OwnerTagValue, role.state.Tags[OwnerTagKey])
+	assert.JSONEq(t, mustPolicy(t), role.policies[PolicyName])
+	assert.Equal(t, OwnerTagValue, bucket.tags[OwnerTagKey])
+}
+
+// A crash between CreateRole and PutRolePolicy leaves Lambda's tagged role with
+// no inline policy; the next run adds it.
+func TestProvision_RetriesOwnRoleWithoutPolicy(t *testing.T) {
+	role, bucket := ownRole(), &fakeBucket{}
+
+	require.NoError(t, Provision(t.Context(), testRegion, role, bucket))
+
+	assert.Zero(t, role.created)
+	assert.JSONEq(t, mustPolicy(t), role.policies[PolicyName])
+}
+
+func TestProvision_RestoresDriftedOwnPolicy(t *testing.T) {
+	role, bucket := ownRole(PolicyName), &fakeBucket{}
+	role.policies = map[string]string{PolicyName: `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"*"}]}`}
+
+	require.NoError(t, Provision(t.Context(), testRegion, role, bucket))
+	assert.JSONEq(t, mustPolicy(t), role.policies[PolicyName])
+}
+
+func TestProvision_RefusesRoleNotLambdas(t *testing.T) {
+	cases := map[string]*RoleState{
+		"no owner tag":          {TrustPolicy: TrustPolicy},
+		"other trust":           {TrustPolicy: `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"arn:aws:iam::000000000000:root"},"Action":"sts:AssumeRole"}]}`, Tags: ownerTags()},
+		"attached policy":       {TrustPolicy: TrustPolicy, Tags: ownerTags(), AttachedPolicies: 1},
+		"foreign inline policy": {TrustPolicy: TrustPolicy, Tags: ownerTags(), InlinePolicies: []string{PolicyName, "admin"}},
+	}
+	for name, st := range cases {
+		t.Run(name, func(t *testing.T) {
+			role, bucket := &fakeRole{state: st}, &fakeBucket{}
+
+			err := Provision(t.Context(), testRegion, role, bucket)
+
+			require.ErrorIs(t, err, ErrNotOwned)
+			assert.Nil(t, role.policies, "no policy written to a role that is not Lambda's")
+			assert.False(t, bucket.exists, "provisioning stops before the bucket")
+		})
+	}
+}
+
+func TestProvision_AcceptsOwnTaggedBucket(t *testing.T) {
+	bucket := &fakeBucket{exists: true, tags: ownerTags()}
+
+	require.NoError(t, Provision(t.Context(), testRegion, &fakeRole{}, bucket))
+	assert.Zero(t, bucket.tagWrites)
+}
+
+// A crash after CreateBucket and before its tags leaves an untagged bucket that
+// cannot be told apart from another system component's, so it is refused.
+func TestProvision_RefusesUntaggedBucketAfterPartialCreate(t *testing.T) {
+	bucket := &fakeBucket{tagErr: errors.New("connection reset")}
+	require.Error(t, Provision(t.Context(), testRegion, &fakeRole{}, bucket))
+
+	bucket.tagErr = nil
+	err := Provision(t.Context(), testRegion, &fakeRole{}, bucket)
+
+	require.ErrorIs(t, err, ErrNotOwned)
+	assert.Contains(t, err.Error(), "lambda-code-ap-southeast-2 exists without the spinifex:owner=lambda-code-store tag")
+	assert.Zero(t, bucket.tagWrites, "an existing bucket is never tagged into use")
+}
+
+func TestProvision_RefusesBucketOwnedElsewhere(t *testing.T) {
+	err := Provision(t.Context(), testRegion, &fakeRole{}, &fakeBucket{foreign: true})
+	require.ErrorIs(t, err, ErrNotOwned)
+}
+
+func TestRolePolicy_ScopesToRegionalBucket(t *testing.T) {
+	assert.JSONEq(t, `{"Version":"2012-10-17","Statement":[
+		{"Effect":"Allow","Action":["s3:PutObject","s3:GetObject","s3:DeleteObject"],"Resource":"arn:aws:s3:::lambda-code-ap-southeast-2/*"},
+		{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":"arn:aws:s3:::lambda-code-ap-southeast-2"}]}`,
+		mustPolicy(t))
+}

spinifex/domains/lambda/codestore/store.go

@@ -0,0 +1,297 @@
+package codestore
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"regexp"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+)
+
+// MaxCodeBytes bounds one stored code copy, AWS's direct-upload ZIP limit.
+const MaxCodeBytes = 50 << 20
+
+var (
+	// ErrCodeNotFound means no copy exists for the reference.
+	ErrCodeNotFound = errors.New("lambda code copy not found")
+	// ErrDigestMismatch means the stored bytes do not match the recorded SHA-256.
+	ErrDigestMismatch = errors.New("lambda code copy digest mismatch")
+	// ErrCodeTooLarge means the copy exceeds MaxCodeBytes.
+	ErrCodeTooLarge = errors.New("lambda code copy too large")
+	// ErrInvalidRef means a reference segment is empty or unsafe as a key segment.
+	ErrInvalidRef = errors.New("invalid lambda code reference")
+	// ErrNoPresignEndpoint means no off-host Predastore address is configured.
+	ErrNoPresignEndpoint = errors.New("lambda code store: no off-host endpoint for download URLs")
+)
+
+var (
+	refSegment = regexp.MustCompile(`^[A-Za-z0-9_-][A-Za-z0-9._-]{0,127}$`)
+	digestHex  = regexp.MustCompile(`^[0-9a-f]{64}$`)
+)
+
+// Ref names one code revision; keys group by account for S11 inventory.
+type Ref struct {
+	AccountID  string
+	FunctionID string
+	RevisionID string
+}
+
+func (r Ref) prefix() (string, error) {
+	for _, seg := range []string{r.AccountID, r.FunctionID, r.RevisionID} {
+		if !refSegment.MatchString(seg) {
+			return "", fmt.Errorf("%w: %q", ErrInvalidRef, seg)
+		}
+	}
+	return r.AccountID + "/" + r.FunctionID + "/" + r.RevisionID + "/", nil
+}
+
+// key names a copy by its digest, so different bytes for the same revision land
+// on a different object and can never replace the recorded copy.
+func (r Ref) key(sha256Hex string) (string, error) {
+	p, err := r.prefix()
+	if err != nil {
+		return "", err
+	}
+	if !digestHex.MatchString(sha256Hex) {
+		return "", fmt.Errorf("%w: digest %q", ErrInvalidRef, sha256Hex)
+	}
+	return p + sha256Hex + ".zip", nil
+}
+
+// Store reads and writes code copies in one Region's code bucket.
+type Store struct {
+	bucket  string
+	objects objectstore.ObjectStore
+	presign presignFunc
+
+	// creds and provider are the scoped session behind presign; nil on stores
+	// built without one.
+	creds    *credentials.Credentials
+	provider *sessionProvider
+	now      func() time.Time
+}
+
+// presignMargin is slack kept between a signed URL's end and its session's.
+const presignMargin = 30 * time.Second
+
+// presignFunc signs a GET for the input; nil on stores that cannot presign.
+type presignFunc func(in *s3.GetObjectInput, ttl time.Duration) (string, error)
+
+// presignWith signs GETs with the session credentials behind client.
+func presignWith(client *s3.S3) presignFunc {
+	return func(in *s3.GetObjectInput, ttl time.Duration) (string, error) {
+		req, _ := client.GetObjectRequest(in)
+		return req.Presign(ttl)
+	}
+}
+
+// OpenConfig is what the composition layer supplies to open the scoped store.
+type OpenConfig struct {
+	Region        string // Spinifex Region naming the bucket
+	Endpoint      string // Predastore URL
+	SigningRegion string // Predastore SigV4 Region
+	Assume        AssumeFunc
+	HTTPClient    *http.Client
+	now           func() time.Time
+	// PresignEndpoint is the Predastore URL off-host clients reach; download
+	// URLs are signed for it. Empty disables download URLs.
+	PresignEndpoint string
+}
+
+// Open builds the store on a refreshing code-store session and draws the first
+// session, so an unassumable role fails here rather than on first use.
+func Open(cfg OpenConfig) (*Store, error) {
+	if cfg.Region == "" || cfg.Endpoint == "" || cfg.Assume == nil {
+		return nil, errors.New("lambda code store: region, endpoint and assume are required")
+	}
+	now := cfg.now
+	if now == nil {
+		now = time.Now
+	}
+	client := cfg.HTTPClient
+	if client == nil {
+		client = &http.Client{Timeout: 120 * time.Second}
+	}
+	provider := newSessionProvider(cfg.Assume, now)
+	creds := credentials.NewCredentials(provider)
+	if _, err := creds.Get(); err != nil {
+		return nil, err
+	}
+	sess, err := session.NewSession(&aws.Config{
+		Endpoint:         aws.String(cfg.Endpoint),
+		Region:           aws.String(cfg.SigningRegion),
+		Credentials:      creds,
+		S3ForcePathStyle: aws.Bool(true),
+		HTTPClient:       client,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("lambda code store session: %w", err)
+	}
+	s3Client := s3.New(sess)
+	store := newStore(cfg.Region, objectstore.NewS3ObjectStore(s3Client))
+	// Without an off-host address a download URL would name an internal one.
+	if cfg.PresignEndpoint != "" {
+		store.presign = presignWith(s3.New(sess, &aws.Config{Endpoint: aws.String(cfg.PresignEndpoint)}))
+	}
+	store.creds, store.provider, store.now = creds, provider, now
+	return store, nil
+}
+
+func newStore(region string, objects objectstore.ObjectStore) *Store {
+	return &Store{bucket: BucketName(region), objects: objects}
+}
+
+// Put stores code under its digest and returns the hex SHA-256 the revision
+// record keeps. A retry with the same bytes rewrites identical content.
+func (s *Store) Put(ctx context.Context, ref Ref, code []byte) (string, error) {
+	if len(code) > MaxCodeBytes {
+		return "", ErrCodeTooLarge
+	}
+	sum := sha256.Sum256(code)
+	digest := hex.EncodeToString(sum[:])
+	key, err := ref.key(digest)
+	if err != nil {
+		return "", err
+	}
+	if _, err := s.objects.PutObject(ctx, &s3.PutObjectInput{
+		Bucket:        aws.String(s.bucket),
+		Key:           aws.String(key),
+		Body:          bytes.NewReader(code),
+		ContentLength: aws.Int64(int64(len(code))),
+	}); err != nil {
+		return "", fmt.Errorf("put lambda code copy: %w", err)
+	}
+	return digest, nil
+}
+
+// Get returns the copy only if it matches wantSHA256, the recorded digest.
+func (s *Store) Get(ctx context.Context, ref Ref, wantSHA256 string) ([]byte, error) {
+	key, err := ref.key(wantSHA256)
+	if err != nil {
+		return nil, err
+	}
+	out, err := s.objects.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)})
+	if err != nil {
+		if objectstore.IsNoSuchKeyError(err) {
+			return nil, ErrCodeNotFound
+		}
+		return nil, fmt.Errorf("get lambda code copy: %w", err)
+	}
+	defer out.Body.Close()
+	code, err := io.ReadAll(io.LimitReader(out.Body, MaxCodeBytes+1))
+	if err != nil {
+		return nil, fmt.Errorf("read lambda code copy: %w", err)
+	}
+	if len(code) > MaxCodeBytes {
+		return nil, ErrCodeTooLarge
+	}
+	sum := sha256.Sum256(code)
+	if hex.EncodeToString(sum[:]) != wantSHA256 {
+		return nil, ErrDigestMismatch
+	}
+	return code, nil
+}
+
+// ensureSession guarantees the signing session outlasts ttl, drawing a new one
+// if it does not, and fails rather than sign a URL that would outlive it.
+func (s *Store) ensureSession(ttl time.Duration) error {
+	if s.creds == nil || s.provider == nil {
+		return nil
+	}
+	need := ttl + presignMargin
+	if _, err := s.creds.Get(); err != nil {
+		return fmt.Errorf("lambda code store session: %w", err)
+	}
+	for attempt := range 2 {
+		exp, err := s.creds.ExpiresAt()
+		if err != nil {
+			return fmt.Errorf("lambda code store session expiry: %w", err)
+		}
+		if !exp.Before(s.now().Add(need)) {
+			return nil
+		}
+		if attempt == 1 {
+			break
+		}
+		s.provider.requireRemaining(need)
+		s.creds.Expire()
+		if _, err := s.creds.Get(); err != nil {
+			return fmt.Errorf("lambda code store session: %w", err)
+		}
+	}
+	return errors.New("lambda code store: session expires before the presigned URL would")
+}
+
+// PresignGet returns a time-limited GET URL for the copy with the recorded
+// digest, signed by a session that outlasts the URL.
+func (s *Store) PresignGet(ctx context.Context, ref Ref, sha256Hex string, ttl time.Duration) (string, error) {
+	if s.presign == nil {
+		return "", ErrNoPresignEndpoint
+	}
+	key, err := ref.key(sha256Hex)
+	if err != nil {
+		return "", err
+	}
+	if err := ctx.Err(); err != nil {
+		return "", err
+	}
+	if err := s.ensureSession(ttl); err != nil {
+		return "", err
+	}
+	url, err := s.presign(&s3.GetObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)}, ttl)
+	if err != nil {
+		return "", fmt.Errorf("presign lambda code copy: %w", err)
+	}
+	return url, nil
+}
+
+// Delete removes the copy with the recorded digest; an absent copy is already
+// deleted.
+func (s *Store) Delete(ctx context.Context, ref Ref, sha256Hex string) error {
+	key, err := ref.key(sha256Hex)
+	if err != nil {
+		return err
+	}
+	return s.deleteKey(ctx, key)
+}
+
+// DeleteRevision removes every copy under the revision, including ones a failed
+// or superseded attempt left. Callers own the guarantee that no create or update
+// for the revision is still in progress.
+func (s *Store) DeleteRevision(ctx context.Context, ref Ref) error {
+	prefix, err := ref.prefix()
+	if err != nil {
+		return err
+	}
+	objects, _, err := objectstore.ListAll(ctx, s.objects, &s3.ListObjectsV2Input{
+		Bucket: aws.String(s.bucket), Prefix: aws.String(prefix),
+	})
+	if err != nil {
+		return fmt.Errorf("list lambda code copies: %w", err)
+	}
+	for _, o := range objects {
+		if err := s.deleteKey(ctx, aws.StringValue(o.Key)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *Store) deleteKey(ctx context.Context, key string) error {
+	_, err := s.objects.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)})
+	if err != nil && !objectstore.IsNoSuchKeyError(err) {
+		return fmt.Errorf("delete lambda code copy: %w", err)
+	}
+	return nil
+}

spinifex/domains/lambda/codestore/store_test.go

@@ -0,0 +1,169 @@
+package codestore
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/sts"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var testRef = Ref{AccountID: "000000000001", FunctionID: "fn-0123", RevisionID: "rev-1"}
+
+func copyKey(sum string) string { return "000000000001/fn-0123/rev-1/" + sum + ".zip" }
+
+func TestStore_PutGetDelete(t *testing.T) {
+	mem := objectstore.NewMemoryObjectStore()
+	s := newStore(testRegion, mem)
+	code := []byte("PK\x03\x04 zip bytes")
+
+	sum, err := s.Put(t.Context(), testRef, code)
+	require.NoError(t, err)
+	want := sha256.Sum256(code)
+	assert.Equal(t, hex.EncodeToString(want[:]), sum)
+
+	_, err = mem.HeadObject(t.Context(), &s3.HeadObjectInput{
+		Bucket: aws.String("lambda-code-ap-southeast-2"), Key: aws.String(copyKey(sum))})
+	require.NoError(t, err, "copy is keyed by account, function, revision and digest")
+
+	got, err := s.Get(t.Context(), testRef, sum)
+	require.NoError(t, err)
+	assert.Equal(t, code, got)
+
+	require.NoError(t, s.Delete(t.Context(), testRef, sum))
+	_, err = s.Get(t.Context(), testRef, sum)
+	require.ErrorIs(t, err, ErrCodeNotFound)
+	require.NoError(t, s.Delete(t.Context(), testRef, sum), "deleting an absent copy succeeds")
+}
+
+// A retry or a racing write with different bytes for the same revision lands on
+// its own digest key, so the recorded copy is never replaced.
+func TestStore_DifferentBytesNeverReplaceRecordedCopy(t *testing.T) {
+	s := newStore(testRegion, objectstore.NewMemoryObjectStore())
+	recorded, err := s.Put(t.Context(), testRef, []byte("first"))
+	require.NoError(t, err)
+
+	var wg sync.WaitGroup
+	others := make([]string, 8)
+	for i := range others {
+		wg.Go(func() {
+			sum, err := s.Put(t.Context(), testRef, []byte(fmt.Sprintf("other-%d", i)))
+			assert.NoError(t, err)
+			others[i] = sum
+		})
+	}
+	wg.Wait()
+
+	got, err := s.Get(t.Context(), testRef, recorded)
+	require.NoError(t, err)
+	assert.Equal(t, []byte("first"), got)
+	for i, sum := range others {
+		got, err := s.Get(t.Context(), testRef, sum)
+		require.NoError(t, err)
+		assert.Equal(t, []byte(fmt.Sprintf("other-%d", i)), got)
+	}
+}
+
+// DeleteRevision clears every copy of one revision, including a failed
+// attempt's, and leaves other revisions alone.
+func TestStore_DeleteRevisionRemovesAllCopiesOfThatRevisionOnly(t *testing.T) {
+	s := newStore(testRegion, objectstore.NewMemoryObjectStore())
+	a, err := s.Put(t.Context(), testRef, []byte("kept attempt"))
+	require.NoError(t, err)
+	b, err := s.Put(t.Context(), testRef, []byte("failed attempt"))
+	require.NoError(t, err)
+	sibling := Ref{AccountID: testRef.AccountID, FunctionID: testRef.FunctionID, RevisionID: "rev-10"}
+	c, err := s.Put(t.Context(), sibling, []byte("next revision"))
+	require.NoError(t, err)
+
+	require.NoError(t, s.DeleteRevision(t.Context(), testRef))
+
+	for _, sum := range []string{a, b} {
+		_, err := s.Get(t.Context(), testRef, sum)
+		require.ErrorIs(t, err, ErrCodeNotFound)
+	}
+	_, err = s.Get(t.Context(), sibling, c)
+	require.NoError(t, err, "rev-10 shares the rev-1 string prefix but not its key prefix")
+}
+
+func TestStore_GetRefusesDigestMismatch(t *testing.T) {
+	mem := objectstore.NewMemoryObjectStore()
+	s := newStore(testRegion, mem)
+	sum, err := s.Put(t.Context(), testRef, []byte("original"))
+	require.NoError(t, err)
+	_, err = mem.PutObject(t.Context(), &s3.PutObjectInput{
+		Bucket: aws.String(BucketName(testRegion)), Key: aws.String(copyKey(sum)),
+		Body: bytes.NewReader([]byte("replaced"))})
+	require.NoError(t, err)
+
+	_, err = s.Get(t.Context(), testRef, sum)
+	require.ErrorIs(t, err, ErrDigestMismatch)
+}
+
+func TestStore_RejectsUnsafeRefAndOversize(t *testing.T) {
+	s := newStore(testRegion, objectstore.NewMemoryObjectStore())
+	for _, ref := range []Ref{
+		{AccountID: "", FunctionID: "f", RevisionID: "r"},
+		{AccountID: "a", FunctionID: "..", RevisionID: "r"},
+		{AccountID: "a", FunctionID: "f/../../other", RevisionID: "r"},
+	} {
+		_, err := s.Put(t.Context(), ref, []byte("x"))
+		require.ErrorIs(t, err, ErrInvalidRef, "%+v", ref)
+	}
+	_, err := s.Put(t.Context(), testRef, make([]byte, MaxCodeBytes+1))
+	require.ErrorIs(t, err, ErrCodeTooLarge)
+	_, err = s.Get(t.Context(), testRef, "../../other")
+	require.ErrorIs(t, err, ErrInvalidRef, "a digest is never used as a raw key segment")
+}
+
+func TestOpen_FailsWhenRoleNotAssumable(t *testing.T) {
+	_, err := Open(OpenConfig{Region: testRegion, Endpoint: "https://127.0.0.1:1", SigningRegion: testRegion,
+		Assume: func() (*sts.AssumeRoleOutput, error) { return nil, errors.New("AccessDenied") }})
+	require.Error(t, err)
+}
+
+func TestManager_InitialisesOnFirstUseWithBackoff(t *testing.T) {
+	attempts := 0
+	m := NewManager(func(context.Context) error {
+		attempts++
+		if attempts < 3 {
+			return ErrNotOwned
+		}
+		return nil
+	}, func() (*Store, error) { return newStore(testRegion, objectstore.NewMemoryObjectStore()), nil })
+	now := time.Unix(1_000_000, 0)
+	m.now = func() time.Time { return now }
+	assert.Zero(t, attempts, "construction does no provisioning")
+
+	_, err := m.Store(t.Context())
+	require.ErrorIs(t, err, ErrNotReady)
+	require.ErrorIs(t, err, ErrNotOwned)
+	_, err = m.Store(t.Context())
+	require.ErrorIs(t, err, ErrNotReady)
+	assert.Equal(t, 1, attempts, "a call inside the backoff window does not retry")
+
+	now = now.Add(time.Second)
+	_, err = m.Store(t.Context())
+	require.ErrorIs(t, err, ErrNotReady)
+	now = now.Add(time.Second)
+	_, err = m.Store(t.Context())
+	require.ErrorIs(t, err, ErrNotReady, "backoff doubled to 2s")
+	now = now.Add(time.Second)
+	first, err := m.Store(t.Context())
+	require.NoError(t, err)
+	again, err := m.Store(t.Context())
+	require.NoError(t, err)
+	assert.Same(t, first, again)
+	assert.Equal(t, 3, attempts)
+}

spinifex/domains/lambda/function/client.go

@@ -0,0 +1,67 @@
+package function
+
+import (
+	"context"
+	"time"
+
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/nats-io/nats.go"
+)
+
+const metaRequestTimeout = 30 * time.Second
+
+// Client is the gateway's view of the daemon-owned function records.
+type Client interface {
+	// Create binds the name and returns the KV revision.
+	Create(ctx context.Context, accountID, region string, rec Record) (uint64, error)
+	// Get returns the record and revision, or ErrNotFound.
+	Get(ctx context.Context, accountID, region, name string) (Record, uint64, error)
+	// CodeStored marks the code stored and returns the stored record and revision.
+	CodeStored(ctx context.Context, accountID, region, name string, revision uint64, sha256Hex string, size int64) (Record, uint64, error)
+	// RollbackCreate frees the name while the record is still storing at revision.
+	RollbackCreate(ctx context.Context, accountID, region, name string, revision uint64) error
+}
+
+// NATSClient is the Client over NATS request/reply to the daemon.
+type NATSClient struct {
+	conn *nats.Conn
+}
+
+var _ Client = (*NATSClient)(nil)
+
+// NewNATSClient builds a NATS-backed Client.
+func NewNATSClient(conn *nats.Conn) *NATSClient { return &NATSClient{conn: conn} }
+
+func (c *NATSClient) Create(ctx context.Context, accountID, region string, rec Record) (uint64, error) {
+	resp, err := natsmsg.NATSRequest[CreateResponse](ctx, c.conn, SubjectCreate, &CreateRequest{Region: region, Record: rec}, metaRequestTimeout, accountID)
+	if err != nil {
+		return 0, err
+	}
+	return resp.Revision, nil
+}
+
+func (c *NATSClient) Get(ctx context.Context, accountID, region, name string) (Record, uint64, error) {
+	resp, err := natsmsg.NATSRequest[GetResponse](ctx, c.conn, SubjectGet, &GetRequest{Region: region, FunctionName: name}, metaRequestTimeout, accountID)
+	if err != nil {
+		return Record{}, 0, err
+	}
+	if !resp.Found {
+		return Record{}, 0, ErrNotFound
+	}
+	return resp.Record, resp.Revision, nil
+}
+
+func (c *NATSClient) CodeStored(ctx context.Context, accountID, region, name string, revision uint64, sha256Hex string, size int64) (Record, uint64, error) {
+	resp, err := natsmsg.NATSRequest[CodeStoredResponse](ctx, c.conn, SubjectCodeStored,
+		&CodeStoredRequest{Region: region, FunctionName: name, Revision: revision, SHA256: sha256Hex, Size: size}, metaRequestTimeout, accountID)
+	if err != nil {
+		return Record{}, 0, err
+	}
+	return resp.Record, resp.Revision, nil
+}
+
+func (c *NATSClient) RollbackCreate(ctx context.Context, accountID, region, name string, revision uint64) error {
+	_, err := natsmsg.NATSRequest[RollbackCreateResponse](ctx, c.conn, SubjectRollbackCreate,
+		&RollbackCreateRequest{Region: region, FunctionName: name, Revision: revision}, metaRequestTimeout, accountID)
+	return err
+}

spinifex/domains/lambda/function/envcrypt.go

@@ -0,0 +1,56 @@
+package function
+
+import (
+	"crypto/cipher"
+	"crypto/rand"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+)
+
+func envAAD(accountID, functionID string) []byte {
+	return []byte("lambda-env/v1|" + accountID + "|" + functionID)
+}
+
+// SealEnv encrypts the variables as base64(nonce||ciphertext), bound to the
+// account and function. No variables yield "".
+func SealEnv(aead cipher.AEAD, accountID, functionID string, vars map[string]string) (string, error) {
+	if len(vars) == 0 {
+		return "", nil
+	}
+	plain, err := json.Marshal(vars)
+	if err != nil {
+		return "", fmt.Errorf("encode environment: %w", err)
+	}
+	nonce := make([]byte, aead.NonceSize())
+	if _, err := rand.Read(nonce); err != nil {
+		return "", fmt.Errorf("environment nonce: %w", err)
+	}
+	sealed := aead.Seal(nonce, nonce, plain, envAAD(accountID, functionID))
+	return base64.StdEncoding.EncodeToString(sealed), nil
+}
+
+// OpenEnv reverses SealEnv. A blob sealed for another account or function fails.
+func OpenEnv(aead cipher.AEAD, accountID, functionID, sealed string) (map[string]string, error) {
+	if sealed == "" {
+		return nil, nil
+	}
+	raw, err := base64.StdEncoding.DecodeString(sealed)
+	if err != nil {
+		return nil, fmt.Errorf("decode environment: %w", err)
+	}
+	n := aead.NonceSize()
+	if len(raw) < n+aead.Overhead() {
+		return nil, errors.New("environment blob too short")
+	}
+	plain, err := aead.Open(nil, raw[:n], raw[n:], envAAD(accountID, functionID))
+	if err != nil {
+		return nil, fmt.Errorf("open environment: %w", err)
+	}
+	var vars map[string]string
+	if err := json.Unmarshal(plain, &vars); err != nil {
+		return nil, fmt.Errorf("decode environment: %w", err)
+	}
+	return vars, nil
+}

spinifex/domains/lambda/function/envcrypt_test.go

@@ -0,0 +1,62 @@
+package function
+
+import (
+	"testing"
+
+	"github.com/mulgadc/bluebottle/pkg/masterkey"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func testAEAD(t *testing.T) *masterkey.Key {
+	t.Helper()
+	k, err := masterkey.New(make([]byte, 32))
+	require.NoError(t, err)
+	return k
+}
+
+func TestEnv_RoundTripAndNoPlaintext(t *testing.T) {
+	k := testAEAD(t)
+	vars := map[string]string{"DB_PASSWORD": "hunter2", "MODE": "prod"}
+
+	sealed, err := SealEnv(k.AEAD, "000000000001", "fn1", vars)
+	require.NoError(t, err)
+	assert.NotContains(t, sealed, "hunter2")
+
+	got, err := OpenEnv(k.AEAD, "000000000001", "fn1", sealed)
+	require.NoError(t, err)
+	assert.Equal(t, vars, got)
+
+	again, err := SealEnv(k.AEAD, "000000000001", "fn1", vars)
+	require.NoError(t, err)
+	assert.NotEqual(t, sealed, again, "each seal draws a fresh nonce")
+}
+
+func TestEnv_BoundToAccountAndFunction(t *testing.T) {
+	k := testAEAD(t)
+	sealed, err := SealEnv(k.AEAD, "000000000001", "fn1", map[string]string{"A": "b"})
+	require.NoError(t, err)
+
+	_, err = OpenEnv(k.AEAD, "000000000001", "fn2", sealed)
+	require.Error(t, err)
+	_, err = OpenEnv(k.AEAD, "000000000002", "fn1", sealed)
+	require.Error(t, err)
+	_, err = OpenEnv(k.AEAD, "000000000001", "fn1", "not base64 !!")
+	require.Error(t, err)
+	_, err = OpenEnv(k.AEAD, "000000000001", "fn1", "AAAA")
+	require.Error(t, err)
+}
+
+func TestEnv_EmptyIsEmpty(t *testing.T) {
+	k := testAEAD(t)
+	sealed, err := SealEnv(k.AEAD, "a", "f", nil)
+	require.NoError(t, err)
+	assert.Empty(t, sealed)
+	sealed, err = SealEnv(k.AEAD, "a", "f", map[string]string{})
+	require.NoError(t, err)
+	assert.Empty(t, sealed)
+
+	got, err := OpenEnv(k.AEAD, "a", "f", "")
+	require.NoError(t, err)
+	assert.Empty(t, got)
+}

spinifex/domains/lambda/function/meta.go

@@ -0,0 +1,259 @@
+package function
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"regexp"
+	"sync"
+	"time"
+
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// ClaimTTL is how long a record still storing its code holds its name. It must
+// exceed the gateway request deadline plus the longest permitted code write.
+const ClaimTTL = 5 * time.Minute
+
+// ErrNotFound means no record is bound to the name.
+var ErrNotFound = errors.New("lambda function not found")
+
+var (
+	nameRE   = regexp.MustCompile(`^[a-zA-Z0-9-_]{1,64}$`)
+	regionRE = regexp.MustCompile(`^[a-z0-9-]{1,32}$`)
+	digestRE = regexp.MustCompile(`^[0-9a-f]{64}$`)
+)
+
+// ValidName reports whether name is a legal function name.
+func ValidName(name string) bool { return nameRE.MatchString(name) }
+
+func badRegion() error {
+	return awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, "region is invalid")
+}
+
+// MetaService is the daemon-side owner of function records. Every update is a
+// compare-and-swap on the revision the caller read or was handed.
+type MetaService struct {
+	js  jetstream.JetStream
+	now func() time.Time
+
+	mu      sync.Mutex
+	buckets map[string]jetstream.KeyValue
+}
+
+// NewMetaService builds the service over a JetStream handle.
+func NewMetaService(js jetstream.JetStream) *MetaService {
+	return &MetaService{js: js, now: time.Now, buckets: make(map[string]jetstream.KeyValue)}
+}
+
+func (s *MetaService) bucket(ctx context.Context, accountID string) (jetstream.KeyValue, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if kv, ok := s.buckets[accountID]; ok {
+		return kv, nil
+	}
+	name := KVAccountBucket(accountID)
+	kv, err := kvutil.GetOrCreateBucket(ctx, s.js, name, KVBucketAccountHistory)
+	if err != nil {
+		return nil, fmt.Errorf("lambda: open account bucket %s: %w", name, err)
+	}
+	s.buckets[accountID] = kv
+	return kv, nil
+}
+
+func conflict(format string, args ...any) error {
+	return awserrors.Errorf(awserrors.ErrorResourceConflictException, format, args...)
+}
+
+func decode(data []byte) (Record, error) {
+	var rec Record
+	if err := json.Unmarshal(data, &rec); err != nil {
+		return Record{}, awserrors.Errorf(awserrors.ErrorServiceException, "function record is unreadable")
+	}
+	if rec.FormatVersion != FormatVersion {
+		return Record{}, awserrors.Errorf(awserrors.ErrorServiceException,
+			"function record has unsupported format version %d", rec.FormatVersion)
+	}
+	return rec, nil
+}
+
+// Create binds the name by writing the record with kv.Create. The record starts
+// Pending with its code storing and a claim that expires after ClaimTTL. A name
+// held only by an expired, still-storing claim is taken over once.
+//
+// Orphaned code objects from rolled-back or taken-over claims are not cleaned up yet.
+func (s *MetaService) Create(ctx context.Context, req *CreateRequest, accountID string) (*CreateResponse, error) {
+	rec := req.Record
+	if !regionRE.MatchString(req.Region) {
+		return nil, badRegion()
+	}
+	if !ValidName(rec.FunctionName) || rec.FunctionID == "" || rec.Code.RevisionID == "" || accountID == "" {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, "function record is incomplete")
+	}
+	kv, err := s.bucket(ctx, accountID)
+	if err != nil {
+		return nil, err
+	}
+	now := s.now().UTC()
+	rec.FormatVersion = FormatVersion
+	rec.AccountID, rec.Region = accountID, req.Region
+	rec.State = StatePending
+	rec.StateReason, rec.StateReasonCode = "", ""
+	rec.Code = Code{RevisionID: rec.Code.RevisionID, State: CodeStoring}
+	rec.ClaimExpiresAt = now.Add(ClaimTTL)
+	rec.CreatedAt, rec.LastModified = now, now
+	data, err := json.Marshal(rec)
+	if err != nil {
+		return nil, err
+	}
+	key := KVFunctionKey(req.Region, rec.FunctionName)
+
+	rev, err := kv.Create(ctx, key, data)
+	if err == nil {
+		return &CreateResponse{Revision: rev}, nil
+	}
+	if !errors.Is(err, jetstream.ErrKeyExists) {
+		return nil, fmt.Errorf("lambda: create function record: %w", err)
+	}
+
+	exists := conflict("Function already exist: %s", rec.FunctionName)
+	entry, err := kv.Get(ctx, key)
+	if err != nil {
+		return nil, exists
+	}
+	old, err := decode(entry.Value())
+	if err != nil || old.Code.State != CodeStoring || !s.now().After(old.ClaimExpiresAt) {
+		return nil, exists
+	}
+	if err := kv.Delete(ctx, key, jetstream.LastRevision(entry.Revision())); err != nil {
+		return nil, exists
+	}
+	slog.WarnContext(ctx, "Lambda: took over an expired function name claim",
+		"function_name", rec.FunctionName, "old_function_id", old.FunctionID)
+	rev, err = kv.Create(ctx, key, data)
+	if err != nil {
+		if errors.Is(err, jetstream.ErrKeyExists) {
+			return nil, exists
+		}
+		return nil, fmt.Errorf("lambda: create function record: %w", err)
+	}
+	return &CreateResponse{Revision: rev}, nil
+}
+
+// Get returns the record, or Found false when the name is unbound.
+func (s *MetaService) Get(ctx context.Context, req *GetRequest, accountID string) (*GetResponse, error) {
+	if !regionRE.MatchString(req.Region) {
+		return nil, badRegion()
+	}
+	kv, err := s.bucket(ctx, accountID)
+	if err != nil {
+		return nil, err
+	}
+	entry, err := kv.Get(ctx, KVFunctionKey(req.Region, req.FunctionName))
+	if errors.Is(err, jetstream.ErrKeyNotFound) {
+		return &GetResponse{}, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("lambda: get function record: %w", err)
+	}
+	rec, err := decode(entry.Value())
+	if err != nil {
+		return nil, err
+	}
+	return &GetResponse{Found: true, Record: rec, Revision: entry.Revision()}, nil
+}
+
+// readStoring loads the record and checks it is the one the caller created: the
+// revision matches and the code is still storing.
+func (s *MetaService) readStoring(ctx context.Context, kv jetstream.KeyValue, region, name string, revision uint64) (Record, error) {
+	entry, err := kv.Get(ctx, KVFunctionKey(region, name))
+	if errors.Is(err, jetstream.ErrKeyNotFound) {
+		return Record{}, ErrNotFound
+	}
+	if err != nil {
+		return Record{}, fmt.Errorf("lambda: get function record: %w", err)
+	}
+	if entry.Revision() != revision {
+		return Record{}, conflict("Function was modified by another request: %s", name)
+	}
+	rec, err := decode(entry.Value())
+	if err != nil {
+		return Record{}, err
+	}
+	if rec.Code.State != CodeStoring {
+		return Record{}, conflict("Function code is not being stored: %s", name)
+	}
+	return rec, nil
+}
+
+// CodeStored marks the code stored. It is refused, leaving the record unchanged,
+// when the revision moved, the code is not storing, or the claim has expired.
+func (s *MetaService) CodeStored(ctx context.Context, req *CodeStoredRequest, accountID string) (*CodeStoredResponse, error) {
+	if !regionRE.MatchString(req.Region) {
+		return nil, badRegion()
+	}
+	if !digestRE.MatchString(req.SHA256) || req.Size < 0 {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValueException, "code digest is invalid")
+	}
+	kv, err := s.bucket(ctx, accountID)
+	if err != nil {
+		return nil, err
+	}
+	rec, err := s.readStoring(ctx, kv, req.Region, req.FunctionName, req.Revision)
+	if errors.Is(err, ErrNotFound) {
+		return nil, awserrors.Errorf(awserrors.ErrorResourceNotFoundException, "Function not found: %s", req.FunctionName)
+	}
+	if err != nil {
+		return nil, err
+	}
+	now := s.now()
+	if now.After(rec.ClaimExpiresAt) {
+		return nil, conflict("Function creation claim expired: %s", req.FunctionName)
+	}
+	rec.Code.State, rec.Code.SHA256, rec.Code.Size = CodeStored, req.SHA256, req.Size
+	rec.State, rec.StateReasonCode, rec.StateReason = StatePending, ReasonCodeCreating, ReasonCreating
+	rec.LastModified = now.UTC()
+	data, err := json.Marshal(rec)
+	if err != nil {
+		return nil, err
+	}
+	rev, err := kv.Update(ctx, KVFunctionKey(req.Region, req.FunctionName), data, req.Revision)
+	if err != nil {
+		if errors.Is(err, jetstream.ErrKeyExists) {
+			return nil, conflict("Function was modified by another request: %s", req.FunctionName)
+		}
+		return nil, fmt.Errorf("lambda: update function record: %w", err)
+	}
+	return &CodeStoredResponse{Record: rec, Revision: rev}, nil
+}
+
+// RollbackCreate frees the name of a create whose own code write failed. It
+// deletes only a record still storing at exactly the revision create returned;
+// an already-absent name is success.
+func (s *MetaService) RollbackCreate(ctx context.Context, req *RollbackCreateRequest, accountID string) (*RollbackCreateResponse, error) {
+	if !regionRE.MatchString(req.Region) {
+		return nil, badRegion()
+	}
+	kv, err := s.bucket(ctx, accountID)
+	if err != nil {
+		return nil, err
+	}
+	_, err = s.readStoring(ctx, kv, req.Region, req.FunctionName, req.Revision)
+	if errors.Is(err, ErrNotFound) {
+		return &RollbackCreateResponse{}, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	if err := kv.Delete(ctx, KVFunctionKey(req.Region, req.FunctionName), jetstream.LastRevision(req.Revision)); err != nil {
+		if errors.Is(err, jetstream.ErrKeyExists) {
+			return nil, conflict("Function was modified by another request: %s", req.FunctionName)
+		}
+		return nil, fmt.Errorf("lambda: delete function record: %w", err)
+	}
+	return &RollbackCreateResponse{}, nil
+}

spinifex/domains/lambda/function/meta_test.go

@@ -0,0 +1,263 @@
+package function
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/testkit"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	testAccount = "000000000001"
+	testRegion  = "ap-southeast-2"
+	testDigest  = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+)
+
+type clock struct{ t time.Time }
+
+func (c *clock) now() time.Time { return c.t }
+
+func newTestService(t *testing.T) (*MetaService, *clock) {
+	t.Helper()
+	_, nc, _ := testutil.StartTestJetStream(t)
+	svc := NewMetaService(testutil.NewJetStream(t, nc))
+	c := &clock{t: time.Date(2026, 11, 2, 10, 0, 0, 0, time.UTC)}
+	svc.now = c.now
+	return svc, c
+}
+
+func createReq(name, functionID string) *CreateRequest {
+	return &CreateRequest{Region: testRegion, Record: Record{
+		FunctionID: functionID, FunctionName: name, Role: "arn:aws:iam::000000000001:role/r",
+		Runtime: "provided.al2023", Handler: "bootstrap", Timeout: 3, MemorySize: 128,
+		Code: Code{RevisionID: "rev-" + functionID},
+	}}
+}
+
+func mustCreate(t *testing.T, svc *MetaService, name, functionID string) uint64 {
+	t.Helper()
+	resp, err := svc.Create(t.Context(), createReq(name, functionID), testAccount)
+	require.NoError(t, err)
+	return resp.Revision
+}
+
+func mustGet(t *testing.T, svc *MetaService, name string) *GetResponse {
+	t.Helper()
+	resp, err := svc.Get(t.Context(), &GetRequest{Region: testRegion, FunctionName: name}, testAccount)
+	require.NoError(t, err)
+	return resp
+}
+
+func requireCode(t *testing.T, err error, code string) {
+	t.Helper()
+	require.Error(t, err)
+	got, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok, "no AWS code on %v", err)
+	assert.Equal(t, code, got)
+}
+
+func TestMeta_CreateOwnsStateAndRejectsDuplicate(t *testing.T) {
+	svc, c := newTestService(t)
+	req := createReq("hello", "f1")
+	req.Record.State = StateActive
+	req.Record.Code.State = CodeStored
+	req.Record.AccountID = "someone-else"
+	resp, err := svc.Create(t.Context(), req, testAccount)
+	require.NoError(t, err)
+
+	got := mustGet(t, svc, "hello")
+	require.True(t, got.Found)
+	assert.Equal(t, resp.Revision, got.Revision)
+	assert.Equal(t, StatePending, got.Record.State, "the daemon owns state")
+	assert.Equal(t, CodeStoring, got.Record.Code.State)
+	assert.Equal(t, testAccount, got.Record.AccountID, "the account comes from the request header")
+	assert.Equal(t, FormatVersion, got.Record.FormatVersion)
+	assert.Equal(t, c.t.Add(ClaimTTL), got.Record.ClaimExpiresAt)
+
+	_, err = svc.Create(t.Context(), createReq("hello", "f2"), testAccount)
+	requireCode(t, err, awserrors.ErrorResourceConflictException)
+	assert.Equal(t, "f1", mustGet(t, svc, "hello").Record.FunctionID, "the first record is untouched")
+
+	other, err := svc.Create(t.Context(), createReq("hello", "f3"), "000000000002")
+	require.NoError(t, err, "names are per account")
+	assert.NotZero(t, other.Revision)
+}
+
+func TestMeta_CreateRefusesIncompleteRecords(t *testing.T) {
+	svc, _ := newTestService(t)
+	for name, mutate := range map[string]func(*Record){
+		"bad name":    func(r *Record) { r.FunctionName = "has space" },
+		"no id":       func(r *Record) { r.FunctionID = "" },
+		"no revision": func(r *Record) { r.Code.RevisionID = "" },
+	} {
+		t.Run(name, func(t *testing.T) {
+			req := createReq("ok", "f1")
+			mutate(&req.Record)
+			_, err := svc.Create(t.Context(), req, testAccount)
+			requireCode(t, err, awserrors.ErrorInvalidParameterValueException)
+		})
+	}
+}
+
+func TestMeta_CodeStoredUsesCAS(t *testing.T) {
+	svc, _ := newTestService(t)
+	rev := mustCreate(t, svc, "hello", "f1")
+
+	_, err := svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: rev + 5, SHA256: testDigest, Size: 9}, testAccount)
+	requireCode(t, err, awserrors.ErrorResourceConflictException)
+	assert.Equal(t, CodeStoring, mustGet(t, svc, "hello").Record.Code.State, "a stale revision changes nothing")
+
+	_, err = svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: rev, SHA256: "xyz", Size: 9}, testAccount)
+	requireCode(t, err, awserrors.ErrorInvalidParameterValueException)
+
+	resp, err := svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: rev, SHA256: testDigest, Size: 9}, testAccount)
+	require.NoError(t, err)
+	got := mustGet(t, svc, "hello")
+	assert.Equal(t, resp.Revision, got.Revision)
+	assert.Equal(t, CodeStored, got.Record.Code.State)
+	assert.Equal(t, testDigest, got.Record.Code.SHA256)
+	assert.Equal(t, int64(9), got.Record.Code.Size)
+	assert.Equal(t, StatePending, got.Record.State)
+	assert.Equal(t, ReasonCodeCreating, got.Record.StateReasonCode)
+	assert.Equal(t, ReasonCreating, got.Record.StateReason)
+
+	_, err = svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: resp.Revision, SHA256: testDigest, Size: 9}, testAccount)
+	requireCode(t, err, awserrors.ErrorResourceConflictException)
+
+	_, err = svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "ghost", Revision: 1, SHA256: testDigest}, testAccount)
+	requireCode(t, err, awserrors.ErrorResourceNotFoundException)
+}
+
+func TestMeta_RollbackDeletesOnlyStoringAtTheRightRevision(t *testing.T) {
+	svc, _ := newTestService(t)
+	rev := mustCreate(t, svc, "hello", "f1")
+
+	_, err := svc.RollbackCreate(t.Context(), &RollbackCreateRequest{Region: testRegion, FunctionName: "hello", Revision: rev + 1}, testAccount)
+	requireCode(t, err, awserrors.ErrorResourceConflictException)
+	require.True(t, mustGet(t, svc, "hello").Found)
+
+	_, err = svc.RollbackCreate(t.Context(), &RollbackCreateRequest{Region: testRegion, FunctionName: "hello", Revision: rev}, testAccount)
+	require.NoError(t, err)
+	assert.False(t, mustGet(t, svc, "hello").Found)
+	_, err = svc.RollbackCreate(t.Context(), &RollbackCreateRequest{Region: testRegion, FunctionName: "hello", Revision: rev}, testAccount)
+	require.NoError(t, err, "rolling back an absent name is success")
+	mustCreate(t, svc, "hello", "f2")
+	assert.Equal(t, "f2", mustGet(t, svc, "hello").Record.FunctionID, "the name is free again")
+
+	rev = mustGet(t, svc, "hello").Revision
+	stored, err := svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: rev, SHA256: testDigest, Size: 1}, testAccount)
+	require.NoError(t, err)
+	_, err = svc.RollbackCreate(t.Context(), &RollbackCreateRequest{Region: testRegion, FunctionName: "hello", Revision: stored.Revision}, testAccount)
+	requireCode(t, err, awserrors.ErrorResourceConflictException)
+	assert.True(t, mustGet(t, svc, "hello").Found, "a stored record is never rolled back")
+}
+
+func TestMeta_GetMissingAndUnknownFormat(t *testing.T) {
+	svc, _ := newTestService(t)
+	assert.False(t, mustGet(t, svc, "ghost").Found)
+
+	kv, err := svc.bucket(t.Context(), testAccount)
+	require.NoError(t, err)
+	data, err := json.Marshal(Record{FormatVersion: 99, FunctionName: "future"})
+	require.NoError(t, err)
+	_, err = kv.Create(t.Context(), KVFunctionKey(testRegion, "future"), data)
+	require.NoError(t, err)
+
+	_, err = svc.Get(t.Context(), &GetRequest{Region: testRegion, FunctionName: "future"}, testAccount)
+	requireCode(t, err, awserrors.ErrorServiceException)
+	_, err = svc.Create(t.Context(), createReq("future", "f1"), testAccount)
+	requireCode(t, err, awserrors.ErrorResourceConflictException)
+	_, err = svc.RollbackCreate(t.Context(), &RollbackCreateRequest{Region: testRegion, FunctionName: "future", Revision: 1}, testAccount)
+	requireCode(t, err, awserrors.ErrorServiceException)
+}
+
+func TestMeta_ExpiredStoringClaimIsTakenOver(t *testing.T) {
+	svc, c := newTestService(t)
+	oldRev := mustCreate(t, svc, "hello", "old")
+
+	c.t = c.t.Add(ClaimTTL - time.Second)
+	_, err := svc.Create(t.Context(), createReq("hello", "new"), testAccount)
+	requireCode(t, err, awserrors.ErrorResourceConflictException)
+	assert.Equal(t, "old", mustGet(t, svc, "hello").Record.FunctionID, "an unexpired claim holds the name")
+
+	c.t = c.t.Add(2 * time.Second)
+	newRev := mustCreate(t, svc, "hello", "new")
+	got := mustGet(t, svc, "hello")
+	assert.Equal(t, "new", got.Record.FunctionID)
+	assert.Equal(t, "rev-new", got.Record.Code.RevisionID)
+	assert.Equal(t, newRev, got.Revision)
+	assert.Equal(t, c.t.Add(ClaimTTL), got.Record.ClaimExpiresAt)
+
+	_, err = svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: oldRev, SHA256: testDigest, Size: 1}, testAccount)
+	requireCode(t, err, awserrors.ErrorResourceConflictException)
+	_, err = svc.RollbackCreate(t.Context(), &RollbackCreateRequest{Region: testRegion, FunctionName: "hello", Revision: oldRev}, testAccount)
+	requireCode(t, err, awserrors.ErrorResourceConflictException)
+	got = mustGet(t, svc, "hello")
+	assert.Equal(t, "new", got.Record.FunctionID, "the stale gateway cannot resurrect or free the old claim")
+	assert.Equal(t, CodeStoring, got.Record.Code.State)
+}
+
+func TestMeta_StoredRecordIsNeverTakenOver(t *testing.T) {
+	svc, c := newTestService(t)
+	rev := mustCreate(t, svc, "hello", "f1")
+	_, err := svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: rev, SHA256: testDigest, Size: 1}, testAccount)
+	require.NoError(t, err)
+
+	c.t = c.t.Add(10 * ClaimTTL)
+	_, err = svc.Create(t.Context(), createReq("hello", "f2"), testAccount)
+	requireCode(t, err, awserrors.ErrorResourceConflictException)
+	assert.Equal(t, "f1", mustGet(t, svc, "hello").Record.FunctionID)
+}
+
+func TestMeta_CodeStoredAfterExpiryIsRefused(t *testing.T) {
+	svc, c := newTestService(t)
+	rev := mustCreate(t, svc, "hello", "f1")
+
+	c.t = c.t.Add(ClaimTTL + time.Second)
+	_, err := svc.CodeStored(t.Context(), &CodeStoredRequest{Region: testRegion, FunctionName: "hello", Revision: rev, SHA256: testDigest, Size: 1}, testAccount)
+	requireCode(t, err, awserrors.ErrorResourceConflictException)
+	got := mustGet(t, svc, "hello")
+	assert.Equal(t, CodeStoring, got.Record.Code.State, "a refused update leaves the record unchanged")
+	assert.Equal(t, rev, got.Revision)
+}
+
+func TestMeta_RegionsAreIndependent(t *testing.T) {
+	svc, _ := newTestService(t)
+	sydney := createReq("hello", "syd")
+	oregon := createReq("hello", "pdx")
+	oregon.Region = "us-west-2"
+	_, err := svc.Create(t.Context(), sydney, testAccount)
+	require.NoError(t, err)
+	_, err = svc.Create(t.Context(), oregon, testAccount)
+	require.NoError(t, err, "the same name in another Region is a different function")
+
+	got, err := svc.Get(t.Context(), &GetRequest{Region: "us-west-2", FunctionName: "hello"}, testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, "pdx", got.Record.FunctionID)
+	assert.Equal(t, "us-west-2", got.Record.Region)
+	assert.Equal(t, "syd", mustGet(t, svc, "hello").Record.FunctionID)
+	assert.Equal(t, testRegion, mustGet(t, svc, "hello").Record.Region)
+
+	_, err = svc.Create(t.Context(), createReq("hello", "again"), testAccount)
+	requireCode(t, err, awserrors.ErrorResourceConflictException)
+}
+
+func TestMeta_RefusesBadRegion(t *testing.T) {
+	svc, _ := newTestService(t)
+	for _, region := range []string{"", "AP-SOUTH", "a.b", "a b", "x/y"} {
+		req := createReq("hello", "f1")
+		req.Region = region
+		_, err := svc.Create(t.Context(), req, testAccount)
+		requireCode(t, err, awserrors.ErrorInvalidParameterValueException)
+		_, err = svc.Get(t.Context(), &GetRequest{Region: region, FunctionName: "hello"}, testAccount)
+		requireCode(t, err, awserrors.ErrorInvalidParameterValueException)
+		_, err = svc.CodeStored(t.Context(), &CodeStoredRequest{Region: region, FunctionName: "hello", Revision: 1, SHA256: testDigest}, testAccount)
+		requireCode(t, err, awserrors.ErrorInvalidParameterValueException)
+		_, err = svc.RollbackCreate(t.Context(), &RollbackCreateRequest{Region: region, FunctionName: "hello", Revision: 1}, testAccount)
+		requireCode(t, err, awserrors.ErrorInvalidParameterValueException)
+	}
+}

spinifex/domains/lambda/function/record.go

@@ -0,0 +1,131 @@
+// Package function holds the Lambda function record and the daemon-owned
+// metadata service that creates and updates it.
+package function
+
+import "time"
+
+// FormatVersion is the only record layout this build reads or writes.
+const FormatVersion = 1
+
+// Function states and the code copy states.
+const (
+	StatePending = "Pending"
+	StateActive  = "Active"
+	StateFailed  = "Failed"
+
+	CodeStoring = "storing"
+	CodeStored  = "stored"
+	CodeFailed  = "failed"
+)
+
+// Reason reported while a function waits for a runtime image to exist.
+const (
+	ReasonCodeCreating = "Creating"
+	ReasonCreating     = "The function is waiting for its runtime image."
+)
+
+// Code describes the stored copy of a function's code.
+type Code struct {
+	RevisionID string `json:"revisionId"`
+	SHA256     string `json:"sha256,omitempty"`
+	Size       int64  `json:"size,omitempty"`
+	State      string `json:"state"`
+}
+
+// Record is one function. Its KV key both binds the name and holds it, so name
+// reservation and the record are a single atomic write.
+type Record struct {
+	FormatVersion   int    `json:"formatVersion"`
+	FunctionID      string `json:"functionId"`
+	AccountID       string `json:"accountId"`
+	Region          string `json:"region"`
+	FunctionName    string `json:"functionName"`
+	Role            string `json:"role"`
+	Runtime         string `json:"runtime"`
+	Handler         string `json:"handler"`
+	Description     string `json:"description,omitempty"`
+	Timeout         int64  `json:"timeout"`
+	MemorySize      int64  `json:"memorySize"`
+	EnvCipher       string `json:"envCipher,omitempty"`
+	State           string `json:"state"`
+	StateReason     string `json:"stateReason,omitempty"`
+	StateReasonCode string `json:"stateReasonCode,omitempty"`
+	Code            Code   `json:"code"`
+	// ClaimExpiresAt bounds how long a record still storing its code holds the name.
+	ClaimExpiresAt time.Time `json:"claimExpiresAt"`
+	CreatedAt      time.Time `json:"createdAt"`
+	LastModified   time.Time `json:"lastModified"`
+}
+
+// KV bucket and key layout. The bucket is per account and owned by the daemon.
+const (
+	KVBucketAccountPrefix  = "lambda-account-"
+	KVBucketAccountHistory = 1
+	kvFunctionKeyPrefix    = "fn."
+)
+
+// KVAccountBucket returns the per-account JetStream KV bucket name.
+func KVAccountBucket(accountID string) string { return KVBucketAccountPrefix + accountID }
+
+// KVFunctionKey returns the key that binds a function name in a Region and
+// holds its record. JetStream is shared across Regions, so the Region is part
+// of the key.
+func KVFunctionKey(region, name string) string { return kvFunctionKeyPrefix + region + "." + name }
+
+// NATS subjects served by the daemon's metadata service.
+const (
+	SubjectCreate         = "lambda.function.create"
+	SubjectGet            = "lambda.function.get"
+	SubjectCodeStored     = "lambda.function.code-stored"
+	SubjectRollbackCreate = "lambda.function.rollback-create"
+)
+
+// CreateRequest carries the record the gateway built; the daemon owns the
+// state, claim and timestamps and overwrites whatever the request holds there.
+type CreateRequest struct {
+	Region string `json:"region"`
+	Record Record `json:"record"`
+}
+
+// CreateResponse returns the KV revision the later steps must present.
+type CreateResponse struct {
+	Revision uint64 `json:"revision"`
+}
+
+// GetRequest names the function.
+type GetRequest struct {
+	Region       string `json:"region"`
+	FunctionName string `json:"functionName"`
+}
+
+// GetResponse holds the record when Found.
+type GetResponse struct {
+	Found    bool   `json:"found"`
+	Record   Record `json:"record"`
+	Revision uint64 `json:"revision"`
+}
+
+// CodeStoredRequest records that the code copy landed.
+type CodeStoredRequest struct {
+	Region       string `json:"region"`
+	FunctionName string `json:"functionName"`
+	Revision     uint64 `json:"revision"`
+	SHA256       string `json:"sha256"`
+	Size         int64  `json:"size"`
+}
+
+// CodeStoredResponse returns the record as stored and its revision.
+type CodeStoredResponse struct {
+	Record   Record `json:"record"`
+	Revision uint64 `json:"revision"`
+}
+
+// RollbackCreateRequest withdraws a create whose own code write failed.
+type RollbackCreateRequest struct {
+	Region       string `json:"region"`
+	FunctionName string `json:"functionName"`
+	Revision     uint64 `json:"revision"`
+}
+
+// RollbackCreateResponse is empty: success means the name is free.
+type RollbackCreateResponse struct{}

spinifex/domains/network/ovn/client.go

@@ -88,7 +88,6 @@ type Client interface {
 	CreateDHCPOptions(ctx context.Context, opts *nbdb.DHCPOptions) (string, error)
 	UpdateDHCPOptionsOptions(ctx context.Context, uuid string, options map[string]string) error
 	DeleteDHCPOptions(ctx context.Context, uuid string) error
-	FindDHCPOptionsByCIDR(ctx context.Context, cidr string) (*nbdb.DHCPOptions, error)
 	FindDHCPOptionsByExternalID(ctx context.Context, key, value string) (*nbdb.DHCPOptions, error)
 	ListDHCPOptions(ctx context.Context) ([]nbdb.DHCPOptions, error)
 

spinifex/domains/network/ovn/conformance_test.go

@@ -573,31 +573,26 @@ func conformDHCPOptions(t *testing.T, cli ovn.Client, ctx context.Context) {
 	require.NoError(t, err)
 	require.NotEmpty(t, uuid, "create must return the row UUID callers store")
 
-	byCIDR, err := cli.FindDHCPOptionsByCIDR(ctx, "10.0.1.0/24")
-	require.NoError(t, err)
-	assert.Equal(t, uuid, byCIDR.UUID)
-	assert.Equal(t, "10.0.1.1", byCIDR.Options["server_id"])
-
 	byExtID, err := cli.FindDHCPOptionsByExternalID(ctx, "spinifex:subnet_id", "subnet-1")
 	require.NoError(t, err)
 	assert.Equal(t, uuid, byExtID.UUID)
+	assert.Equal(t, "10.0.1.0/24", byExtID.CIDR)
+	assert.Equal(t, "10.0.1.1", byExtID.Options["server_id"])
 
 	require.NoError(t, cli.UpdateDHCPOptionsOptions(ctx, uuid, map[string]string{
 		"server_id": "10.0.1.1", "lease_time": "7200",
 	}))
-	byCIDR, err = cli.FindDHCPOptionsByCIDR(ctx, "10.0.1.0/24")
+	byExtID, err = cli.FindDHCPOptionsByExternalID(ctx, "spinifex:subnet_id", "subnet-1")
 	require.NoError(t, err)
-	assert.Equal(t, "7200", byCIDR.Options["lease_time"], "update must replace the options map")
+	assert.Equal(t, "7200", byExtID.Options["lease_time"], "update must replace the options map")
 
 	all, err := cli.ListDHCPOptions(ctx)
 	require.NoError(t, err)
 	assert.Len(t, all, 1)
 
-	_, err = cli.FindDHCPOptionsByCIDR(ctx, "10.9.9.0/24")
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "10.9.9.0/24")
 	_, err = cli.FindDHCPOptionsByExternalID(ctx, "spinifex:subnet_id", "subnet-nope")
-	assert.Error(t, err)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "subnet-nope")
 
 	require.NoError(t, cli.DeleteDHCPOptions(ctx, uuid))
 	all, err = cli.ListDHCPOptions(ctx)

spinifex/domains/network/ovn/live.go

@@ -826,20 +826,6 @@ func (c *LiveClient) DeleteDHCPOptions(ctx context.Context, uuid string) error {
 	return nil
 }
 
-func (c *LiveClient) FindDHCPOptionsByCIDR(ctx context.Context, cidr string) (*nbdb.DHCPOptions, error) {
-	var options []nbdb.DHCPOptions
-	err := c.client.WhereCache(func(o *nbdb.DHCPOptions) bool {
-		return o.CIDR == cidr
-	}).List(ctx, &options)
-	if err != nil {
-		return nil, fmt.Errorf("find DHCP options by CIDR: %w", err)
-	}
-	if len(options) == 0 {
-		return nil, fmt.Errorf("DHCP options for CIDR %q not found", cidr)
-	}
-	return &options[0], nil
-}
-
 func (c *LiveClient) FindDHCPOptionsByExternalID(ctx context.Context, key, value string) (*nbdb.DHCPOptions, error) {
 	var options []nbdb.DHCPOptions
 	err := c.client.WhereCache(func(o *nbdb.DHCPOptions) bool {

spinifex/domains/network/ovn/mock/mock.go

@@ -443,18 +443,6 @@ func (m *Client) DeleteDHCPOptions(_ context.Context, uuid string) error {
 	return nil
 }
 
-func (m *Client) FindDHCPOptionsByCIDR(_ context.Context, cidr string) (*nbdb.DHCPOptions, error) {
-	m.Mu.Lock()
-	defer m.Mu.Unlock()
-	for _, opts := range m.DHCPOpts {
-		if opts.CIDR == cidr {
-			result := *opts
-			return &result, nil
-		}
-	}
-	return nil, fmt.Errorf("DHCP options for CIDR %q not found", cidr)
-}
-
 func (m *Client) FindDHCPOptionsByExternalID(_ context.Context, key, value string) (*nbdb.DHCPOptions, error) {
 	m.Mu.Lock()
 	defer m.Mu.Unlock()

spinifex/domains/network/reconcile/apply.go

@@ -374,7 +374,8 @@ func (r *reconciler) applyPorts(ctx context.Context, intent IntentState, actual
 			continue
 		}
 
-		if _, err := r.ovn.GetLogicalSwitchPort(ctx, portName); err != nil {
+		existing, err := r.ovn.GetLogicalSwitchPort(ctx, portName)
+		if err != nil {
 			addrStr := spec.MAC.String() + " " + spec.PrivateIP.String()
 			lsp := &nbdb.LogicalSwitchPort{
 				Name:         portName,
@@ -398,6 +399,10 @@ func (r *reconciler) applyPorts(ctx context.Context, intent IntentState, actual
 			continue
 		}
 
+		if !spec.SuppressDHCP {
+			r.convergePortDHCP(ctx, existing, spec.SubnetID, res)
+		}
+
 		currentPGs, err := r.ovn.ListPortGroupsForPort(ctx, portName)
 		if err != nil {
 			slog.Warn("reconcile/apply: list port groups for port failed", "port", portName, "err", err)
@@ -420,6 +425,26 @@ func (r *reconciler) applyPorts(ctx context.Context, intent IntentState, actual
 	r.pruneOrphanPorts(ctx, intent, res)
 }
 
+// convergePortDHCP points an existing LSP's dhcpv4_options at its subnet's
+// current DHCP_Options row. The reference is weak, so deleting or recreating
+// the row leaves the port without OVN DHCP until it is re-linked here.
+func (r *reconciler) convergePortDHCP(ctx context.Context, lsp *nbdb.LogicalSwitchPort, subnetID string, res *passResult) {
+	dhcpOpts, err := r.ovn.FindDHCPOptionsByExternalID(ctx, "spinifex:subnet_id", subnetID)
+	if err != nil || dhcpOpts == nil {
+		return
+	}
+	if lsp.DHCPv4Options != nil && *lsp.DHCPv4Options == dhcpOpts.UUID {
+		return
+	}
+	lsp.DHCPv4Options = &dhcpOpts.UUID
+	if err := r.ovn.UpdateLogicalSwitchPort(ctx, lsp); err != nil {
+		slog.Warn("reconcile/apply: re-link port DHCP options failed", "port", lsp.Name, "subnet_id", subnetID, "err", err)
+		res.fail(classPort, lsp.Name, err)
+		return
+	}
+	slog.Info("reconcile/apply: re-linked port DHCP options", "port", lsp.Name, "subnet_id", subnetID)
+}
+
 // pruneOrphanPorts deletes guest LSPs whose spinifex:eni_id has no matching
 // intent ENI, closing the create-only gap that leaks ports across instance
 // terminate and host reinstall. DeletePort clears PG memberships then removes
@@ -872,7 +897,8 @@ func (r *reconciler) forgetIdlePorts(specs []policy.EIPSpec) {
 // never re-asserted, so a vpcd/OVN rebuild dropped it and its guest-port hop never
 // converged (inbound DNATs at the gateway but never reaches the guest; ICMP is
 // answered by the gateway LR, so ping works while TCP hangs). A user EIP on the same
-// private IP wins; the auto-assigned entry is skipped to avoid a duplicate rule.
+// private IP in the same VPC wins; the auto-assigned entry is skipped to avoid a
+// duplicate rule.
 func (r *reconciler) floatingIPSpecs(intent IntentState) []policy.EIPSpec {
 	specs := make([]policy.EIPSpec, 0, len(intent.EIPs)+len(intent.Ports))
 	for _, spec := range intent.EIPs {
@@ -883,7 +909,7 @@ func (r *reconciler) floatingIPSpecs(intent IntentState) []policy.EIPSpec {
 			continue
 		}
 		logicalIP := p.PrivateIP.String()
-		if _, hasEIP := intent.EIPs[logicalIP]; hasEIP {
+		if _, hasEIP := intent.EIPs[eipIntentKey(p.VPCID, logicalIP)]; hasEIP {
 			continue
 		}
 		specs = append(specs, policy.EIPSpec{
@@ -1014,7 +1040,7 @@ func (r *reconciler) applyPublicInstanceEgress(ctx context.Context, intent Inten
 		}
 	}
 	for _, e := range intent.EIPs {
-		r.ensureEIPEgressExemption(ctx, intent, e.VPCID, subnetIDForIP(intent.Subnets, e.LogicalIP), e.LogicalIP, res)
+		r.ensureEIPEgressExemption(ctx, intent, e.VPCID, subnetIDForIP(intent.Subnets, e.VPCID, e.LogicalIP), e.LogicalIP, res)
 	}
 }
 
@@ -1042,14 +1068,15 @@ func (r *reconciler) ensureEIPEgressExemption(ctx context.Context, intent Intent
 	}
 }
 
-// subnetIDForIP returns the SubnetID whose CIDR contains ip, or "" if none match.
-func subnetIDForIP(subnets map[string]topology.SubnetSpec, ip string) string {
+// subnetIDForIP returns the SubnetID in vpcID whose CIDR contains ip, or "" if
+// none match. VPC CIDRs overlap across tenants, so the CIDR alone is ambiguous.
+func subnetIDForIP(subnets map[string]topology.SubnetSpec, vpcID, ip string) string {
 	addr, err := netip.ParseAddr(ip)
 	if err != nil {
 		return ""
 	}
 	for _, s := range subnets {
-		if s.CIDR.IsValid() && s.CIDR.Contains(addr) {
+		if s.VPCID == vpcID && s.CIDR.IsValid() && s.CIDR.Contains(addr) {
 			return s.SubnetID
 		}
 	}

spinifex/domains/network/reconcile/apply_eip_concurrency_test.go

@@ -135,11 +135,11 @@ func TestApplyEIPs_HealthyPortDoesNotWaitForDeadOnes(t *testing.T) {
 	}
 
 	intent := IntentState{EIPs: map[string]policy.EIPSpec{
-		"10.0.0.1": {VPCID: "vpc-a", ExternalIP: "192.0.2.1", LogicalIP: "10.0.0.1", PortName: live},
+		eipIntentKey("vpc-a", "10.0.0.1"): {VPCID: "vpc-a", ExternalIP: "192.0.2.1", LogicalIP: "10.0.0.1", PortName: live},
 	}}
 	for i, name := range dead {
 		ip := "10.0.0." + string(rune('2'+i))
-		intent.EIPs[ip] = policy.EIPSpec{VPCID: "vpc-a", ExternalIP: "192.0.2.9", LogicalIP: ip, PortName: name}
+		intent.EIPs[eipIntentKey("vpc-a", ip)] = policy.EIPSpec{VPCID: "vpc-a", ExternalIP: "192.0.2.9", LogicalIP: ip, PortName: name}
 	}
 
 	res := &passResult{}

spinifex/domains/network/reconcile/apply_eip_overlap_test.go

@@ -0,0 +1,242 @@
+package reconcile
+
+import (
+	"context"
+	"net/netip"
+	"slices"
+	"sync"
+	"testing"
+
+	ec2eip "github.com/mulgadc/spinifex/spinifex/domains/ec2/eip"
+	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
+	"github.com/mulgadc/spinifex/spinifex/domains/network/external"
+	"github.com/mulgadc/spinifex/spinifex/domains/network/policy"
+	"github.com/mulgadc/spinifex/spinifex/domains/network/topology"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// hostBindings stands in for the routed-mode host route and proxy-ARP table.
+type hostBindings struct {
+	mu    sync.Mutex
+	bound map[string]struct{}
+}
+
+func (h *hostBindings) hooks() policy.HostEIPBinder {
+	return policy.HostEIPBinder{
+		Bind: func(eip policy.EIPSpec, _ string) error {
+			h.mu.Lock()
+			defer h.mu.Unlock()
+			h.bound[eip.ExternalIP] = struct{}{}
+			return nil
+		},
+		Unbind: func(externalIP string) error {
+			h.mu.Lock()
+			defer h.mu.Unlock()
+			delete(h.bound, externalIP)
+			return nil
+		},
+		List: func() ([]string, error) {
+			h.mu.Lock()
+			defer h.mu.Unlock()
+			out := make([]string, 0, len(h.bound))
+			for ip := range h.bound {
+				out = append(out, ip)
+			}
+			slices.Sort(out)
+			return out, nil
+		},
+	}
+}
+
+// seedOverlappingTenants writes two accounts' default-style VPCs on the same
+// CIDR, each with one ENI at 172.31.0.4. eni-a carries publicA on its stamp.
+func seedOverlappingTenants(t *testing.T, js jetstream.JetStream, publicA string) {
+	t.Helper()
+	for _, tenant := range []struct{ acct, suffix, mac, public string }{
+		{"111111111111", "a", "02:00:00:00:0a:04", publicA},
+		{"222222222222", "b", "02:00:00:00:0b:04", ""},
+	} {
+		vpcID, subnetID, sgID, eniID := "vpc-"+tenant.suffix, "subnet-"+tenant.suffix, "sg-"+tenant.suffix, "eni-"+tenant.suffix
+		seedKV(t, js, ec2vpc.KVBucketVPCs, tenant.acct+"."+vpcID, ec2vpc.VPCRecord{
+			VpcId: vpcID, CidrBlock: "172.31.0.0/16", AZ: "us-east-1a", VNI: 100,
+		})
+		seedKV(t, js, ec2vpc.KVBucketSubnets, tenant.acct+"."+subnetID, ec2vpc.SubnetRecord{
+			SubnetId: subnetID, VpcId: vpcID, CidrBlock: "172.31.0.0/20",
+		})
+		seedKV(t, js, ec2vpc.KVBucketSecurityGroups, tenant.acct+"."+sgID, ec2vpc.SecurityGroupRecord{
+			GroupId: sgID, VpcId: vpcID,
+		})
+		seedKV(t, js, ec2vpc.KVBucketENIs, tenant.acct+"."+eniID, ec2vpc.ENIRecord{
+			NetworkInterfaceId: eniID, SubnetId: subnetID, VpcId: vpcID,
+			PrivateIpAddress: "172.31.0.4", MacAddress: tenant.mac,
+			SecurityGroupIds: []string{sgID}, PublicIpAddress: tenant.public,
+		})
+	}
+}
+
+func seedAssociatedEIP(t *testing.T, js jetstream.JetStream, acct, suffix, publicIP, mac string) {
+	t.Helper()
+	seedKV(t, js, ec2eip.KVBucketEIPs, acct+".eipalloc-"+suffix, ec2eip.EIPRecord{
+		AllocationId: "eipalloc-" + suffix, PublicIp: publicIP, State: "associated",
+		AssociationId: "eipassoc-" + suffix, ENIId: "eni-" + suffix,
+		PrivateIp: "172.31.0.4", MacAddress: mac, VpcId: "vpc-" + suffix,
+	})
+}
+
+// Two tenants' guests share a private IP on overlapping VPC CIDRs. Tenant B
+// associating its own EIP must leave tenant A's public address, NAT row and
+// host binding alone, whether A's address is a user EIP or auto-assigned.
+func TestReconcile_OverlappingVPCsKeepEachTenantsPublicIP(t *testing.T) {
+	cases := []struct {
+		name    string
+		publicA string
+		eipA    bool
+	}{
+		{name: "eip beside eip", publicA: "203.0.113.10", eipA: true},
+		{name: "eip beside auto-assigned", publicA: "198.51.100.20"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx := context.Background()
+			js := startKV(t)
+			seedOverlappingTenants(t, js, tc.publicA)
+			if tc.eipA {
+				seedAssociatedEIP(t, js, "111111111111", "a", tc.publicA, "02:00:00:00:0a:04")
+			}
+
+			r, m := newTestReconciler(t)
+			hosts := &hostBindings{bound: map[string]struct{}{}}
+			nat, err := policy.NewNATManager(m, policy.NATModeRouted, policy.WithHostEIPBinder(hosts.hooks()))
+			if err != nil {
+				t.Fatalf("NewNATManager: %v", err)
+			}
+			r.nat = nat
+			r.reloadIntent = func(ctx context.Context) (IntentState, error) {
+				return LoadIntentFromKV(ctx, js, "us-east-1a")
+			}
+
+			pass := func() IntentState {
+				t.Helper()
+				intent, err := r.reloadIntent(ctx)
+				if err != nil {
+					t.Fatalf("LoadIntentFromKV: %v", err)
+				}
+				if err := r.Reconcile(ctx, intent); err != nil {
+					t.Fatalf("Reconcile: %v", err)
+				}
+				if err := r.ReconcileHostEIPs(ctx); err != nil {
+					t.Fatalf("ReconcileHostEIPs: %v", err)
+				}
+				return intent
+			}
+			assertPublic := func(stage, externalIP, port string) {
+				t.Helper()
+				row := findNATByExternal(m, "dnat_and_snat", externalIP)
+				if row == nil {
+					t.Fatalf("%s: dnat_and_snat for %s missing", stage, externalIP)
+				}
+				if got := row.ExternalIDs["spinifex:logical_port"]; got != topology.Port(port) {
+					t.Errorf("%s: %s row owned by %q, want %q", stage, externalIP, got, topology.Port(port))
+				}
+				if _, ok := hosts.bound[externalIP]; !ok {
+					t.Errorf("%s: host binding for %s missing", stage, externalIP)
+				}
+			}
+
+			pass()
+			assertPublic("before tenant B associates", tc.publicA, "eni-a")
+
+			seedAssociatedEIP(t, js, "222222222222", "b", "203.0.113.20", "02:00:00:00:0b:04")
+			for _, stage := range []string{"after tenant B associates", "on the next pass"} {
+				intent := pass()
+				// eni-a's stamp alone would keep A's row alive, so check that
+				// intent itself holds every tenant's association.
+				wantEIPs := map[string]string{eipIntentKey("vpc-b", "172.31.0.4"): "203.0.113.20"}
+				if tc.eipA {
+					wantEIPs[eipIntentKey("vpc-a", "172.31.0.4")] = tc.publicA
+				}
+				if len(intent.EIPs) != len(wantEIPs) {
+					t.Errorf("%s: intent holds %d EIPs, want %d: %+v", stage, len(intent.EIPs), len(wantEIPs), intent.EIPs)
+				}
+				for key, externalIP := range wantEIPs {
+					if got := intent.EIPs[key].ExternalIP; got != externalIP {
+						t.Errorf("%s: intent EIP %s = %q, want %q", stage, key, got, externalIP)
+					}
+				}
+				assertPublic(stage, tc.publicA, "eni-a")
+				assertPublic(stage, "203.0.113.20", "eni-b")
+			}
+		})
+	}
+}
+
+// The drop-gate exemption must name the EIP's own subnet, not a subnet in
+// another VPC whose CIDR happens to contain the same private IP.
+func TestSubnetIDForIP_ScopedToVPC(t *testing.T) {
+	subnets := map[string]topology.SubnetSpec{
+		"subnet-a": {SubnetID: "subnet-a", VPCID: "vpc-a", CIDR: netip.MustParsePrefix("172.31.0.0/20")},
+		"subnet-b": {SubnetID: "subnet-b", VPCID: "vpc-b", CIDR: netip.MustParsePrefix("172.31.0.0/20")},
+	}
+	for range 20 {
+		if got := subnetIDForIP(subnets, "vpc-a", "172.31.0.4"); got != "subnet-a" {
+			t.Fatalf("vpc-a lookup = %q, want subnet-a", got)
+		}
+		if got := subnetIDForIP(subnets, "vpc-b", "172.31.0.4"); got != "subnet-b" {
+			t.Fatalf("vpc-b lookup = %q, want subnet-b", got)
+		}
+	}
+	if got := subnetIDForIP(subnets, "vpc-c", "172.31.0.4"); got != "" {
+		t.Errorf("lookup in a VPC with no subnets = %q, want empty", got)
+	}
+}
+
+// egressCall is one EnsureEIPInstanceEgress invocation.
+type egressCall struct{ vpcID, subnetID, instanceIP string }
+
+// recordingIGW records EnsureEIPInstanceEgress calls instead of writing OVN.
+type recordingIGW struct {
+	external.IGWManager
+
+	calls []egressCall
+}
+
+func (g *recordingIGW) EnsureEIPInstanceEgress(_ context.Context, vpcID, subnetID, instanceIP string) error {
+	g.calls = append(g.calls, egressCall{vpcID, subnetID, instanceIP})
+	return nil
+}
+
+// An EIP's drop-gate exemption goes to its own VPC's subnet even when another
+// tenant's gated subnet has the same CIDR.
+func TestApplyPublicInstanceEgress_EIPExemptsOwnVPCSubnet(t *testing.T) {
+	cidr := netip.MustParsePrefix("172.31.0.0/20")
+	intent := IntentState{
+		Subnets: map[string]topology.SubnetSpec{
+			"subnet-a": {SubnetID: "subnet-a", VPCID: "vpc-a", CIDR: cidr},
+			"subnet-b": {SubnetID: "subnet-b", VPCID: "vpc-b", CIDR: cidr},
+		},
+		EIPs: map[string]policy.EIPSpec{
+			eipIntentKey("vpc-b", "172.31.0.4"): {VPCID: "vpc-b", ExternalIP: "203.0.113.20", LogicalIP: "172.31.0.4"},
+		},
+		DropGates: map[string]SubnetEgressIntent{},
+	}
+	for _, s := range intent.Subnets {
+		intent.DropGates[subnetEgressKey(s.SubnetID, netip.MustParsePrefix("0.0.0.0/0"))] = SubnetEgressIntent{
+			VPCID: s.VPCID, SubnetID: s.SubnetID, DestCIDR: netip.MustParsePrefix("0.0.0.0/0"),
+		}
+	}
+
+	for range 20 {
+		r, _ := newTestReconciler(t)
+		igw := new(recordingIGW)
+		r.igw = igw
+		res := new(passResult)
+		r.applyPublicInstanceEgress(context.Background(), intent, ActualState{}, res)
+		want := []egressCall{{"vpc-b", "subnet-b", "172.31.0.4"}}
+		if !slices.Equal(igw.calls, want) {
+			t.Fatalf("EnsureEIPInstanceEgress calls = %+v, want %+v", igw.calls, want)
+		}
+		if len(res.failures) != 0 {
+			t.Fatalf("pass failures = %+v, want none", res.failures)
+		}
+	}
+}

spinifex/domains/network/reconcile/apply_guest_port_idle_test.go

@@ -27,7 +27,7 @@ func captureReconcileLogs(t *testing.T) *bytes.Buffer {
 
 func idleEIPIntent(idle bool) IntentState {
 	intent := IntentState{EIPs: map[string]policy.EIPSpec{
-		"10.0.0.4": {VPCID: "vpc-a", ExternalIP: "192.0.2.46", LogicalIP: "10.0.0.4", PortName: "port-eni-1"},
+		eipIntentKey("vpc-a", "10.0.0.4"): {VPCID: "vpc-a", ExternalIP: "192.0.2.46", LogicalIP: "10.0.0.4", PortName: "port-eni-1"},
 	}}
 	if idle {
 		intent.IdlePorts = map[string]string{"port-eni-1": "i-stopped"}

spinifex/domains/network/reconcile/apply_test.go

@@ -115,6 +115,53 @@ func TestReconcile_PortSuppressDHCP(t *testing.T) {
 	}
 }
 
+// A deleted DHCP_Options row clears dhcpv4_options on every LSP that pointed at
+// it. The next pass recreates the row under a new UUID and must re-link the
+// existing ports to it, leaving a SuppressDHCP port unlinked.
+func TestReconcile_ExistingPortRelinkedToRecreatedDHCPOptions(t *testing.T) {
+	rec, m := newTestReconciler(t)
+	ctx := context.Background()
+
+	intent := freshIntent(t)
+	mac, _ := net.ParseMAC("02:00:00:00:00:09")
+	intent.Ports["eni-static"] = topology.PortSpec{
+		PortID: "eni-static", SubnetID: "subnet-a", VPCID: "vpc-a",
+		PrivateIP: netip.MustParseAddr("10.0.1.11"), MAC: mac,
+		SGIDs: []string{"sg-a"}, SuppressDHCP: true,
+	}
+	if err := rec.Reconcile(ctx, intent); err != nil {
+		t.Fatalf("Reconcile #1: %v", err)
+	}
+
+	old, err := m.FindDHCPOptionsByExternalID(ctx, "spinifex:subnet_id", "subnet-a")
+	if err != nil {
+		t.Fatalf("DHCP options after first pass: %v", err)
+	}
+	if err := m.DeleteDHCPOptions(ctx, old.UUID); err != nil {
+		t.Fatalf("DeleteDHCPOptions: %v", err)
+	}
+	portName := "port-" + intent.Ports["eni-a"].PortID
+	m.Ports[portName].DHCPv4Options = nil
+
+	if err := rec.Reconcile(ctx, intent); err != nil {
+		t.Fatalf("Reconcile #2: %v", err)
+	}
+
+	cur, err := m.FindDHCPOptionsByExternalID(ctx, "spinifex:subnet_id", "subnet-a")
+	if err != nil {
+		t.Fatalf("DHCP options after second pass: %v", err)
+	}
+	if cur.UUID == old.UUID {
+		t.Fatalf("DHCP options row was not recreated")
+	}
+	if got := m.Ports[portName].DHCPv4Options; got == nil || *got != cur.UUID {
+		t.Errorf("existing LSP dhcpv4_options = %v, want %s", got, cur.UUID)
+	}
+	if got := m.Ports["port-eni-static"].DHCPv4Options; got != nil {
+		t.Errorf("SuppressDHCP LSP must stay unlinked, got %q", *got)
+	}
+}
+
 func TestReconcile_PortJoinsPortGroupAtomically(t *testing.T) {
 	rec, m := newTestReconciler(t)
 	ctx := context.Background()
@@ -1160,10 +1207,10 @@ func TestFloatingIPSpecs(t *testing.T) {
 	intent := IntentState{
 		EIPs: map[string]policy.EIPSpec{
 			// User EIP on 172.31.0.9.
-			"172.31.0.9": {VPCID: "vpc-a", ExternalIP: "192.168.1.200", LogicalIP: "172.31.0.9", PortName: topology.Port("eni-eip")},
+			eipIntentKey("vpc-a", "172.31.0.9"): {VPCID: "vpc-a", ExternalIP: "192.168.1.200", LogicalIP: "172.31.0.9", PortName: topology.Port("eni-eip")},
 			// User EIP that also has an auto-assigned public IP recorded on its
 			// port: the EIP must win and the auto-assigned entry be skipped.
-			"172.31.0.4": {VPCID: "vpc-a", ExternalIP: "192.168.1.201", LogicalIP: "172.31.0.4", PortName: topology.Port("eni-dup")},
+			eipIntentKey("vpc-a", "172.31.0.4"): {VPCID: "vpc-a", ExternalIP: "192.168.1.201", LogicalIP: "172.31.0.4", PortName: topology.Port("eni-dup")},
 		},
 		Ports: map[string]topology.PortSpec{
 			// Auto-assigned public IP with no user EIP -> must produce a spec.
@@ -1172,6 +1219,9 @@ func TestFloatingIPSpecs(t *testing.T) {
 			// Same private IP as the user EIP above -> deduped out.
 			"eni-dup": {PortID: "eni-dup", VPCID: "vpc-a", PrivateIP: netip.MustParseAddr("172.31.0.4"),
 				PublicIP: netip.MustParseAddr("192.168.1.117"), MAC: mac},
+			// Same private IP as vpc-a's user EIP but in another VPC -> kept.
+			"eni-other-vpc": {PortID: "eni-other-vpc", VPCID: "vpc-b", PrivateIP: netip.MustParseAddr("172.31.0.4"),
+				PublicIP: netip.MustParseAddr("192.168.1.118"), MAC: mac},
 			// No public IP -> skipped.
 			"eni-nopub": {PortID: "eni-nopub", VPCID: "vpc-b", PrivateIP: netip.MustParseAddr("172.31.0.6")},
 		},
@@ -1203,9 +1253,12 @@ func TestFloatingIPSpecs(t *testing.T) {
 	if _, ok := byExternal["192.168.1.117"]; ok {
 		t.Errorf("auto-assigned 192.168.1.117 should be deduped by the user EIP on 172.31.0.4")
 	}
+	if _, ok := byExternal["192.168.1.118"]; !ok {
+		t.Errorf("auto-assigned 192.168.1.118 in vpc-b must not be deduped by vpc-a's EIP on the same private IP")
+	}
 	// No public IP -> no spec for that port.
-	if len(specs) != 3 {
-		t.Errorf("want 3 specs (2 EIP + 1 auto), got %d: %+v", len(specs), specs)
+	if len(specs) != 4 {
+		t.Errorf("want 4 specs (2 EIP + 2 auto), got %d: %+v", len(specs), specs)
 	}
 
 	// A NAT gateway's address holds an snat, never a dnat_and_snat, so it must
@@ -1230,7 +1283,7 @@ func TestHostBindSpecs_CarriesNATGatewayAddresses(t *testing.T) {
 
 	intent := IntentState{
 		EIPs: map[string]policy.EIPSpec{
-			"172.31.0.9": {VPCID: "vpc-a", ExternalIP: "192.168.1.200", LogicalIP: "172.31.0.9", PortName: topology.Port("eni-eip")},
+			eipIntentKey("vpc-a", "172.31.0.9"): {VPCID: "vpc-a", ExternalIP: "192.168.1.200", LogicalIP: "172.31.0.9", PortName: topology.Port("eni-eip")},
 		},
 		NATGWs: map[string]policy.NATGWSpec{
 			// One gateway, two associated subnets: the same address arrives
@@ -1283,7 +1336,7 @@ func TestHostBindSpecs_DedupesAgainstAnExistingEIP(t *testing.T) {
 
 	specs := r.hostBindSpecs(IntentState{
 		EIPs: map[string]policy.EIPSpec{
-			"172.31.0.9": {VPCID: "vpc-a", ExternalIP: "192.168.1.240", LogicalIP: "172.31.0.9",
+			eipIntentKey("vpc-a", "172.31.0.9"): {VPCID: "vpc-a", ExternalIP: "192.168.1.240", LogicalIP: "172.31.0.9",
 				PortName: topology.Port("eni-eip"), MAC: mac.String()},
 		},
 		NATGWs: map[string]policy.NATGWSpec{
@@ -1431,7 +1484,7 @@ func TestReconcile_MarksIGWAttachedOnlyOnSuccess(t *testing.T) {
 		// A distributed-NAT gateway LRP is link-local and gates the probe off,
 		// so an EIP is what gives the datapath check a target.
 		intent := igwIntent(t)
-		intent.EIPs["10.0.1.5"] = policy.EIPSpec{VPCID: "vpc-a", ExternalIP: "203.0.113.5", LogicalIP: "10.0.1.5"}
+		intent.EIPs[eipIntentKey("vpc-a", "10.0.1.5")] = policy.EIPSpec{VPCID: "vpc-a", ExternalIP: "203.0.113.5", LogicalIP: "10.0.1.5"}
 		if err := rec.Reconcile(ctx, intent); err != nil {
 			t.Fatalf("Reconcile = %v, want nil", err)
 		}
@@ -1607,7 +1660,7 @@ func TestReconcile_FailedAddEIPReportsPassIncomplete(t *testing.T) {
 	rec.nat = &stubNAT{NATManager: rec.nat, addErr: errors.New("ovsdb unreachable")}
 
 	intent := freshIntent(t)
-	intent.EIPs["10.0.1.10"] = policy.EIPSpec{
+	intent.EIPs[eipIntentKey("vpc-a", "10.0.1.10")] = policy.EIPSpec{
 		VPCID: "vpc-a", ExternalIP: "192.168.1.10", LogicalIP: "10.0.1.10",
 		PortName: topology.Port("eni-a"),
 	}

spinifex/domains/network/reconcile/intent.go

@@ -32,7 +32,7 @@ type IntentState struct {
 	Ports       map[string]topology.PortSpec
 	SGs         map[string]policy.SGSpec
 	IGWs        map[string]external.IGWSpec // attached only
-	EIPs        map[string]policy.EIPSpec   // keyed by logicalIP; associated only
+	EIPs        map[string]policy.EIPSpec   // keyed by eipIntentKey; associated only
 	NATGWs      map[string]policy.NATGWSpec
 	IGWRoutes   map[string]SubnetEgressIntent // per (subnet, dest) IGW egress reroute
 	NATGWRoutes map[string]SubnetEgressIntent // per (subnet, dest) NATGW egress reroute
@@ -57,6 +57,12 @@ type SubnetEgressIntent struct {
 	DestCIDR netip.Prefix
 }
 
+// eipIntentKey is the EIPs map key. Private IPs repeat across VPCs (every
+// default VPC is 172.31.0.0/16), so the VPC is part of the identity.
+func eipIntentKey(vpcID, privateIP string) string {
+	return vpcID + "|" + privateIP
+}
+
 // subnetEgressKey is the IGWRoutes/NATGWRoutes map key.
 func subnetEgressKey(subnetID string, prefix netip.Prefix) string {
 	return subnetID + "|" + prefix.String()
@@ -565,7 +571,7 @@ func loadEIPs(ctx context.Context, js jetstream.JetStream, localVPCs map[string]
 		}
 		// MAC drives distributed dnat_and_snat; empty falls back to centralised.
 		spec.MAC = rec.MacAddress
-		out[rec.PrivateIp] = spec
+		out[eipIntentKey(rec.VpcId, rec.PrivateIp)] = spec
 	}
 	return nil
 }

spinifex/domains/network/reconcile/intent_test.go

@@ -183,10 +183,10 @@ func TestLoadIntentFromKV_EIPStateFilter(t *testing.T) {
 		t.Fatalf("LoadIntentFromKV: %v", err)
 	}
 
-	if _, ok := intent.EIPs["10.0.1.5"]; !ok {
+	if _, ok := intent.EIPs[eipIntentKey("vpc-local", "10.0.1.5")]; !ok {
 		t.Errorf("associated EIP missing from intent")
 	}
-	if _, ok := intent.EIPs["10.0.1.6"]; ok {
+	if _, ok := intent.EIPs[eipIntentKey("vpc-local", "10.0.1.6")]; ok {
 		t.Errorf("non-associated EIP leaked into intent")
 	}
 }

spinifex/domains/network/reconcile/scenario_kitchensink_live_test.go

@@ -60,7 +60,7 @@ func kitchenSinkIntent(t *testing.T) IntentState {
 			"vpc-a": {VPCID: "vpc-a", InternetGatewayID: "igw-a"},
 		},
 		EIPs: map[string]policy.EIPSpec{
-			"10.0.1.10": {VPCID: "vpc-a", ExternalIP: "192.168.0.50", LogicalIP: "10.0.1.10",
+			eipIntentKey("vpc-a", "10.0.1.10"): {VPCID: "vpc-a", ExternalIP: "192.168.0.50", LogicalIP: "10.0.1.10",
 				PortName: topology.Port("eni-pub"), MAC: "02:00:00:00:00:01"},
 		},
 		NATGWs: map[string]policy.NATGWSpec{