fix(rds): cap an agent command by the budget its issuer has left

The agent measured each command's deadline from receipt, so a first copy
delayed by a gateway or bus outage was granted a whole fresh budget when
its issuer had nearly given up; a late quiesce could then take a hold
nobody would release. The issuer now stamps every republished copy with
the budget it has left, a duration that needs no clock agreement, and the
agent uses the smaller of that and its own type budget.

contracts/ecs/v1/messages.go

@@ -126,8 +126,8 @@ type Assign struct {
 	ENIPrivateIP  string `json:"eniPrivateIp,omitempty"`
 	ENISubnetID   string `json:"eniSubnetId,omitempty"`
 	// TaskRoleARN is the task's IAM role (from the taskdef); when set, the agent
-	// serves its credentials at 169.254.170.2 under CredID, defaulting CredID to
-	// the taskID when the scheduler leaves it empty.
+	// serves its credentials at 169.254.170.2 under CredID, a scheduler-minted
+	// secret. An assign without a CredID gets no credentials.
 	CredID      string `json:"credId,omitempty"`
 	TaskRoleARN string `json:"taskRoleArn,omitempty"`
 	// ExecutionRoleARN is the task's execution role (from the taskdef); when set,

docs/COMMANDS.md

@@ -152,12 +152,12 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | `spx admin account create` | `--name` (required) | Connects to NATS → refuses if an account already holds the name (case- and whitespace-insensitive) → reserves the name in `spinifex-account-names` KV → CAS loop on `spinifex-account-counter:next_id` for sequential 12-digit ID → creates Account record in `spinifex-accounts` KV → creates default VPC → creates `admin` user, AdministratorAccess policy (Action:*, Resource:*) and one access key → prints credentials and writes a `spinifex-<name>` AWS CLI profile |
 | `spx admin account create --remote` | `--name` (required), `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA), `--client-token` (default: generated), `--source` (default: `spx-cli`) | Same account, created through `POST /admin/CreateAccount` over SigV4 instead of NATS — the path the public signup form uses, so a shell can exercise it. Credentials come from the standard AWS chain (env vars or `AWS_PROFILE`), never from the cluster master key; the caller must be an IAM user in the super-admin account holding `spinifex:CreateAccount`. Reuse the printed `--client-token` to retry: a fresh token would create a second account. |
 | `spx admin account list` | — | Connects to NATS → IAMService.ListAccounts() → prints table with Account ID, Name, Status, Created |
-| `spx admin account list --remote` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Same listing read through `POST /admin/ListAccounts` over SigV4 instead of NATS, so an off-cluster operator or the load-test harness sees its tenants without SSH access to a node. Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:ListAccounts`. |
+| `spx admin account list --remote` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Same listing read through `POST /admin/ListAccounts` over SigV4 instead of NATS, so an off-cluster operator or the load-test harness sees its tenants without SSH access to a node. Credentials come from the standard AWS credential chain: on a cluster node that means `AWS_PROFILE=spinifex` (there is no default profile, so without it the command fails to find credentials), or equivalent credentials; the caller must be an IAM user in the super-admin account holding `spinifex:ListAccounts`. |
 | `spx admin account quota get <account-id>` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Reads the limits in force for one account through `POST /admin/GetAccountQuota` over SigV4, and reports for each dimension whether it came from the `[quota]` block in `awsgw.toml` (`config`) or from an override stored against the account (`override`). Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:GetAccountQuota`. |
 | `spx admin account quota set <account-id>` | `--vcpus`, `--vpcs`, `--subnets`, `--eips`, `--volumes`, `--volumes-gib`, `--rds-instances`, `--load-balancers`, `--clear`, plus `--endpoint`, `--region`, `--ca-bundle` | Overrides individual limits for one account through `POST /admin/PutAccountQuota`, leaving every dimension not named inheriting the configured baseline. `-1` uncaps a dimension; `0` is a real limit that denies every request on it. `--clear` removes the whole override record and cannot be combined with a limit flag. The account must already exist. Prints the resulting limits, so a caller never has to guess what it applied. The caller must hold `spinifex:PutAccountQuota`. |
-| `spx admin account delete <account-id>` | `--dry-run`, `--force`, `--yes` (requires `--name`), `--name` (account name, must match the stored record) | Prints the inventory it is about to remove → confirms the account name (prompted unless `--name` is given) → marks the account `TERMINATING`, which blocks its credentials cluster-wide → deletes its resources in dependency order (compute, attachments, storage, network, platform, identity, account), waiting for each stage to actually drain before starting the next → seals the account's quota override record so no later write can recreate it, which needs every gateway node the cluster configures to answer a read-only probe first (otherwise nothing is sealed and the account stays `TERMINATING`) → releases the name reservation and quota counter → deletes the account record. Anything that will not delete is reported as **stuck** with the reason, and the account stays `TERMINATING` so the residue keeps an owner. `000000000000` and `000000000001` are refused. `--dry-run` lists and changes nothing. The compute stage reaps ECS clusters, EKS nodegroups and clusters, and RDS instances before it reaps plain instances, because their backing VMs belong to the system account and a tenant-scoped instance listing cannot see them. The storage stage reaps the account's S3 buckets: uploads in flight are aborted, then the objects are removed a page at a time, then the bucket. Only buckets the account owns are ever listed, so the shared platform buckets are out of reach rather than filtered out. Platform-managed serving endpoints are reaped ahead of instances, because only their own delete releases the ENI and attached data volume alongside the VM, and spot instance requests are cancelled first so a live request cannot launch a replacement for something already terminated. The network stage reaps load balancers, then target groups, then NAT gateways and egress-only internet gateways before the subnets, security groups and addresses they hold. The platform stage reaps ACM certificates. |
+| `spx admin account delete <account-id>` | `--dry-run`, `--force`, `--yes` (requires `--name`), `--name` (account name, must match the stored record), `--endpoint`, `--region`, `--ca-bundle`, `--client-token` | Always runs through `POST /admin/DeleteAccount` over SigV4 and follows `DescribeAccountDeletion` to completion; the CLI never tears an account down in its own process, so a CLI delete and an API delete cannot race. Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:DeleteAccount` and `spinifex:DescribeAccountDeletion`. Prints the inventory it is about to remove → confirms the account name (prompted unless `--name` is given) → marks the account `TERMINATING`, which blocks its credentials cluster-wide → deletes its resources in dependency order (compute, attachments, storage, network, platform, identity, account), waiting for each stage to actually drain before starting the next → seals the account's quota override record so no later write can recreate it, which needs every gateway node the cluster configures to answer a read-only probe first (otherwise nothing is sealed and the account stays `TERMINATING`) → releases the name reservation and quota counter → deletes the account record. Anything that will not delete is reported as **stuck** with the reason, and the account stays `TERMINATING` so the residue keeps an owner. `000000000000` and `000000000001` are refused. `--dry-run` lists and changes nothing. The compute stage reaps ECS clusters, EKS nodegroups and clusters, and RDS instances before it reaps plain instances, because their backing VMs belong to the system account and a tenant-scoped instance listing cannot see them. The storage stage reaps the account's S3 buckets: uploads in flight are aborted, then the objects are removed a page at a time, then the bucket. Only buckets the account owns are ever listed, so the shared platform buckets are out of reach rather than filtered out. Platform-managed serving endpoints are reaped ahead of instances, because only their own delete releases the ENI and attached data volume alongside the VM, and spot instance requests are cancelled first so a live request cannot launch a replacement for something already terminated. The network stage reaps load balancers, then target groups, then NAT gateways and egress-only internet gateways before the subnets, security groups and addresses they hold. The platform stage reaps ACM certificates. |
 | `spx admin account delete --force` | as above | Escalates only for resources the ordinary API refuses: it clears a volume attachment in the control plane without the guest's cooperation (`ec2.ForceDetachVolume`), hard-destroys after a graceful attempt times out, and treats an already-missing resource as deleted. It never reorders the stages and never widens scope beyond the account. Use it once the guest is being destroyed — this is the escape from a volume stuck attached to an instance that will not terminate, where neither can otherwise be deleted. |
-| `spx admin account delete --remote` | as above, plus `--endpoint`, `--region`, `--ca-bundle`, `--client-token` | Same teardown driven through `POST /admin/DeleteAccount` over SigV4 instead of NATS, then follows `DescribeAccountDeletion` to completion, printing each stage as it lands. Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:DeleteAccount` and `spinifex:DescribeAccountDeletion`. |
+| `spx admin account delete --remote` | as above | Accepted for existing scripts and ignored: every delete already goes through the admin API. |
 | `spx admin principal create <name>` | `--grant` (repeatable or comma-separated; default: every admin method — `CreateAccount`, `DeleteAccount`, `DescribeAccountDeletion`, `GetAccountQuota`, `ListAccounts`, `PutAccountQuota`) | Creates an IAM user in the super-admin account with an inline `spx-admin-methods` policy allowing the requested methods, revokes any access key left from a previous run, and mints one new key. Each method is named rather than granted as `spinifex:*`, so a later addition to the admin surface is not authorised by an existing key. The principal's grants are exactly that one policy: re-running with fewer `--grant` values narrows it, and any other inline policy on the user is removed, so a rename cannot leave a wider grant behind. The secret is printed once; store it in a secret manager or an AWS profile, never in a config file. `admin` is refused as a name — it is the account's own administrator and rotating its key would revoke the bootstrap credential. |
 | `spx admin principal list` | — | Lists every principal in the super-admin account with its access-key count and what it may call. An attached policy is reported by name (`AdministratorAccess (attached)`) rather than expanded, which is how an unscoped principal is spotted. |
 | `spx admin principal revoke <name>` | — | Deletes every access key the named principal holds, leaving the user and its policy in place. Immediate and cluster-wide; re-issue with `principal create`. |

docs/admin/account-management/README.md

@@ -236,12 +236,12 @@ Each stage runs to empty before the next begins, so nothing is deleted while som
 A large account takes minutes to tear down. The API does not hold the connection open for it: `DeleteAccount` starts the job, returns a `deletionId` immediately, and progress is read from `DescribeAccountDeletion`.
 
 ```bash
-AWS_PROFILE=spinifex-operator spx admin account delete 000000000042 --remote \
+AWS_PROFILE=spinifex-operator spx admin account delete 000000000042 \
   --name customer@example.com --yes \
   --endpoint https://node1.example.com:9999 --region us-west-1
 ```
 
-`spx --remote` starts the job and then polls, printing each stage as it lands. Your own client can do the same, or start the job and check back later — the deletion record outlives the account, so `DescribeAccountDeletion` stays answerable after the account is gone.
+`spx admin account delete` always goes through the admin API, so it signs with the standard AWS credential chain. On a cluster node that means `AWS_PROFILE=spinifex` (there is no default profile, so without it the command cannot find credentials), or equivalent credentials for an IAM user holding `spinifex:DeleteAccount` and `spinifex:DescribeAccountDeletion`, and a reachable gateway. The older `--remote` flag is accepted and ignored. The command starts the job and then polls, printing each stage as it lands. Your own client can do the same, or start the job and check back later — the deletion record outlives the account, so `DescribeAccountDeletion` stays answerable after the account is gone.
 
 One job runs per account. A second `DeleteAccount` with a different token while one is running returns `OperationInProgress`; with the *same* token it returns the running job, which is what makes a retry safe.
 

docs/containers/eks/README.md

@@ -58,7 +58,7 @@ Authentication uses **EKS access entries** (the API authentication mode) — IAM
 **Spinifex specifics**
 
 - **Access entries are the only auth path.** `authentication_mode` accepts `API` (the default when unset) and `API_AND_CONFIG_MAP`; bare `CONFIG_MAP` is rejected. Spinifex has no `aws-auth` ConfigMap path, so `API_AND_CONFIG_MAP` behaves as `API`: grant access with access entries.
-- **Security groups are auto-managed.** Spinifex creates the cluster and node-group security groups deterministically (e.g. `eks-cluster-<name>-nodegroup-sg`); `vpc_config.security_group_ids` is ignored. To expose a workload you add an ingress rule to the auto-managed node-group SG.
+- **The cluster security group is auto-managed.** As on EKS, Spinifex creates one `eks-cluster-sg-<name>-<id>` in your VPC, tagged `kubernetes.io/cluster/<name>=owned` and `aws:eks:cluster-name`, allowing all traffic from itself. It is attached to the workers and the private-endpoint network interface, and `DescribeCluster` reports it as `clusterSecurityGroupId`. A launch template that names security groups replaces it on the workers, as on AWS. Clusters created before this change keep their `-control-plane-sg` and `-nodegroup-sg` until you recreate them. Deleting a cluster deletes the cluster security group and removes only the rules on your other security groups that reference it. `vpc_config.security_group_ids` is ignored.
 - **The worker image is fixed.** Node groups always boot Spinifex's `eks-node` image. `ami_type` is recorded but does not select the image, and the image must be registered on the cluster before you create a node group.
 - **Workers launch with IMDSv2 required and an IMDS hop limit of 2.**
   This matches AWS managed node groups and lets a pod without IRSA reach the node role through the SDK default credential chain.
@@ -143,6 +143,25 @@ No user-facing operation replaces a control plane, and automatic replacement doe
 Recreate the cluster to adopt it.
 User-workload injection through a cluster created with an older image is not measured.
 
+### Add-on versions
+
+Add-on versions use the AWS form `v<x.y.z>-eksbuild.<n>`, for example `v1.40.1-eksbuild.1`.
+`DescribeAddonVersions` returns versions in this form, newest first, and every other add-on response reports the version the same way.
+Spinifex ships one EKS build of each upstream version, so `<n>` is `1` today.
+
+`DescribeAddonVersions` honours `addonName`, `kubernetesVersion`, `types`, `publishers`, `owners`, `maxResults` and `nextToken`.
+Each version carries `compatibilities` with the `clusterVersion`, the `platformVersions`, and a `defaultVersion` flag that is true for exactly one version per add-on and cluster version.
+A `kubernetesVersion` that Spinifex does not run returns no add-ons.
+
+`CreateAddon` and `UpdateAddon` accept the AWS form of a version in the catalogue.
+They also accept the bare upstream form, such as `1.40.1`, and Spinifex logs each use at Info.
+This is a Spinifex leniency that AWS does not offer, and it exists so that older scripts keep working.
+Any other value is refused with a message that names `addonVersion` and lists the supported versions, for example `v1.40.1-eksbuild.2`, `1.40` or `v1.40.1`.
+
+In Terraform, pass the AWS form to `aws_eks_addon.addon_version`, either as a literal or from `data.aws_eks_addon_version`.
+The AWS provider validates the format before it calls the API, so it rejects the bare form.
+Leaving `addon_version` unset still installs the catalogue default.
+
 ## Prerequisites
 
 > [!IMPORTANT]
@@ -225,12 +244,11 @@ The list shows the node group workers only. The control-plane VMs never appear a
 
 ### 4. Expose a workload
 
-Workers join an auto-managed security group that only allows intra-cluster traffic. To reach a NodePort service, add one ingress rule to that SG (look it up by its deterministic name):
+Workers join the cluster security group, which only allows intra-cluster traffic. To reach a NodePort service, add one ingress rule to it (`DescribeCluster` reports its id):
 
 ```bash
-SG=$(aws ec2 describe-security-groups \
-  --filters Name=group-name,Values=eks-cluster-demo-nodegroup-sg \
-  --query 'SecurityGroups[0].GroupId' --output text)
+SG=$(aws eks describe-cluster --name demo \
+  --query 'cluster.resourcesVpcConfig.clusterSecurityGroupId' --output text)
 
 aws ec2 authorize-security-group-ingress \
   --group-id "$SG" --protocol tcp --port 30080 --cidr 0.0.0.0/0
@@ -272,7 +290,7 @@ The form notes that the authentication mode is API and that the legacy aws-auth
 
 - The **Access** tab lists IAM principals and their cluster-access policies — add an access entry to let another user or role reach the cluster.
 - The **Addons** tab installs and shows the health of managed addons.
-- The **Networking** tab shows the cluster's VPC, subnets, endpoint access, and the auto-managed security groups.
+- The **Networking** tab shows the cluster's VPC, subnets, endpoint access, and the auto-managed cluster security group.
 
 To run `kubectl` against the cluster, follow the AWS CLI tab's `update-kubeconfig` step.
 
@@ -355,11 +373,11 @@ Control plane readiness and kubectl authentication above lists which clusters re
 - Confirm the workers can pull images — a default route to an Internet Gateway (public) or NAT Gateway (private).
 - Confirm the node role has `AmazonEKSWorkerNodePolicy`, `AmazonEKS_CNI_Policy`, and `AmazonEC2ContainerRegistryReadOnly`.
 
-**Workload unreachable from outside the cluster.** The node-group security group only allows intra-cluster traffic by default. Add an ingress rule for your NodePort to `eks-cluster-<name>-nodegroup-sg`, as shown in the CLI tab.
+**Workload unreachable from outside the cluster.** The cluster security group only allows intra-cluster traffic by default. Add an ingress rule for your NodePort to it, as shown in the CLI tab. An Ingress served by the AWS Load Balancer Controller needs no rule: the controller adds it to the cluster security group.
 
 **`create-cluster` rejected with an authentication-mode error.** `CONFIG_MAP` is the one mode Spinifex rejects, because it has no `aws-auth` ConfigMap path. Set `authentication_mode` to `API` or `API_AND_CONFIG_MAP`, or leave it unset, and grant access with access entries.
 
-**Addon install fails.** Only addons bundled into the registered `eks-node` image install successfully. If you pin an addon version in Terraform, leave `addon_version` unset so Spinifex selects the catalog default.
+**Addon install fails.** Only addons bundled into the registered `eks-node` image install successfully. If you pin an addon version in Terraform, use the AWS form from `DescribeAddonVersions` (see Add-on versions above), or leave `addon_version` unset so Spinifex selects the catalog default.
 
 ## Reference Architectures
 

docs/coverage/README.md

@@ -17,7 +17,7 @@ tags:
 
 ## Overview
 
-The platform serves **405 operations** across the AWS APIs below. Every page names the operations its service implements from the pinned model, generated from the dispatch tables on each build rather than written by hand.
+The platform serves **406 operations** across the AWS APIs below. Every page names the operations its service implements from the pinned model, generated from the dispatch tables on each build rather than written by hand.
 
 | Service | Operations |
 |---|---:|
@@ -27,9 +27,9 @@ The platform serves **405 operations** across the AWS APIs below. Every page nam
 | [ECS](/coverage/ecs) | 31 |
 | [EKS](/coverage/eks) | 27 |
 | [ELBv2](/coverage/elbv2) | 36 |
-| [IAM](/coverage/iam) | 81 |
+| [IAM](/coverage/iam) | 82 |
 | [RDS](/coverage/rds) | 27 |
 | [Route 53](/coverage/route53) | 13 |
 | [S3](/coverage/s3) | 26 |
 | [STS](/coverage/sts) | 5 |
-| **Total** | **405** |
+| **Total** | **406** |

docs/coverage/coverage.json

@@ -2938,7 +2938,8 @@
           "UpdateAccessKey",
           "UpdateAssumeRolePolicy",
           "UpdateRole",
-          "UpdateRoleDescription"
+          "UpdateRoleDescription",
+          "UpdateUser"
         ],
         "implemented": [
           "AddRoleToInstanceProfile",
@@ -3021,7 +3022,8 @@
           "UpdateAccessKey",
           "UpdateAssumeRolePolicy",
           "UpdateRole",
-          "UpdateRoleDescription"
+          "UpdateRoleDescription",
+          "UpdateUser"
         ],
         "stubbed": [],
         "unsupported": [],
@@ -3121,7 +3123,6 @@
           "UpdateServerCertificate",
           "UpdateServiceSpecificCredential",
           "UpdateSigningCertificate",
-          "UpdateUser",
           "UploadSSHPublicKey",
           "UploadServerCertificate",
           "UploadSigningCertificate"

docs/coverage/iam/README.md

@@ -18,7 +18,7 @@ tags:
 
 ## Overview
 
-Spinifex implements **81 operations** in the IAM `2010-05-08` API model.
+Spinifex implements **82 operations** in the IAM `2010-05-08` API model.
 
 ### Scope
 
@@ -109,3 +109,4 @@ All IAM operations are account-scoped. The root user of an account bypasses poli
 | `UpdateAssumeRolePolicy` |
 | `UpdateRole` |
 | `UpdateRoleDescription` |
+| `UpdateUser` |

docs/coverage/internal-report.md

@@ -987,7 +987,7 @@ spot-price: On owned hardware there is no spot-to-on-demand price differential,
 
 ## iam
 
-`2010-05-08` — 81 of 180 modelled operations implemented (45.0%), 0 stubbed, 19 not applicable, 0 registered outside the pinned model.
+`2010-05-08` — 82 of 180 modelled operations implemented (45.6%), 0 stubbed, 19 not applicable, 0 registered outside the pinned model.
 
 ### Not implemented
 
@@ -1069,7 +1069,6 @@ spot-price: On owned hardware there is no spot-to-on-demand price differential,
 - `UpdateOpenIDConnectProviderThumbprint`
 - `UpdateSAMLProvider`
 - `UpdateServerCertificate`
-- `UpdateUser`
 - `UploadServerCertificate`
 
 ### Not applicable — do not build

docs/package-boundary/eks-addon-lifecycle.md

@@ -20,7 +20,7 @@ There is no generation, so the record mixes desired fields with the latest obser
 | Durable record | `clusters/{cluster}/addons/{addon}` holds `Record` (`domains/eks/addon/record.go`): `addonName`, `addonVersion`, `status`, `serviceAccountRoleArn`, `configurationValues`, `health`, `arn`, `tags`, `createdAt`, `modifiedAt` (the optional ones `omitempty`). `clusters/{cluster}/addons/{addon}/manifest` holds `Manifest` (`domains/eks/addon/record.go`): `addonName`, `addonVersion`, `serviceAccountRoleArn`, `configurationValues`. Listing skips the manifest sub-key by segment count. | `TestAddonPersistedLayout_RecordAndManifestKeysAndFieldNames`, `TestAddonRecord_DecodesHandWrittenJSON`, `domains/eks/addon/store_test.go` `TestRecord_RoundTrip`, `TestList_SkipsManifestSubKeys` |
 | Writers | Record: the owner in `domains/eks/addon/owner.go`, reached from the `CreateAddon`, `UpdateAddon` and `DeleteAddon` handlers, the cluster reconciler's report relay and the GPU node group launch: `Owner.Create` (unconditional put), `Owner.Update` and its failure marking (revision CAS), `Owner.Delete` (get then delete), `Owner.ApplyReport` (revision CAS), `Owner.EnsureGPUDevicePlugin` through `Owner.Create`; and the cluster prefix purge (`DeleteClusterChildren`). Manifest: `StagingInstaller.Install` (unconditional put on create and update), `StagingInstaller.Uninstall` (`domains/eks/addon/delivery.go`), `Owner.Delete` and the prefix purge. | `TestAddonPersistedLayout_RecordAndManifestKeysAndFieldNames`, `TestUpdateAddon_ReportsSuccessfulWhileRecordIsUpdating`, `TestDeleteAddon_ReturnsDeletingAfterErasingRecordAndManifest`, `TestDeleteClusterChildren_SweepsAddonRecordsAndManifestsScopedToCluster`, `domains/eks/addon/store_test.go` `TestCasUpdate`, `TestDeleteRecord_RemovesRecordAndManifest` |
 | Readers | `ListAddons`, `DescribeAddon`, `CreateAddon` (existence check) and the reconciler read the record. `ListStagedAddonManifests` reads every `/manifest` key for the cluster (`addon.ListManifests`); it is served on `eks.ListStagedAddonManifests` (`daemon/daemon.go`) and reached only through the gateway's internal-addons route. | `handlers/eks/addons_test.go` `TestAddon_DescribeListDelete`, `TestListStagedAddonManifests` |
-| Catalogue | Static in-binary catalogue (`domains/eks/addon/catalog.go`): `aws-load-balancer-controller`, `argocd`, `aws-ebs-csi-driver`, and hidden-but-creatable `nvidia-device-plugin` and `spinifex-noop`; one version each, the first being the default. Create and update reject unknown names and versions with `InvalidParameterValue`. | `domains/eks/addon/catalog_test.go`, `handlers/eks/addons_test.go` `TestDescribeAddonVersions_ReturnsCatalog`, `TestCreateAddon_UnknownAddonRejected`, `TestCreateAddon_UnknownVersionRejected`, `TestUpdateAddon_UnknownVersionRejected` |
+| Catalogue | Static in-binary catalogue (`domains/eks/addon/catalog.go`): `aws-load-balancer-controller`, `argocd`, `aws-ebs-csi-driver`, and hidden-but-creatable `nvidia-device-plugin` and `spinifex-noop`; one version each, the first being the default. Versions are stored and used internally in the bare upstream form, and API responses show the AWS form `v<x.y.z>-eksbuild.<n>` (`Spec.AWSVersion`). Create and update accept the AWS form or the exact bare form and reject unknown names and other version strings with `InvalidParameterValue`, which the gateway returns as `InvalidParameterException`. | `domains/eks/addon/catalog_test.go`, `handlers/eks/addons_test.go` `TestDescribeAddonVersions_ReturnsCatalog`, `TestCreateAddon_UnknownAddonRejected`, `TestCreateAddon_UnknownVersionRejected`, `TestUpdateAddon_UnknownVersionRejected` |
 | Desired state | Version, `serviceAccountRoleArn`, `configurationValues` and create-time tags. `resolveConflicts`, `podIdentityAssociations` and `clientRequestToken` are accepted and dropped; `preserve` on delete is ignored. `configurationValues` reaches the guest but the agent discards it (`mulga-eks-addon-sync.sh` `sync_once`), so it has no effect. | `TestAddonPersistedLayout_RecordAndManifestKeysAndFieldNames`, `TestDeleteAddon_ReturnsDeletingAfterErasingRecordAndManifest` |
 | Observed state | `status` and `health` are overwritten by guest reports on the same record as the desired fields. `health` is the latest failure message, projected as one `AddonIssue` with only `message`. Statuses held: `CREATING`, `ACTIVE`, `UPDATING`, `DEGRADED`, `CREATE_FAILED`; `DELETING` is defined but never stored. | `TestAddonRecord_DecodesHandWrittenJSON`, `handlers/eks/addon_status_reconciler_test.go` `TestApplyAddonStatusReport` |
 | Generation | None. Reports carry `version` and `ts`, and neither is read: any report for the add-on name applies to whatever is currently desired. | `TestAddonStatusReport_MismatchedVersionIsAccepted` |

docs/terraform-workbooks/eks-gitops-argocd/README.md

@@ -79,7 +79,8 @@ Terraform manages the cluster, the addons, the ACM cert, and the HTTPS Ingress;
 - The `argocd` and `aws-ebs-csi-driver` bundles must be baked into the `eks-node` AMI — `describe-addon` returns `no baked bundle` otherwise.
 - The EBS-CSI default StorageClass uses `provisioner: ebs.csi.aws.com`; a PVC against it provisions a **Viperblock-backed EBS volume**.
 - If the PVC stays `Pending`, the app still serves (the counter falls back to in-memory) — see [Troubleshooting](#troubleshooting).
-- Leave addon versions unset (the AWS provider and the catalog disagree on the version-string format).
+- Addon versions are unset, so Spinifex installs each catalog default.
+  To pin one, use the AWS form that `DescribeAddonVersions` returns, such as `v3.0.23-eksbuild.1`.
 - The Argo CD `Application` is a `kubernetes_manifest`, so the `argoproj.io` CRDs must exist before `apply` — apply the parent and let the addon reach `ACTIVE` first.
 
 **Prerequisites:**

docs/terraform-workbooks/eks-gitops-argocd/main.tf

@@ -471,8 +471,7 @@ resource "aws_eks_cluster" "this" {
   # expects: access_config reads as absent (and its ForceNew
   # bootstrap_cluster_creator_admin_permissions would replace the live cluster on
   # re-add), and vpc_config.security_group_ids reads back as drift. Ignore both so
-  # later applies (e.g. adding the Argo CD NodePort rule) neither replace the
-  # cluster nor detach its security groups.
+  # later applies neither replace the cluster nor detach its security groups.
   lifecycle {
     ignore_changes = [access_config, vpc_config[0].security_group_ids]
   }
@@ -520,8 +519,8 @@ resource "aws_eks_node_group" "workers" {
 #
 # Spinifex wires the controller's AWS credentials and ELB-eligible subnets at the
 # node level, so no IRSA role or subnet tagging is needed here. addon_version is
-# omitted: the AWS provider demands a v-prefixed version that the catalog rejects,
-# so let Spinifex default to its catalog version.
+# omitted so Spinifex installs its catalog default. To pin one, use the AWS form
+# from DescribeAddonVersions, for example v2.11.0-eksbuild.1.
 # ---------------------------------------------------------------------------
 
 resource "aws_eks_addon" "lbc" {
@@ -542,7 +541,7 @@ resource "aws_eks_addon" "lbc" {
 # Installed through the EKS API; Spinifex stages the bundle host-side and the
 # worker renders it into the K3s auto-deploy dir. The workloads module registers
 # the git repo and creates the Argo CD Application that syncs the demo app.
-# addon_version omitted (see the LBC note).
+# addon_version omitted: the catalog default is installed (see the LBC note).
 # ---------------------------------------------------------------------------
 
 resource "aws_eks_addon" "argocd" {
@@ -562,7 +561,7 @@ resource "aws_eks_addon" "argocd" {
 #
 # Ships a default gp3 StorageClass (provisioner ebs.csi.aws.com). A PVC in the
 # demo app's git manifests dynamically provisions a Viperblock-backed EBS volume
-# so the app's state survives pod restarts. addon_version omitted (see LBC note).
+# so the app's state survives pod restarts. addon_version omitted: catalog default.
 # ---------------------------------------------------------------------------
 
 resource "aws_eks_addon" "ebs_csi" {
@@ -642,53 +641,6 @@ resource "aws_acm_certificate" "ingress" {
   }
 }
 
-# ---------------------------------------------------------------------------
-# Let the ALB reach the workers' NodePort
-#
-# Spinifex auto-manages the nodegroup SG and admits only intra-cluster traffic.
-# The LBC-provisioned ALB lives in this VPC, so admit the VPC CIDR on the
-# NodePort. Look the SG up by its deterministic name and add one rule.
-# ---------------------------------------------------------------------------
-
-data "aws_security_group" "nodegroup" {
-  filter {
-    name   = "group-name"
-    values = ["eks-cluster-${var.cluster_name}-nodegroup-sg"]
-  }
-
-  filter {
-    name   = "vpc-id"
-    values = [aws_vpc.main.id]
-  }
-
-  depends_on = [aws_eks_node_group.workers]
-}
-
-resource "aws_vpc_security_group_ingress_rule" "nodeport_from_vpc" {
-  security_group_id = data.aws_security_group.nodegroup.id
-  cidr_ipv4         = aws_vpc.main.cidr_block
-  from_port         = var.node_port
-  to_port           = var.node_port
-  ip_protocol       = "tcp"
-
-  tags = {
-    Name = "${var.cluster_name}-alb-nodeport"
-  }
-}
-
-# Same, for the Argo CD UI NodePort its ALB forwards to.
-resource "aws_vpc_security_group_ingress_rule" "argocd_nodeport_from_vpc" {
-  security_group_id = data.aws_security_group.nodegroup.id
-  cidr_ipv4         = aws_vpc.main.cidr_block
-  from_port         = var.argocd_node_port
-  to_port           = var.argocd_node_port
-  ip_protocol       = "tcp"
-
-  tags = {
-    Name = "${var.cluster_name}-argocd-nodeport"
-  }
-}
-
 # ---------------------------------------------------------------------------
 # Outputs
 # ---------------------------------------------------------------------------

docs/terraform-workbooks/eks-https-ingress/README.md

@@ -64,7 +64,6 @@ The LBC is installed as a managed **addon** (`aws_eks_addon`). Spinifex wires th
 | Node Group | `workers` | `node_desired_size` `t3.large` worker(s) — 1 or 3 |
 | Addon | `aws-load-balancer-controller` | Provisions the ALB from the Ingress |
 | ACM Certificate | `eks-https-ingress` | Self-signed, imported; attached to the HTTPS listener |
-| SG Ingress Rule | `eks-https-alb-nodeport` | Admits the VPC CIDR to the workers' NodePort |
 | K8s Deployment + Service | `spinifex-demo` | Themed demo image, 2 replicas, NodePort |
 | K8s Ingress | `spinifex-demo` | `ingressClassName: alb`, HTTPS via the ACM cert |
 
@@ -72,9 +71,10 @@ The LBC is installed as a managed **addon** (`aws_eks_addon`). Spinifex wires th
 
 - **The LBC needs no IRSA or subnet tagging here.** Credentials and ELB-eligible subnets are injected by Spinifex at the node level and into the `alb` `IngressClassParams`. The Ingress just sets `ingressClassName: alb` plus the `alb.ingress.kubernetes.io/*` annotations.
 - **`spinifex.io/managed-ingress = "false"`** disables the K3s built-in traefik/servicelb so the LBC owns ingress, matching AWS parity. Omit the tag (or set it `true`) and the built-in ingress stays on.
-- The ALB targets the workers' **NodePort** (`target-type: instance`). The one SG rule admits the VPC CIDR on that port so the in-VPC ALB can reach the workers; the worker SG is otherwise auto-managed.
+- The ALB targets the workers' **NodePort** (`target-type: instance`). The AWS Load Balancer Controller adds the ALB-to-NodePort rule to the cluster security group itself, so the workbook creates no security group rule.
 - Spinifex ACM supports **ImportCertificate only** — the cert is generated by the `tls` provider and imported, not requested.
-- **Leave `addon_version` unset** (see the [quickstart/addons notes](/docs/eks-quickstart)) — the AWS provider and the Spinifex catalog disagree on the version string format, so let the server choose.
+- **`addon_version` is unset**, so Spinifex installs its catalog default.
+  To pin a version, use the AWS form that `DescribeAddonVersions` returns, such as `v2.11.0-eksbuild.1`, or read it from `data.aws_eks_addon_version`.
 
 **Prerequisites:**
 
@@ -179,16 +179,17 @@ Confirm the addon is `ACTIVE` and the cluster carries `spinifex.io/managed-ingre
 
 ### HTTPS Loads but Returns 502 / 504
 
-The ALB reached the workers but the backend didn't answer. Check the demo pods and the NodePort SG rule:
+The ALB reached the workers but the backend didn't answer. Check the demo pods and the rule the controller added to the cluster security group:
 
 ```bash
 kubectl get pods -o wide
 kubectl get svc spinifex-demo
-aws ec2 describe-security-groups --filters "Name=group-name,Values=eks-cluster-eks-https-nodegroup-sg" \
+SG=$(aws eks describe-cluster --name eks-https --query 'cluster.resourcesVpcConfig.clusterSecurityGroupId' --output text)
+aws ec2 describe-security-groups --group-ids "$SG" \
   --query 'SecurityGroups[0].IpPermissions'
 ```
 
-The `eks-https-alb-nodeport` rule must admit the VPC CIDR on the NodePort, and the target group health check (`/healthz`) must pass.
+The cluster security group must admit the load balancer's security group on the NodePort, and the target group health check (`/healthz`) must pass.
 
 ### Addon Fails With "no baked bundle"
 

docs/terraform-workbooks/eks-https-ingress/main.tf

@@ -16,7 +16,8 @@
 # The workloads/ module then creates a Deployment, a NodePort Service, and an
 # Ingress (ingressClassName: alb). The LBC reconciles that Ingress into an
 # internet-facing ALB that terminates TLS with the ACM cert and forwards to the
-# workers' NodePort. The cluster's ELB-eligible subnets are injected by Spinifex
+# workers' NodePort, adding the ALB-to-NodePort rule to the cluster security group
+# itself. The cluster's ELB-eligible subnets are injected by Spinifex
 # into the alb IngressClassParams, so the Ingress needs no subnet annotations.
 #
 # Usage:
@@ -446,8 +447,8 @@ resource "aws_eks_node_group" "workers" {
 #
 # Spinifex wires the controller's AWS credentials and ELB-eligible subnets at the
 # node level, so no IRSA role or subnet tagging is needed here. addon_version is
-# omitted: the AWS provider demands a v-prefixed version that the catalog rejects,
-# so let Spinifex default to its catalog version.
+# omitted so Spinifex installs its catalog default. To pin one, use the AWS form
+# from DescribeAddonVersions, for example v2.11.0-eksbuild.1.
 # ---------------------------------------------------------------------------
 
 resource "aws_eks_addon" "lbc" {
@@ -503,40 +504,6 @@ resource "aws_acm_certificate" "ingress" {
   }
 }
 
-# ---------------------------------------------------------------------------
-# Let the ALB reach the workers' NodePort
-#
-# Spinifex auto-manages the nodegroup SG and admits only intra-cluster traffic.
-# The LBC-provisioned ALB lives in this VPC, so admit the VPC CIDR on the
-# NodePort. Look the SG up by its deterministic name and add one rule.
-# ---------------------------------------------------------------------------
-
-data "aws_security_group" "nodegroup" {
-  filter {
-    name   = "group-name"
-    values = ["eks-cluster-${var.cluster_name}-nodegroup-sg"]
-  }
-
-  filter {
-    name   = "vpc-id"
-    values = [aws_vpc.main.id]
-  }
-
-  depends_on = [aws_eks_node_group.workers]
-}
-
-resource "aws_vpc_security_group_ingress_rule" "nodeport_from_vpc" {
-  security_group_id = data.aws_security_group.nodegroup.id
-  cidr_ipv4         = aws_vpc.main.cidr_block
-  from_port         = var.node_port
-  to_port           = var.node_port
-  ip_protocol       = "tcp"
-
-  tags = {
-    Name = "${var.cluster_name}-alb-nodeport"
-  }
-}
-
 # ---------------------------------------------------------------------------
 # Outputs
 # ---------------------------------------------------------------------------

docs/terraform-workbooks/eks-quickstart/README.md

@@ -45,7 +45,7 @@ Under the hood it keeps things minimal: a VPC with two public subnets, an IAM ro
 - Creating the EKS cluster and worker IAM roles with faithful managed-policy attachments
 - Provisioning an `aws_eks_cluster` and a managed `aws_eks_node_group`
 - Pointing the Kubernetes provider at the new cluster and deploying a workload from the same `apply`
-- Opening one rule on the auto-managed worker security group to expose a NodePort
+- Opening one rule on the cluster security group to expose a NodePort
 
 **What gets created**
 
@@ -58,13 +58,13 @@ Under the hood it keeps things minimal: a VPC with two public subnets, an IAM ro
 | ECR Repository | `spinifex-demo` | Holds the demo image the workers pull |
 | EKS Cluster | `eks-quickstart` | Public API endpoint, API auth mode, Kubernetes 1.32 |
 | Node Group | `default` | `node_desired_size` `t3.medium` worker(s) — 1 or 3 |
-| SG Ingress Rule | `eks-quickstart-demo-nodeport` | Opens the demo NodePort on the auto-managed worker SG |
+| SG Ingress Rule | `eks-quickstart-demo-nodeport` | Opens the demo NodePort on the cluster security group |
 | K8s Deployment | `spinifex-demo` | The themed demo image from ECR, 2 replicas |
 | K8s Service | `spinifex-demo` | NodePort publishing the demo on the worker |
 
 **Spinifex specifics**
 
-- The cluster's security groups are **auto-managed** — `vpc_config.security_group_ids` is ignored. To reach a NodePort, this workbook looks the worker SG up by its deterministic name (`eks-cluster-<name>-nodegroup-sg`) and adds a single ingress rule.
+- The cluster security group is **auto-managed**, as on EKS: Spinifex creates `eks-cluster-sg-<name>-<id>`, attaches it to the workers, and reports it as `vpc_config[0].cluster_security_group_id`. `vpc_config.security_group_ids` is ignored. To reach a NodePort, this workbook adds a single ingress rule to that security group.
 - The worker AMI is always Spinifex's `eks-node` image; `ami_type` is recorded but does **not** select the image.
 - `authentication_mode` can be `"API"` or `"API_AND_CONFIG_MAP"`, and bare `"CONFIG_MAP"` is rejected. Access is granted through access entries, as there is no `aws-auth` ConfigMap path.
 - The workers need outbound internet (here via the IGW) to pull the demo container image.
@@ -182,8 +182,9 @@ aws ec2 describe-instances --filters "Name=tag:spinifex:eks-cluster,Values=eks-q
 # Did the demo pods roll out?
 kubectl get pods -o wide
 
-# Is the NodePort rule on the worker SG?
-aws ec2 describe-security-groups --filters "Name=group-name,Values=eks-cluster-eks-quickstart-nodegroup-sg" \
+# Is the NodePort rule on the cluster security group?
+SG=$(tofu output -raw cluster_security_group_id)
+aws ec2 describe-security-groups --group-ids "$SG" \
   --query 'SecurityGroups[0].IpPermissions'
 ```
 

docs/terraform-workbooks/eks-quickstart/main.tf

@@ -310,29 +310,15 @@ resource "aws_eks_node_group" "default" {
 }
 
 # ---------------------------------------------------------------------------
-# Open the NodePort on the auto-managed nodegroup SG
+# Open the NodePort on the cluster security group
 #
-# Spinifex creates the worker SG itself (vpc_config.security_group_ids is
-# ignored) and admits only intra-cluster traffic. To reach the demo NodePort
-# from a browser, look the SG up by its deterministic name and add one rule.
+# Spinifex creates the cluster security group itself, as EKS does, and attaches
+# it to the workers. It admits only intra-cluster traffic. To reach the demo
+# NodePort from a browser, add one rule to the SG the cluster reports.
 # ---------------------------------------------------------------------------
 
-data "aws_security_group" "nodegroup" {
-  filter {
-    name   = "group-name"
-    values = ["eks-cluster-${var.cluster_name}-nodegroup-sg"]
-  }
-
-  filter {
-    name   = "vpc-id"
-    values = [aws_vpc.main.id]
-  }
-
-  depends_on = [aws_eks_node_group.default]
-}
-
 resource "aws_vpc_security_group_ingress_rule" "nodeport" {
-  security_group_id = data.aws_security_group.nodegroup.id
+  security_group_id = aws_eks_cluster.this.vpc_config[0].cluster_security_group_id
   cidr_ipv4         = var.browse_cidr
   from_port         = var.node_port
   to_port           = var.node_port
@@ -367,6 +353,10 @@ output "region" {
   value = var.region
 }
 
+output "cluster_security_group_id" {
+  value = aws_eks_cluster.this.vpc_config[0].cluster_security_group_id
+}
+
 output "node_port" {
   value = var.node_port
 }

spinifex/agents/ecs/agent/credendpoint.go

@@ -47,6 +47,13 @@ type credResponse struct {
 
 // credEntry caches one task's assumed-role credentials alongside the role it was
 // assumed from.
+// credRole is the task role a credID serves and the task it belongs to; the
+// task ID names the STS session, since the credID is a secret.
+type credRole struct {
+	roleARN string
+	taskID  string
+}
+
 type credEntry struct {
 	roleARN string
 	creds   stsauth.Credentials
@@ -71,7 +78,7 @@ type credEndpoint struct {
 	assume func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error)
 
 	mu         sync.Mutex
-	roles      map[string]string    // credID -> taskRoleARN
+	roles      map[string]credRole  // credID -> task role
 	cache      map[string]credEntry // credID -> cached creds
 	httpClient *http.Client
 
@@ -98,7 +105,7 @@ func newCredEndpoint(provider credentials.CredentialsProvider, region, gatewayUR
 		port:       port,
 		proxyPort:  defaultCredProxyPort,
 		run:        run,
-		roles:      map[string]string{},
+		roles:      map[string]credRole{},
 		cache:      map[string]credEntry{},
 	}
 	c.assume = c.assumeOverGateway
@@ -107,12 +114,12 @@ func newCredEndpoint(provider credentials.CredentialsProvider, region, gatewayUR
 
 // Register associates a credID with the task role it serves; Deregister removes
 // it and drops any cached credentials. Both are no-ops for an empty credID.
-func (c *credEndpoint) Register(credID, roleARN string) {
+func (c *credEndpoint) Register(credID, taskID, roleARN string) {
 	if credID == "" || roleARN == "" {
 		return
 	}
 	c.mu.Lock()
-	c.roles[credID] = roleARN
+	c.roles[credID] = credRole{roleARN: roleARN, taskID: taskID}
 	c.mu.Unlock()
 }
 
@@ -142,25 +149,25 @@ func (c *credEndpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, "not found", http.StatusNotFound)
 		return
 	}
-	creds, roleARN, err := c.fetch(r.Context(), credID)
+	creds, role, err := c.fetch(r.Context(), credID)
 	if err != nil {
 		if errors.Is(err, errUnknownCredID) {
 			http.Error(w, "not found", http.StatusNotFound)
 			return
 		}
-		slog.Error("ecs-agent: assume task role failed", "credId", credID, "err", err)
+		slog.Error("ecs-agent: assume task role failed", "task", role.taskID, "role", role.roleARN, "err", err)
 		http.Error(w, "credentials unavailable", http.StatusBadGateway)
 		return
 	}
 	w.Header().Set("Content-Type", "application/json")
 	if err := json.NewEncoder(w).Encode(credResponse{
-		RoleArn:         roleARN,
+		RoleArn:         role.roleARN,
 		AccessKeyID:     creds.AccessKeyID,
 		SecretAccessKey: creds.SecretAccessKey,
 		Token:           creds.SessionToken,
 		Expiration:      creds.Expiration.UTC().Format(time.RFC3339),
 	}); err != nil {
-		slog.Warn("ecs-agent: encode credential response", "credId", credID, "err", err)
+		slog.Warn("ecs-agent: encode credential response", "task", role.taskID, "err", err)
 	}
 }
 
@@ -169,28 +176,28 @@ var errUnknownCredID = fmt.Errorf("unknown credId")
 
 // fetch returns valid credentials for credID, reusing the cache until within the
 // refresh margin and otherwise assuming the task role over the gateway.
-func (c *credEndpoint) fetch(ctx context.Context, credID string) (stsauth.Credentials, string, error) {
+func (c *credEndpoint) fetch(ctx context.Context, credID string) (stsauth.Credentials, credRole, error) {
 	c.mu.Lock()
-	roleARN, known := c.roles[credID]
+	role, known := c.roles[credID]
 	if !known {
 		c.mu.Unlock()
-		return stsauth.Credentials{}, "", errUnknownCredID
+		return stsauth.Credentials{}, credRole{}, errUnknownCredID
 	}
-	if e, ok := c.cache[credID]; ok && e.roleARN == roleARN && credValid(e.creds, credRefreshMargin) {
+	if e, ok := c.cache[credID]; ok && e.roleARN == role.roleARN && credValid(e.creds, credRefreshMargin) {
 		c.mu.Unlock()
-		return e.creds, roleARN, nil
+		return e.creds, role, nil
 	}
 	c.mu.Unlock()
 
-	creds, err := c.assume(ctx, roleARN, sessionName(credID))
+	creds, err := c.assume(ctx, role.roleARN, sessionName(role.taskID))
 	if err != nil {
-		return stsauth.Credentials{}, "", err
+		return stsauth.Credentials{}, role, err
 	}
 
 	c.mu.Lock()
-	c.cache[credID] = credEntry{roleARN: roleARN, creds: creds}
+	c.cache[credID] = credEntry{roleARN: role.roleARN, creds: creds}
 	c.mu.Unlock()
-	return creds, roleARN, nil
+	return creds, role, nil
 }
 
 // AssumeProvider returns a CredentialsProvider that mints credentials by assuming
@@ -390,10 +397,10 @@ func credValid(c stsauth.Credentials, margin time.Duration) bool {
 	return time.Now().Add(margin).Before(c.Expiration)
 }
 
-// sessionName derives an AssumeRole RoleSessionName from credID, satisfying the
+// sessionName derives an AssumeRole RoleSessionName from the task ID, satisfying the
 // STS [\w+=,.@-]{2,64} constraint.
-func sessionName(credID string) string {
-	name := "ecs-" + credID
+func sessionName(taskID string) string {
+	name := "ecs-" + taskID
 	if len(name) > 64 {
 		name = name[:64]
 	}

spinifex/agents/ecs/agent/credendpoint_test.go

@@ -29,7 +29,7 @@ func TestCredEndpoint_ServesRegisteredCredID(t *testing.T) {
 	c, _ := newTestCredEndpoint(stsauth.Credentials{
 		AccessKeyID: "AKIA", SecretAccessKey: "secret", SessionToken: "tok", Expiration: exp,
 	})
-	c.Register("cred-1", "arn:aws:iam::111122223333:role/task")
+	c.Register("cred-1", "task-1", "arn:aws:iam::111122223333:role/task")
 
 	rr := httptest.NewRecorder()
 	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
@@ -51,6 +51,26 @@ func TestCredEndpoint_ServesRegisteredCredID(t *testing.T) {
 	}
 }
 
+// The STS session name lands in the assumed-role ARN and in STS logs, so it is
+// built from the public task ID and never from the secret credID.
+func TestCredEndpoint_SessionNameIsTaskIDNotCredID(t *testing.T) {
+	c, _ := newTestCredEndpoint(stsauth.Credentials{})
+	var gotSession string
+	c.assume = func(ctx context.Context, roleARN, session string) (stsauth.Credentials, error) {
+		gotSession = session
+		return stsauth.Credentials{AccessKeyID: "AKIA", SecretAccessKey: "secret"}, nil
+	}
+	c.Register("secret-cred", "task-1", "arn:role")
+	rr := httptest.NewRecorder()
+	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/secret-cred", nil))
+	if rr.Code != http.StatusOK {
+		t.Fatalf("status = %d, want 200", rr.Code)
+	}
+	if gotSession != "ecs-task-1" {
+		t.Errorf("session name = %q, want ecs-task-1", gotSession)
+	}
+}
+
 func TestCredEndpoint_UnknownCredIDIs404(t *testing.T) {
 	c, calls := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
 	rr := httptest.NewRecorder()
@@ -65,7 +85,7 @@ func TestCredEndpoint_UnknownCredIDIs404(t *testing.T) {
 
 func TestCredEndpoint_WrongMethodIs405(t *testing.T) {
 	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
-	c.Register("cred-1", "arn:role")
+	c.Register("cred-1", "task-1", "arn:role")
 	rr := httptest.NewRecorder()
 	c.ServeHTTP(rr, httptest.NewRequest(http.MethodPost, "/v2/credentials/cred-1", nil))
 	if rr.Code != http.StatusMethodNotAllowed {
@@ -77,7 +97,7 @@ func TestCredEndpoint_CachesUntilExpiry(t *testing.T) {
 	c, calls := newTestCredEndpoint(stsauth.Credentials{
 		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Hour),
 	})
-	c.Register("cred-1", "arn:role")
+	c.Register("cred-1", "task-1", "arn:role")
 	for hit := range 3 {
 		rr := httptest.NewRecorder()
 		c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
@@ -95,7 +115,7 @@ func TestCredEndpoint_RefreshesNearExpiry(t *testing.T) {
 	c, calls := newTestCredEndpoint(stsauth.Credentials{
 		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Minute),
 	})
-	c.Register("cred-1", "arn:role")
+	c.Register("cred-1", "task-1", "arn:role")
 	for range 2 {
 		rr := httptest.NewRecorder()
 		c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
@@ -107,7 +127,7 @@ func TestCredEndpoint_RefreshesNearExpiry(t *testing.T) {
 
 func TestCredEndpoint_DeregisterDropsCredID(t *testing.T) {
 	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "x", SecretAccessKey: "y"})
-	c.Register("cred-1", "arn:role")
+	c.Register("cred-1", "task-1", "arn:role")
 	c.Deregister("cred-1")
 	rr := httptest.NewRecorder()
 	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
@@ -121,7 +141,7 @@ func TestCredEndpoint_AssumeFailureIs502(t *testing.T) {
 	c.assume = func(ctx context.Context, roleARN, sessionName string) (stsauth.Credentials, error) {
 		return stsauth.Credentials{}, context.DeadlineExceeded
 	}
-	c.Register("cred-1", "arn:role")
+	c.Register("cred-1", "task-1", "arn:role")
 	rr := httptest.NewRecorder()
 	c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/v2/credentials/cred-1", nil))
 	if rr.Code != http.StatusBadGateway {
@@ -133,7 +153,7 @@ func TestCredEndpoint_StartServesOverLoopback(t *testing.T) {
 	c, _ := newTestCredEndpoint(stsauth.Credentials{
 		AccessKeyID: "AKIA", SecretAccessKey: "secret", Expiration: time.Now().Add(time.Hour),
 	})
-	c.Register("cred-1", "arn:role")
+	c.Register("cred-1", "task-1", "arn:role")
 	c.port = 0 // bind an ephemeral port instead of privileged 80
 	if err := c.Start(); err != nil {
 		t.Fatalf("start: %v", err)

spinifex/agents/ecs/agent/credinject_test.go

@@ -1,8 +1,12 @@
 package agent
 
 import (
+	"net/http"
+	"net/http/httptest"
 	"testing"
 
+	"github.com/mulgadc/spinifex/internal/stsauth"
+
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
 )
 
@@ -13,7 +17,7 @@ func TestTaskCredID(t *testing.T) {
 		want string
 	}{
 		{"no role", ecsv1.Assign{TaskID: "t1"}, ""},
-		{"role, no credID -> taskID", ecsv1.Assign{TaskID: "t1", TaskRoleARN: "arn:role"}, "t1"},
+		{"role, no credID -> none", ecsv1.Assign{TaskID: "t1", TaskRoleARN: "arn:role"}, ""},
 		{"role + credID", ecsv1.Assign{TaskID: "t1", CredID: "c9", TaskRoleARN: "arn:role"}, "c9"},
 	}
 	for _, tc := range cases {
@@ -23,6 +27,37 @@ func TestTaskCredID(t *testing.T) {
 	}
 }
 
+// The task ID is public (it is the task ARN suffix), so naming another task's ID
+// at the credential endpoint must not yield that task's role.
+func TestRegisterTaskCreds_TaskIDDoesNotServeCredentials(t *testing.T) {
+	c, _ := newTestCredEndpoint(stsauth.Credentials{AccessKeyID: "AKIA", SecretAccessKey: "secret"})
+	a := Agent{cred: c}
+	get := func(id string) int {
+		rr := httptest.NewRecorder()
+		c.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, credRelativeURI(id), nil))
+		return rr.Code
+	}
+
+	legacy := ecsv1.Assign{TaskID: "victim-task", TaskRoleARN: "arn:aws:iam::000000000010:role/high"}
+	if got := a.registerTaskCreds(&legacy); got != "" {
+		t.Errorf("assign without CredID registered credID %q", got)
+	}
+	if code := get("victim-task"); code != http.StatusNotFound {
+		t.Errorf("GET by task ID of CredID-less assign = %d, want 404", code)
+	}
+
+	minted := ecsv1.Assign{TaskID: "victim-2", CredID: "secret-cred", TaskRoleARN: "arn:aws:iam::000000000010:role/high"}
+	if got := a.registerTaskCreds(&minted); got != "secret-cred" {
+		t.Fatalf("registerTaskCreds = %q, want secret-cred", got)
+	}
+	if code := get("victim-2"); code != http.StatusNotFound {
+		t.Errorf("GET by task ID = %d, want 404", code)
+	}
+	if code := get("secret-cred"); code != http.StatusOK {
+		t.Errorf("GET by CredID = %d, want 200", code)
+	}
+}
+
 func TestWithCredEnv(t *testing.T) {
 	base := map[string]string{"FOO": "bar"}
 

spinifex/agents/ecs/agent/reconcile_test.go

@@ -63,7 +63,7 @@ func TestReconcile_AdoptsRunningLabeledContainer(t *testing.T) {
 	if !adopted["t-001"] {
 		t.Fatalf("t-001 not adopted; got %v", adopted)
 	}
-	if got := a.cred.roles["t-001"]; got != testTaskRole {
+	if got := a.cred.roles["t-001"].roleARN; got != testTaskRole {
 		t.Errorf("cred not re-registered: roles[t-001] = %q, want %q", got, testTaskRole)
 	}
 	if n := countStatus(cp.taskStates(), "t-001", ecsv1.TaskStatusRunning); n != 1 {

spinifex/agents/ecs/agent/taskrun.go

@@ -306,24 +306,24 @@ func (a *Agent) pullResolver(as *ecsv1.Assign) ctrruntime.Resolver {
 // credential endpoint and returns the credID (empty when the task has no role).
 func (a *Agent) registerTaskCreds(as *ecsv1.Assign) string {
 	credID := taskCredID(as)
+	if credID == "" && as.TaskRoleARN != "" {
+		slog.Warn("ecs-agent: task has a role but no credential ID, starting it without credentials", "task", as.TaskID, "role", as.TaskRoleARN)
+	}
 	if credID == "" || a.cred == nil {
 		return credID
 	}
-	a.cred.Register(credID, as.TaskRoleARN)
+	a.cred.Register(credID, as.TaskID, as.TaskRoleARN)
 	return credID
 }
 
 // taskCredID is the credential ID a task's containers fetch credentials under:
-// the scheduler-assigned CredID, falling back to the taskID. Empty when the task
-// carries no IAM role.
+// the scheduler-minted secret CredID. Empty when the task carries no IAM role or
+// no CredID; the public task ID is never used, since any API caller can learn it.
 func taskCredID(as *ecsv1.Assign) string {
 	if as.TaskRoleARN == "" {
 		return ""
 	}
-	if as.CredID != "" {
-		return as.CredID
-	}
-	return as.TaskID
+	return as.CredID
 }
 
 // withCredEnv returns env with AWS_CONTAINER_CREDENTIALS_RELATIVE_URI set for

spinifex/agents/rds/agent/agent_test.go

@@ -763,10 +763,11 @@ func TestPollDBCommandsOutput_MatchesGatewayShape(t *testing.T) {
 	// either side's tags fails here.
 	payload := awsxml.QueryResponsePayload("PollDBCommands", &rdsawsapi.PollDBCommandsOutput{
 		Commands: []rdsinstance.Command{{
-			CommandID:  "cmd-1",
-			Type:       "reload-parameters",
-			Parameters: []rdsinstance.Parameter{{Name: "work_mem", Value: "8MB"}},
-			IssuedAt:   &issued,
+			CommandID:         "cmd-1",
+			Type:              "reload-parameters",
+			Parameters:        []rdsinstance.Parameter{{Name: "work_mem", Value: "8MB"}},
+			IssuedAt:          &issued,
+			RemainingBudgetMs: 29500,
 		}},
 	})
 	body, err := awsxml.Marshal(payload)
@@ -802,6 +803,9 @@ func TestPollDBCommandsOutput_MatchesGatewayShape(t *testing.T) {
 	if cmd.IssuedAt == nil || !cmd.IssuedAt.Equal(issued) {
 		t.Errorf("IssuedAt = %v, want %v", cmd.IssuedAt, issued)
 	}
+	if cmd.RemainingBudgetMs != 29500 {
+		t.Errorf("RemainingBudgetMs = %d, want 29500: a copy that loses it is granted a fresh budget", cmd.RemainingBudgetMs)
+	}
 }
 
 func readFile(t *testing.T, path string) string {

spinifex/agents/rds/agent/command.go

@@ -2,10 +2,13 @@ package agent
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"log/slog"
+	"slices"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	rdsinstance "github.com/mulgadc/spinifex/spinifex/domains/rds/instance"
@@ -90,35 +93,96 @@ func commandParams(cmd rdsinstance.Command) map[string]string {
 // line rate for as long as it stays broken.
 const pollErrorBackoff = 5 * time.Second
 
+// How many finished command IDs are remembered, so a copy the issuer re-published
+// while the command ran is not executed a second time.
+const completedCommandMemory = 64
+
+// How many engine commands may wait behind the one running. Past this a command
+// is refused at once rather than queued behind work its issuer will not outlive.
+const serialQueueDepth = 16
+
+// The budget for a type the control plane does not issue. Such a command is
+// answered as unsupported before it could use any of it.
+const fallbackCommandBudget = 30 * time.Second
+
+// The only command that runs beside the serial lane. It releases the backup hold
+// an engine command can be blocked on, so it must never queue behind one.
+func outOfBand(commandType string) bool {
+	return commandType == rdsinstance.CommandUnquiesce
+}
+
+// A command waiting its turn on the serial lane, with the moment its issuer
+// stops waiting for it.
+type queuedCommand struct {
+	cmd      rdsinstance.Command
+	deadline time.Time
+}
+
 // The agent's command channel: a long poll that carries back the replies for
-// whatever the previous poll delivered.
+// whatever earlier polls delivered. Engine commands run one at a time on a single
+// lane; the release of a backup hold runs beside it, so it cannot be kept out.
 type commander struct {
 	cp       controlPlane
 	id       identity
 	registry commandRegistry
 	wait     time.Duration
+	// The control plane's budget per type, capped by the remaining budget the
+	// issuer stamps on each copy and measured from receipt. IssuedAt is not
+	// compared: the guest clock is not trusted against the host's.
+	budget func(commandType string) time.Duration
+	serial chan queuedCommand
+
+	mu sync.Mutex
 	// Cleared only once a poll has carried them, so a failed poll re-delivers
 	// rather than dropping the result of work the guest actually did.
 	pending []rdsinstance.CommandReply
+	// Queued or running, so a re-published copy is not queued a second time.
+	inFlight map[string]struct{}
+	// Oldest first, bounded by completedCommandMemory.
+	completed []string
+	// Ends the poll in progress, so a reply that finished during it rides the
+	// next one rather than waiting out the window. Nil between polls.
+	interruptPoll func()
+	interrupted   bool
+	running       sync.WaitGroup
 }
 
 func newCommander(cp controlPlane, registry commandRegistry, wait time.Duration) *commander {
 	if wait <= 0 {
 		wait = defaultPollWait
 	}
-	return &commander{cp: cp, registry: registry, wait: wait}
+	return &commander{
+		cp:       cp,
+		registry: registry,
+		wait:     wait,
+		budget:   controlPlaneBudget,
+		serial:   make(chan queuedCommand, serialQueueDepth),
+		inFlight: make(map[string]struct{}),
+	}
+}
+
+func controlPlaneBudget(commandType string) time.Duration {
+	if budget, ok := rdsinstance.CommandBudget(commandType); ok {
+		return budget
+	}
+	return fallbackCommandBudget
 }
 
 func (c *commander) Run(ctx context.Context) {
+	defer c.running.Wait()
+	c.running.Go(func() { c.serveSerial(ctx) })
 	for {
 		if ctx.Err() != nil {
 			return
 		}
-		commands, err := c.cp.PollCommands(ctx, c.id, c.pending, c.wait)
+		commands, sent, err := c.poll(ctx)
 		if err != nil {
 			if ctx.Err() != nil {
 				return
 			}
+			if errors.Is(err, errReplyReady) {
+				continue
+			}
 			slog.Warn("rds-agent: command poll failed", "err", err)
 			select {
 			case <-ctx.Done():
@@ -127,14 +191,122 @@ func (c *commander) Run(ctx context.Context) {
 			}
 			continue
 		}
-		c.pending = nil
+		c.mu.Lock()
+		c.pending = c.pending[sent:]
+		c.mu.Unlock()
 
 		for _, cmd := range commands {
-			c.pending = append(c.pending, c.execute(ctx, cmd))
+			c.dispatch(ctx, cmd)
 		}
 	}
 }
 
+// Reports that a poll was ended early so a finished reply can ride the next one.
+var errReplyReady = errors.New("a command reply is ready")
+
+// One long poll carrying every reply ready when it starts. It is cut short when
+// another reply becomes ready, so a finished command is never answered a whole
+// poll window late. Returns how many pending replies the poll carried.
+func (c *commander) poll(ctx context.Context) ([]rdsinstance.Command, int, error) {
+	pollCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	c.mu.Lock()
+	replies := slices.Clone(c.pending)
+	c.interruptPoll, c.interrupted = cancel, false
+	c.mu.Unlock()
+
+	commands, err := c.cp.PollCommands(pollCtx, c.id, replies, c.wait)
+
+	c.mu.Lock()
+	interrupted := c.interrupted
+	c.interruptPoll = nil
+	c.mu.Unlock()
+	if err != nil {
+		if interrupted && ctx.Err() == nil {
+			return nil, 0, errReplyReady
+		}
+		return nil, 0, err
+	}
+	return commands, len(replies), nil
+}
+
+// Routes cmd to its lane unless it is already queued, running or answered: the
+// issuer re-publishes until it has a reply, so the same ID arrives again.
+func (c *commander) dispatch(ctx context.Context, cmd rdsinstance.Command) {
+	c.mu.Lock()
+	_, queued := c.inFlight[cmd.CommandID]
+	if queued || slices.Contains(c.completed, cmd.CommandID) {
+		c.mu.Unlock()
+		slog.Debug("rds-agent: duplicate command ignored", "commandID", cmd.CommandID, "type", cmd.Type)
+		return
+	}
+	c.inFlight[cmd.CommandID] = struct{}{}
+	c.mu.Unlock()
+
+	q := queuedCommand{cmd: cmd, deadline: time.Now().Add(c.budgetFor(cmd))}
+	if outOfBand(cmd.Type) {
+		c.running.Go(func() { c.runQueued(ctx, q) })
+		return
+	}
+	select {
+	case c.serial <- q:
+	default:
+		c.finish(cmd, failedReply(cmd, "the agent is busy; too many commands are waiting"))
+	}
+}
+
+// The type's budget, or what the issuer had left when it published this copy if
+// that is less: a copy delayed in transit must not be granted a fresh budget its
+// issuer will not wait out.
+func (c *commander) budgetFor(cmd rdsinstance.Command) time.Duration {
+	budget := c.budget(cmd.Type)
+	if cmd.RemainingBudgetMs > 0 {
+		budget = min(budget, time.Duration(cmd.RemainingBudgetMs)*time.Millisecond)
+	}
+	return budget
+}
+
+func (c *commander) serveSerial(ctx context.Context) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case q := <-c.serial:
+			c.runQueued(ctx, q)
+		}
+	}
+}
+
+// Runs a command inside what is left of its issuer's budget. One whose budget
+// ran out while it waited is answered as expired and never touches the engine,
+// since its issuer has already reported it failed.
+func (c *commander) runQueued(ctx context.Context, q queuedCommand) {
+	if !time.Now().Before(q.deadline) {
+		slog.Warn("rds-agent: command expired before it ran", "commandID", q.cmd.CommandID, "type", q.cmd.Type)
+		c.finish(q.cmd, failedReply(q.cmd, "the command expired before the agent could run it"))
+		return
+	}
+	cmdCtx, cancel := context.WithDeadline(ctx, q.deadline)
+	defer cancel()
+	c.finish(q.cmd, c.execute(cmdCtx, q.cmd))
+}
+
+func (c *commander) finish(cmd rdsinstance.Command, reply rdsinstance.CommandReply) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	delete(c.inFlight, cmd.CommandID)
+	c.completed = append(c.completed, cmd.CommandID)
+	if len(c.completed) > completedCommandMemory {
+		c.completed = c.completed[len(c.completed)-completedCommandMemory:]
+	}
+	c.pending = append(c.pending, reply)
+	if c.interruptPoll != nil {
+		c.interruptPoll()
+		c.interrupted = true
+	}
+}
+
 // Every command gets a reply: the issuer is blocked on this command ID, so a
 // silent drop costs it a full timeout.
 func (c *commander) execute(ctx context.Context, cmd rdsinstance.Command) rdsinstance.CommandReply {

spinifex/agents/rds/agent/command_test.go

@@ -3,6 +3,9 @@ package agent
 import (
 	"context"
 	"errors"
+	"fmt"
+	"slices"
+	"sync"
 	"testing"
 	"time"
 
@@ -70,9 +73,9 @@ func TestCommander_UnknownTypeRepliesFailedNotDropped(t *testing.T) {
 	}
 }
 
-// The reply for work the guest actually did rides the next poll. A poll that
+// The reply for work the guest actually did rides a later poll. A poll that
 // fails must not consume it, or the issuer never learns the outcome.
-func TestCommander_RunCarriesRepliesOnTheNextPoll(t *testing.T) {
+func TestCommander_RunCarriesRepliesOnALaterPoll(t *testing.T) {
 	cp := newFakeControlPlane()
 	cp.pollQueue = [][]rdsinstance.Command{
 		{{CommandID: "cmd-1", Type: "reload-parameters"}},
@@ -87,19 +90,15 @@ func TestCommander_RunCarriesRepliesOnTheNextPoll(t *testing.T) {
 	done := make(chan struct{})
 	go func() { c.Run(ctx); close(done) }()
 
-	waitFor(t, func() bool {
-		replies := cp.snapshotReplies()
-		return len(replies) >= 2 && len(replies[1]) == 1
-	}, "the reply to ride the second poll")
+	waitFor(t, func() bool { return len(repliesFor(cp, "cmd-1")) > 0 }, "the reply to ride a later poll")
 	cancel()
 	<-done
 
-	replies := cp.snapshotReplies()
-	if len(replies[0]) != 0 {
+	if replies := cp.snapshotReplies(); len(replies[0]) != 0 {
 		t.Errorf("first poll carried replies %+v, want none", replies[0])
 	}
-	if got := replies[1][0]; got.CommandID != "cmd-1" || got.Status != rdsinstance.CommandStatusSucceeded {
-		t.Errorf("second poll carried %+v, want the succeeded reply for cmd-1", got)
+	if got := repliesFor(cp, "cmd-1")[0]; got.Status != rdsinstance.CommandStatusSucceeded {
+		t.Errorf("the poll carried %+v, want the succeeded reply for cmd-1", got)
 	}
 }
 
@@ -141,16 +140,19 @@ func TestCommander_RetainsAndRetriesTheCompletePendingBatch(t *testing.T) {
 	done = make(chan struct{})
 	go func() { c.Run(ctx); close(done) }()
 	waitFor(t, func() bool {
-		return len(cp.snapshotReplies()) >= 3
-	}, "the retained batch to succeed and the new reply to reach the following poll")
+		return len(repliesFor(cp, "cmd-3")) > 0
+	}, "the retained batch to succeed and the new reply to reach a later poll")
 	cancel()
 	<-done
 
 	replies = cp.snapshotReplies()
 	assert.Equal(t, pending, replies[1], "the complete failed batch must be retried in order")
-	require.Len(t, replies[2], 1)
-	assert.Equal(t, "cmd-3", replies[2][0].CommandID)
-	assert.Equal(t, rdsinstance.CommandStatusSucceeded, replies[2][0].Status)
+	assert.Equal(t, rdsinstance.CommandStatusSucceeded, repliesFor(cp, "cmd-3")[0].Status)
+	for _, later := range replies[2:] {
+		for _, reply := range later {
+			assert.Equal(t, "cmd-3", reply.CommandID, "a batch a poll carried must not be sent again")
+		}
+	}
 }
 
 // A poll error must not spin the loop: without a backoff a gateway that is down
@@ -171,3 +173,340 @@ func TestCommander_BacksOffAfterAFailedPoll(t *testing.T) {
 		t.Errorf("polled %d times in 200ms against a broken channel, want 1", polls)
 	}
 }
+
+// A MariaDB statement that commits waits out a held BACKUP STAGE, and the
+// release is itself a command. Run one at a time, the release would queue behind
+// the statement it is meant to unblock until the hold's own deadline.
+func TestCommander_ABlockedCommandDoesNotKeepOutTheRelease(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.pollQueue = [][]rdsinstance.Command{
+		{{CommandID: "cmd-password", Type: rdsinstance.CommandSetPassword}},
+		{{CommandID: "cmd-release", Type: rdsinstance.CommandUnquiesce}},
+	}
+	released := make(chan struct{})
+	registry := commandRegistry{
+		rdsinstance.CommandSetPassword: func(ctx context.Context, _ rdsinstance.Command) (string, error) {
+			select {
+			case <-released:
+				return "", nil
+			case <-ctx.Done():
+				return "", ctx.Err()
+			}
+		},
+		rdsinstance.CommandUnquiesce: func(context.Context, rdsinstance.Command) (string, error) {
+			close(released)
+			return "", nil
+		},
+	}
+
+	c := newCommander(cp, registry, 10*time.Millisecond)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan struct{})
+	go func() { c.Run(ctx); close(done) }()
+
+	waitFor(t, func() bool {
+		return len(repliesFor(cp, "cmd-password")) > 0 && len(repliesFor(cp, "cmd-release")) > 0
+	}, "both the release and the command it unblocked to be answered")
+	cancel()
+	<-done
+
+	for _, id := range []string{"cmd-password", "cmd-release"} {
+		assert.Equal(t, rdsinstance.CommandStatusSucceeded, repliesFor(cp, id)[0].Status, id)
+	}
+}
+
+// The issuer re-publishes until it has a reply, so a command still running is
+// delivered again. Running the copy would rotate a password twice or open a
+// second backup alongside the first.
+func TestCommander_ARepublishedCommandRunsOnce(t *testing.T) {
+	cp := newFakeControlPlane()
+	cmd := rdsinstance.Command{CommandID: "cmd-1", Type: "reload-parameters"}
+	cp.pollQueue = [][]rdsinstance.Command{{cmd}, {cmd}}
+	var mu sync.Mutex
+	runs := 0
+	secondDelivered := make(chan struct{})
+	registry := commandRegistry{
+		"reload-parameters": func(ctx context.Context, _ rdsinstance.Command) (string, error) {
+			mu.Lock()
+			runs++
+			mu.Unlock()
+			select {
+			case <-secondDelivered:
+			case <-ctx.Done():
+			}
+			return "done", nil
+		},
+	}
+
+	c := newCommander(cp, registry, 10*time.Millisecond)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan struct{})
+	go func() { c.Run(ctx); close(done) }()
+
+	// The third poll means both copies have been handed to the commander.
+	waitFor(t, func() bool { return len(cp.snapshotReplies()) >= 3 }, "both copies to be delivered")
+	close(secondDelivered)
+	waitFor(t, func() bool { return len(repliesFor(cp, "cmd-1")) > 0 }, "the reply")
+	cancel()
+	<-done
+
+	mu.Lock()
+	defer mu.Unlock()
+	assert.Equal(t, 1, runs, "a re-published copy of a running command must not run again")
+}
+
+// Every reply to id carried on any poll so far.
+func repliesFor(cp *fakeControlPlane, id string) []rdsinstance.CommandReply {
+	var out []rdsinstance.CommandReply
+	for _, batch := range cp.snapshotReplies() {
+		for _, reply := range batch {
+			if reply.CommandID == id {
+				out = append(out, reply)
+			}
+		}
+	}
+	return out
+}
+
+// A copy can arrive after the handler has returned but before a poll has
+// carried its reply. The in-flight set has already let go of it then, so only
+// the completed set stops it running a second time.
+func TestCommander_ACopyArrivingBeforeTheReplyIsCarriedDoesNotRunAgain(t *testing.T) {
+	var mu sync.Mutex
+	runs := map[string]int{}
+	handler := func(_ context.Context, cmd rdsinstance.Command) (string, error) {
+		mu.Lock()
+		runs[cmd.CommandID]++
+		mu.Unlock()
+		return "", nil
+	}
+	c := newCommander(nil, commandRegistry{"reload-parameters": handler}, time.Second)
+	ctx := t.Context()
+	go c.serveSerial(ctx)
+
+	first := rdsinstance.Command{CommandID: "cmd-1", Type: "reload-parameters"}
+	c.dispatch(ctx, first)
+	waitFor(t, func() bool { return len(c.pendingReplies()) == 1 }, "the first run's reply to be pending")
+
+	c.dispatch(ctx, first)
+	// The lane is FIFO, so once this one has answered a queued copy would have run.
+	c.dispatch(ctx, rdsinstance.Command{CommandID: "cmd-2", Type: "reload-parameters"})
+	waitFor(t, func() bool {
+		return slices.ContainsFunc(c.pendingReplies(), func(r rdsinstance.CommandReply) bool { return r.CommandID == "cmd-2" })
+	}, "the following command's reply")
+
+	mu.Lock()
+	defer mu.Unlock()
+	assert.Equal(t, 1, runs["cmd-1"], "a copy arriving before the reply was carried must not run")
+	assert.Len(t, c.pendingReplies(), 2, "the copy must not be answered a second time either")
+}
+
+// A quiesce that waited out its issuer's budget behind another command would
+// put the engine in backup mode for a snapshot nobody is taking.
+func TestCommander_AStaleQuiesceNeverTakesAHold(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.pollQueue = [][]rdsinstance.Command{
+		{{CommandID: "cmd-apply", Type: rdsinstance.CommandApplyParams}},
+		{{CommandID: "cmd-quiesce", Type: rdsinstance.CommandQuiesce, Parameters: []rdsinstance.Parameter{
+			{Name: rdsinstance.CommandParamQuiesceLabel, Value: "orders-db-automated"},
+			{Name: rdsinstance.CommandParamQuiesceDeadlineSeconds, Value: "360"},
+		}}},
+	}
+	engine := &fakeEngine{}
+	registry := newCommandRegistry(engine, &fakeStorage{})
+	applying, release := make(chan struct{}), make(chan struct{})
+	registry[rdsinstance.CommandApplyParams] = func(ctx context.Context, _ rdsinstance.Command) (string, error) {
+		close(applying)
+		<-release
+		return "", nil
+	}
+	c := newCommander(cp, registry, 10*time.Millisecond)
+	c.budget = func(commandType string) time.Duration {
+		if commandType == rdsinstance.CommandQuiesce {
+			return 50 * time.Millisecond
+		}
+		return controlPlaneBudget(commandType)
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan struct{})
+	go func() { c.Run(ctx); close(done) }()
+
+	<-applying
+	waitFor(t, func() bool { return len(cp.snapshotReplies()) >= 3 }, "the quiesce to be delivered behind the apply")
+	time.Sleep(100 * time.Millisecond)
+	close(release)
+	waitFor(t, func() bool { return len(repliesFor(cp, "cmd-quiesce")) > 0 }, "the quiesce to be answered")
+	cancel()
+	<-done
+
+	reply := repliesFor(cp, "cmd-quiesce")[0]
+	assert.Equal(t, rdsinstance.CommandStatusFailed, reply.Status)
+	assert.Contains(t, reply.Message, "expired")
+	assert.Empty(t, engine.label, "an expired quiesce must never reach the engine")
+}
+
+// Engine commands still run one at a time: a parameter apply rewrites the
+// option file a graceful stop would otherwise race.
+func TestCommander_AParameterApplyNeverRunsAlongsideAnEngineStop(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.pollQueue = [][]rdsinstance.Command{
+		{{CommandID: "cmd-apply", Type: rdsinstance.CommandApplyParams}},
+		{{CommandID: "cmd-stop", Type: rdsinstance.CommandStopEngine}},
+	}
+	var mu sync.Mutex
+	active, most := 0, 0
+	release := make(chan struct{})
+	// Each holds the lane until released, so two running at once are both seen
+	// whichever is scheduled first.
+	engineWork := func(context.Context, rdsinstance.Command) (string, error) {
+		mu.Lock()
+		active++
+		most = max(most, active)
+		mu.Unlock()
+		<-release
+		mu.Lock()
+		active--
+		mu.Unlock()
+		return "", nil
+	}
+	registry := commandRegistry{
+		rdsinstance.CommandApplyParams: engineWork,
+		rdsinstance.CommandStopEngine:  engineWork,
+	}
+	c := newCommander(cp, registry, 10*time.Millisecond)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan struct{})
+	go func() { c.Run(ctx); close(done) }()
+
+	waitFor(t, func() bool { return len(cp.snapshotReplies()) >= 3 }, "the stop to be delivered behind the apply")
+	time.Sleep(50 * time.Millisecond)
+	close(release)
+	waitFor(t, func() bool { return len(repliesFor(cp, "cmd-stop")) > 0 }, "the stop to be answered")
+	cancel()
+	<-done
+
+	mu.Lock()
+	defer mu.Unlock()
+	assert.Equal(t, 1, most, "the engine stop ran while the parameter apply was still running")
+}
+
+// The agent bounds each command by its issuer's budget, so a type the control
+// plane has no budget for would run on a guess.
+func TestCommandRegistry_EveryTypeHasAControlPlaneBudget(t *testing.T) {
+	for commandType := range newCommandRegistry(&fakeEngine{}, &fakeStorage{}) {
+		if _, ok := rdsinstance.CommandBudget(commandType); !ok {
+			t.Errorf("%s has no control-plane budget", commandType)
+		}
+	}
+}
+
+func (c *commander) pendingReplies() []rdsinstance.CommandReply {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return slices.Clone(c.pending)
+}
+
+// The first copy can reach the agent late, after a gateway or bus outage, with
+// its issuer about to give up. A quiesce granted a fresh budget then would take
+// a hold nobody is left to release.
+func TestCommander_AQuiesceStampedNearlyOutOfBudgetNeverTakesAHold(t *testing.T) {
+	cp := newFakeControlPlane()
+	cp.pollQueue = [][]rdsinstance.Command{
+		{{CommandID: "cmd-apply", Type: rdsinstance.CommandApplyParams}},
+		{{CommandID: "cmd-quiesce", Type: rdsinstance.CommandQuiesce, RemainingBudgetMs: 50, Parameters: []rdsinstance.Parameter{
+			{Name: rdsinstance.CommandParamQuiesceLabel, Value: "orders-db-automated"},
+			{Name: rdsinstance.CommandParamQuiesceDeadlineSeconds, Value: "360"},
+		}}},
+	}
+	engine := &fakeEngine{}
+	registry := newCommandRegistry(engine, &fakeStorage{})
+	applying, release := make(chan struct{}), make(chan struct{})
+	registry[rdsinstance.CommandApplyParams] = func(context.Context, rdsinstance.Command) (string, error) {
+		close(applying)
+		<-release
+		return "", nil
+	}
+	c := newCommander(cp, registry, 10*time.Millisecond)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	done := make(chan struct{})
+	go func() { c.Run(ctx); close(done) }()
+
+	<-applying
+	waitFor(t, func() bool { return len(cp.snapshotReplies()) >= 3 }, "the quiesce to be delivered behind the apply")
+	// The quiesce was received before the third poll began, so its stamped budget
+	// has run out 50ms after this point.
+	queuedAt := time.Now()
+	waitFor(t, func() bool { return time.Since(queuedAt) > 50*time.Millisecond }, "the stamped budget to pass")
+	close(release)
+	waitFor(t, func() bool { return len(repliesFor(cp, "cmd-quiesce")) > 0 }, "the quiesce to be answered")
+	cancel()
+	<-done
+
+	reply := repliesFor(cp, "cmd-quiesce")[0]
+	assert.Equal(t, rdsinstance.CommandStatusFailed, reply.Status)
+	assert.Contains(t, reply.Message, "expired")
+	assert.Empty(t, engine.label, "a quiesce whose issuer had all but given up must never reach the engine")
+}
+
+// A full lane refuses at once rather than queueing work its issuer will not
+// outlive, and the refusal is the command's answer: a re-published copy must
+// not slip in once the lane has room.
+func TestCommander_AFullLaneRefusesAtOnceAndNeverRunsTheCommand(t *testing.T) {
+	var mu sync.Mutex
+	runs := map[string]int{}
+	running, release := make(chan struct{}, 1), make(chan struct{})
+	handler := func(_ context.Context, cmd rdsinstance.Command) (string, error) {
+		mu.Lock()
+		runs[cmd.CommandID]++
+		mu.Unlock()
+		if cmd.CommandID == "cmd-blocker" {
+			running <- struct{}{}
+			<-release
+		}
+		return "", nil
+	}
+	c := newCommander(nil, commandRegistry{"reload-parameters": handler}, time.Second)
+	ctx := t.Context()
+	go c.serveSerial(ctx)
+
+	c.dispatch(ctx, rdsinstance.Command{CommandID: "cmd-blocker", Type: "reload-parameters"})
+	<-running
+	for i := range serialQueueDepth {
+		c.dispatch(ctx, rdsinstance.Command{CommandID: fmt.Sprintf("cmd-queued-%d", i), Type: "reload-parameters"})
+	}
+
+	overflow := rdsinstance.Command{CommandID: "cmd-overflow", Type: "reload-parameters"}
+	c.dispatch(ctx, overflow)
+	refused := slices.IndexFunc(c.pendingReplies(), func(r rdsinstance.CommandReply) bool { return r.CommandID == overflow.CommandID })
+	require.GreaterOrEqual(t, refused, 0, "a full lane must answer at once, before anything ahead of it finishes")
+	assert.Equal(t, rdsinstance.CommandStatusFailed, c.pendingReplies()[refused].Status)
+	assert.Contains(t, c.pendingReplies()[refused].Message, "busy")
+
+	close(release)
+	last := fmt.Sprintf("cmd-queued-%d", serialQueueDepth-1)
+	waitFor(t, func() bool {
+		return slices.ContainsFunc(c.pendingReplies(), func(r rdsinstance.CommandReply) bool { return r.CommandID == last })
+	}, "the queue to drain")
+	c.dispatch(ctx, overflow)
+	// FIFO again: once this answers, a copy let onto the lane would have run.
+	c.dispatch(ctx, rdsinstance.Command{CommandID: "cmd-after", Type: "reload-parameters"})
+	waitFor(t, func() bool {
+		return slices.ContainsFunc(c.pendingReplies(), func(r rdsinstance.CommandReply) bool { return r.CommandID == "cmd-after" })
+	}, "the command after the re-published copy")
+
+	mu.Lock()
+	defer mu.Unlock()
+	assert.Zero(t, runs[overflow.CommandID], "a refused command must never run")
+	answers := 0
+	for _, r := range c.pendingReplies() {
+		if r.CommandID == overflow.CommandID {
+			answers++
+		}
+	}
+	assert.Equal(t, 1, answers, "a re-published copy of a refused command must be ignored, not answered again")
+}

spinifex/agents/rds/agent/engine_mariadb.go

@@ -94,6 +94,8 @@ type mariadbEngine struct {
 	socket    string
 	rcService string
 	service   string
+	// How long a password rotation waits for a backup hold to end.
+	holdWait time.Duration
 }
 
 var _ engine = (*mariadbEngine)(nil)
@@ -151,6 +153,7 @@ func newMariaDBEngine(cfg Config, rules controlPlaneRules, run commandRunner, st
 		socket:    mariadbSocketPath(cfg),
 		rcService: cfg.RCService,
 		service:   cfg.EngineService,
+		holdWait:  mariadbPasswordHoldWait,
 		parameterManager: parameterManager{
 			probe: probe,
 			params: parameterStore{
@@ -380,6 +383,11 @@ SET SESSION slow_query_log = 0;
 SET SESSION sql_mode = 'NO_ENGINE_SUBSTITUTION';
 `
 
+// ALTER USER can take effect in memory and then wait at its commit for a held
+// BACKUP STAGE to end. Waiting for the agent's own hold before the statement, for
+// less than the control plane's budget, keeps that hold from half-applying it.
+const mariadbPasswordHoldWait = 20 * time.Second
+
 // Rotates the master's password. The statement is built here rather than by the
 // client, which has neither psql's identifier interpolation nor its parameter
 // quoting, and it rides stdin so it is never visible in the process table.
@@ -395,16 +403,34 @@ func (e *mariadbEngine) SetPassword(ctx context.Context, username, password stri
 		return fmt.Errorf("refusing to set the password of role %q: %w", username, err)
 	}
 
+	// Only the agent's own backup hold is waited out before the statement. A lock
+	// someone else holds, such as a customer FTWRL, is met inside it and ended by
+	// lock_wait_timeout, by which point the password may already have changed.
 	escaped := sqlLiteral(password)
-	sql := fmt.Sprintf("%sALTER USER '%s'@'%s' IDENTIFIED BY '%s';\n",
-		mariadbSessionGuard, username, mariadbMasterHost, escaped)
-	if _, err := e.clientRun(ctx, sql); err != nil {
-		// The client echoes the failing statement, which carries the escaped form
-		// of the password rather than the raw one — so redacting only the raw one
-		// would leak exactly the passwords that needed escaping.
-		return fmt.Errorf("apply the master password: %s", redact(redact(err.Error(), password), escaped))
+	return e.outsideHold(ctx, e.holdWait, func() error {
+		// Computed after the hold wait, so the statement gets only what is left.
+		sql := fmt.Sprintf("%sSET SESSION lock_wait_timeout = %d;\nALTER USER '%s'@'%s' IDENTIFIED BY '%s';\n",
+			mariadbSessionGuard, statementLockWait(ctx), username, mariadbMasterHost, escaped)
+		if _, err := e.clientRun(ctx, sql); err != nil {
+			// The client echoes the failing statement, which carries the escaped form
+			// of the password rather than the raw one — so redacting only the raw one
+			// would leak exactly the passwords that needed escaping.
+			return fmt.Errorf("apply the master password: %s", redact(redact(err.Error(), password), escaped))
+		}
+		return nil
+	})
+}
+
+// The server-side lock wait for a statement, in whole seconds: what remains of
+// ctx less a margin for the reply, so the server gives up its wait before the
+// client is killed. A ctx with no deadline gets the full password budget.
+func statementLockWait(ctx context.Context) int {
+	const replyMargin = 2 * time.Second
+	remaining := mariadbPasswordHoldWait
+	if deadline, ok := ctx.Deadline(); ok {
+		remaining = time.Until(deadline) - replyMargin
 	}
-	return nil
+	return max(1, int(remaining.Seconds()))
 }
 
 // Installs the resolved set and applies the half a running server can adopt.
@@ -714,7 +740,9 @@ func (e *mariadbEngine) Quiesce(ctx context.Context, label string, hold time.Dur
 
 	// Held across the whole start, so a second quiesce waits and then finds the
 	// first one's hold rather than opening a concurrent backup alongside it.
-	e.mu.Lock()
+	if err := e.lockWithin(ctx); err != nil {
+		return err
+	}
 	defer e.mu.Unlock()
 	if e.held != nil {
 		return fmt.Errorf("the engine is already quiesced for backup %s", e.held.label)

spinifex/agents/rds/agent/engine_mariadb_test.go

@@ -1151,3 +1151,123 @@ func TestMariaDBEngine_EveryResolvedDynamicValueRenders(t *testing.T) {
 		}
 	}
 }
+
+// ALTER USER under a held BACKUP STAGE swaps the password in memory and then
+// waits at its commit, so a rotation issued mid-backup must not reach the
+// engine until the hold is released.
+func TestMariaDBEngine_SetPasswordWaitsForTheBackupHoldToEnd(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+	engine.startSess = func(context.Context, command) (engineSession, error) { return &fakeSession{}, nil }
+	if err := engine.Quiesce(context.Background(), "orders-db-automated", time.Minute); err != nil {
+		t.Fatalf("Quiesce: %v", err)
+	}
+
+	rotated := make(chan error, 1)
+	go func() { rotated <- engine.SetPassword(context.Background(), "mulgamaster", "n3w-pw") }()
+	select {
+	case err := <-rotated:
+		t.Fatalf("SetPassword returned %v while the engine was held for a backup", err)
+	case <-time.After(3 * holdPollInterval):
+	}
+
+	if err := engine.Unquiesce(context.Background()); err != nil {
+		t.Fatalf("Unquiesce: %v", err)
+	}
+	select {
+	case err := <-rotated:
+		if err != nil {
+			t.Fatalf("SetPassword after the release: %v", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("SetPassword did not run once the hold was released")
+	}
+	if len(runner.calls) != 1 || !strings.Contains(runner.calls[0].Stdin, "ALTER USER") {
+		t.Errorf("ran %d commands, want the one rotation after the release", len(runner.calls))
+	}
+}
+
+// A hold that outlasts the wait fails the rotation before anything is sent, so
+// the control plane's error is the truth rather than half of it.
+func TestMariaDBEngine_SetPasswordFailsUntouchedWhenTheHoldOutlastsItsWait(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+	engine.holdWait = 3 * holdPollInterval
+	engine.startSess = func(context.Context, command) (engineSession, error) { return &fakeSession{}, nil }
+	if err := engine.Quiesce(context.Background(), "orders-db-automated", time.Minute); err != nil {
+		t.Fatalf("Quiesce: %v", err)
+	}
+
+	if err := engine.SetPassword(context.Background(), "mulgamaster", "n3w-pw"); err == nil {
+		t.Fatal("SetPassword succeeded while the engine stayed held for a backup")
+	}
+	if len(runner.calls) != 0 {
+		t.Errorf("ran %d commands against a held engine, want none", len(runner.calls))
+	}
+}
+
+// The wait has to end before the control plane gives up on the command, or the
+// rotation can still land after the caller was told it failed.
+func TestMariaDBEngine_PasswordHoldWaitFitsInsideTheControlPlaneTimeout(t *testing.T) {
+	budget, ok := rdsinstance.CommandBudget(rdsinstance.CommandSetPassword)
+	if !ok {
+		t.Fatal("the control plane has no budget for set-password")
+	}
+	if mariadbPasswordHoldWait >= budget-5*time.Second {
+		t.Errorf("a %s hold wait leaves under 5s of the control plane's %s set-password budget for the statement", mariadbPasswordHoldWait, budget)
+	}
+}
+
+// The statement is bounded by what is left of the command, so the server gives
+// up a lock wait before the client is killed under it.
+func TestMariaDBEngine_SetPasswordBoundsTheStatementByTheRemainingBudget(t *testing.T) {
+	runner := &recordingRunner{}
+	engine := newTestMariaDBEngine(t, runner.run)
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	if err := engine.SetPassword(ctx, "mulgamaster", "n3w-pw"); err != nil {
+		t.Fatalf("SetPassword: %v", err)
+	}
+	sql := runner.calls[0].Stdin
+	at := strings.Index(sql, "SET SESSION lock_wait_timeout = ")
+	if at < 0 || at > strings.Index(sql, "ALTER USER") {
+		t.Fatalf("SQL %q does not bound the statement's lock wait before it runs", sql)
+	}
+	var seconds int
+	if _, err := fmt.Sscanf(sql[at:], "SET SESSION lock_wait_timeout = %d;", &seconds); err != nil {
+		t.Fatalf("parse the lock wait: %v", err)
+	}
+	if seconds < 1 || seconds > 8 {
+		t.Errorf("lock wait = %ds with 10s of budget left, want it inside the budget less the reply margin", seconds)
+	}
+}
+
+// A quiesce that cannot get the hold bookkeeping must give up with its command,
+// not wait on the mutex after its issuer has moved on.
+func TestMariaDBEngine_QuiesceGivesUpWaitingForTheBookkeepingWithItsContext(t *testing.T) {
+	engine := newTestMariaDBEngine(t, (&recordingRunner{}).run)
+	started := 0
+	engine.startSess = func(context.Context, command) (engineSession, error) {
+		started++
+		return &fakeSession{}, nil
+	}
+	engine.mu.Lock()
+	defer engine.mu.Unlock()
+
+	ctx, cancel := context.WithTimeout(context.Background(), 3*holdPollInterval)
+	defer cancel()
+	done := make(chan error, 1)
+	go func() { done <- engine.Quiesce(ctx, "orders-db-automated", time.Minute) }()
+	select {
+	case err := <-done:
+		if err == nil {
+			t.Fatal("Quiesce succeeded without the bookkeeping lock")
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatal("Quiesce waited on the bookkeeping past its context")
+	}
+	if started != 0 {
+		t.Errorf("opened %d backup sessions, want none", started)
+	}
+}

spinifex/agents/rds/agent/engine_postgres.go

@@ -516,7 +516,9 @@ func (e *postgresEngine) Quiesce(ctx context.Context, label string, hold time.Du
 
 	// Held across the whole start, so a second quiesce waits and then finds the
 	// first one's hold rather than opening a concurrent backup alongside it.
-	e.mu.Lock()
+	if err := e.lockWithin(ctx); err != nil {
+		return err
+	}
 	defer e.mu.Unlock()
 	if e.held != nil {
 		return fmt.Errorf("the engine is already quiesced for backup %s", e.held.label)

spinifex/agents/rds/agent/quiesce.go

@@ -114,6 +114,48 @@ func (q *quiesceState) releaseHold(ctx context.Context, releaseSQL string) error
 	return nil
 }
 
+// Takes mu, giving up when ctx ends. A quiesce that cannot get the bookkeeping
+// inside its budget must fail rather than start a backup its issuer abandoned.
+func (q *quiesceState) lockWithin(ctx context.Context) error {
+	for !q.mu.TryLock() {
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("wait for the backup bookkeeping: %w", ctx.Err())
+		case <-time.After(holdPollInterval):
+		}
+	}
+	return nil
+}
+
+// How often a statement waiting for a backup hold to end looks again.
+const holdPollInterval = 100 * time.Millisecond
+
+// Runs fn while no backup hold is on the engine, and keeps a new one from
+// starting until it returns. A hold that outlasts wait fails the call before fn
+// has touched the engine.
+func (q *quiesceState) outsideHold(ctx context.Context, wait time.Duration, fn func() error) error {
+	deadline := time.NewTimer(wait)
+	defer deadline.Stop()
+	for {
+		// Not Lock: a quiesce holds mu while its stages wait on their own locks,
+		// and this wait has to stay bounded by ctx and wait.
+		if q.mu.TryLock() {
+			if q.held == nil {
+				defer q.mu.Unlock()
+				return fn()
+			}
+			q.mu.Unlock()
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-deadline.C:
+			return fmt.Errorf("the engine is still held for a backup after %s", wait)
+		case <-time.After(holdPollInterval):
+		}
+	}
+}
+
 // The deadline. Ending the session is enough — both engines release the hold
 // with it — so this never has to reach the engine itself.
 func (q *quiesceState) expire(label string) {

spinifex/domains/acm/keyalgorithm_test.go

@@ -152,3 +152,30 @@ func TestImportCertificate_SerialZeroRejected(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorValidationException, code)
 	assert.Equal(t, "The serial number in the certificate is not supported by ACM.", msg)
 }
+
+func TestImportCertificate_ExpiredRejected(t *testing.T) {
+	t.Parallel()
+	svc := setupACMService(t)
+	key, err := rsa.GenerateKey(rand.Reader, 1024)
+	require.NoError(t, err)
+	tmpl := &x509.Certificate{
+		SerialNumber: big.NewInt(time.Now().UnixNano()),
+		Subject:      pkix.Name{CommonName: "expired.example.com"},
+		NotBefore:    time.Now().Add(-48 * time.Hour),
+		NotAfter:     time.Now().Add(-24 * time.Hour),
+	}
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
+	require.NoError(t, err)
+	_, err = svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{
+		Certificate: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}),
+		PrivateKey:  pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}),
+	}, testAccountID)
+	code, msg, ok := awserrors.ResolveErrorDetail(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorValidationException, code)
+	assert.Equal(t, "The certificate is expired. You can Import a certificate only during its validity period.", msg)
+
+	list, err := svc.ListCertificates(context.Background(), &acm.ListCertificatesInput{}, testAccountID)
+	require.NoError(t, err)
+	assert.Empty(t, list.CertificateSummaryList)
+}

spinifex/domains/acm/service_impl.go

@@ -223,10 +223,13 @@ func (s *ACMServiceImpl) ImportCertificate(ctx context.Context, input *acm.Impor
 	if leaf.SerialNumber.Sign() == 0 {
 		return nil, awserrors.Errorf(awserrors.ErrorValidationException, "The serial number in the certificate is not supported by ACM.")
 	}
-
 	certArn := aws.StringValue(input.CertificateArn)
-	var inUseBy []string
 	now := time.Now().UTC()
+	if certArn == "" && now.After(leaf.NotAfter) {
+		return nil, awserrors.Errorf(awserrors.ErrorValidationException, "The certificate is expired. You can Import a certificate only during its validity period.")
+	}
+
+	var inUseBy []string
 	createdAt := now
 	tags := tagsToMap(input.Tags)
 	reimport := certArn != ""

spinifex/domains/ec2/awsapi/actions.go

@@ -134,6 +134,9 @@ func marshalEC2Response(action string, output any) ([]byte, error) {
 
 var ec2Actions = map[string]ec2Action{
 	"DescribeInstances": ec2Handler(func(ctx context.Context, input *ec2.DescribeInstancesInput, c *call, accountID string) (any, error) {
+		if err := ec2instanceapi.ValidateDescribeInstancesInput(input); err != nil {
+			return nil, err
+		}
 		out, err := ec2instanceapi.DescribeInstancesChecked(ctx, input, c.NATS, 0, c.NodeIDs, accountID)
 		if err != nil {
 			return out, err
@@ -257,6 +260,9 @@ var ec2Actions = map[string]ec2Action{
 		}
 	}),
 	"DescribeImages": ec2Handler(func(ctx context.Context, input *ec2.DescribeImagesInput, c *call, accountID string) (any, error) {
+		if err := ec2imageapi.ValidateDescribeImagesInput(input); err != nil {
+			return nil, err
+		}
 		return ec2imageapi.DescribeImages(ctx, input, c.NATS, accountID)
 	}),
 	"CreateImage": ec2Handler(func(ctx context.Context, input *ec2.CreateImageInput, c *call, accountID string) (any, error) {
@@ -287,6 +293,9 @@ var ec2Actions = map[string]ec2Action{
 		return ec2zoneapi.DescribeAvailabilityZones(input, c.Region, c.AZ)
 	}),
 	"DescribeVolumes": ec2Handler(func(ctx context.Context, input *ec2.DescribeVolumesInput, c *call, accountID string) (any, error) {
+		if err := ec2volumeapi.ValidateDescribeVolumesInput(input); err != nil {
+			return nil, err
+		}
 		return ec2volumeapi.DescribeVolumes(ctx, input, c.NATS, accountID)
 	}),
 	"ModifyVolume": ec2Handler(func(ctx context.Context, input *ec2.ModifyVolumeInput, c *call, accountID string) (any, error) {

spinifex/domains/ec2/awsapi/actions_test.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -30,3 +31,27 @@ func TestImportKeyPair_Base64PaddingWorkaround(t *testing.T) {
 	assert.NotContains(t, q["PublicKeyMaterial"], "%3D",
 		"Expected no URL-encoded padding remaining")
 }
+
+// A well-prefixed ID of the wrong shape is refused before any service is
+// asked, so it can never come back as NotFound.
+func TestDescribe_MalformedIDRejectedBeforeLookup(t *testing.T) {
+	tests := []struct {
+		action, param, id, code, msg string
+	}{
+		{"DescribeInstances", "InstanceId.1", "i-zzz", awserrors.ErrorInvalidInstanceIDMalformed, `Invalid id: "i-zzz"`},
+		{"DescribeImages", "ImageId.1", "ami-zzz", awserrors.ErrorInvalidAMIIDMalformed, `Invalid id: "ami-zzz"`},
+		{"DescribeVolumes", "VolumeId.1", "vol-zzz", awserrors.ErrorInvalidParameterValue, "Value (vol-zzz) for parameter volumes is invalid. Expected: 'vol-...'."},
+	}
+	for _, tt := range tests {
+		t.Run(tt.action, func(t *testing.T) {
+			h := ec2Actions[tt.action]
+			input, err := h.parse(map[string]string{tt.param: tt.id})
+			require.NoError(t, err)
+			_, err = h.dispatch(t.Context(), tt.action, input, &call{Handler: NewHandler(Deps{})}, "123456789012")
+			code, msg, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok)
+			assert.Equal(t, tt.code, code)
+			assert.Equal(t, tt.msg, msg)
+		})
+	}
+}

spinifex/domains/ec2/awsapi/image/DescribeImages.go

@@ -5,6 +5,7 @@ package image
 import (
 	"context"
 	"errors"
+	"regexp"
 	"strings"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -13,8 +14,12 @@ import (
 	"github.com/nats-io/nats.go"
 )
 
-// ValidateDescribeImagesInput accepts a nil input and rejects any ImageIds entry without the ami-
-// prefix with InvalidAMIID.Malformed.
+// imageIDSuffixPattern is the AWS AMI ID shape after ami-: 8 (legacy) or 17
+// lowercase hex characters.
+var imageIDSuffixPattern = regexp.MustCompile(`^([0-9a-f]{8}|[0-9a-f]{17})$`)
+
+// ValidateDescribeImagesInput accepts a nil input and rejects any ImageIds entry that is not
+// ami- followed by 8 or 17 lowercase hex characters with InvalidAMIID.Malformed.
 func ValidateDescribeImagesInput(input *ec2.DescribeImagesInput) (err error) {
 	if input == nil {
 		return nil
@@ -22,9 +27,16 @@ func ValidateDescribeImagesInput(input *ec2.DescribeImagesInput) (err error) {
 
 	if input.ImageIds != nil {
 		for _, imageId := range input.ImageIds {
-			if imageId != nil && !strings.HasPrefix(*imageId, "ami-") {
+			if imageId == nil {
+				continue
+			}
+			suffix, ok := strings.CutPrefix(*imageId, "ami-")
+			if !ok {
 				return errors.New(awserrors.ErrorInvalidAMIIDMalformed)
 			}
+			if !imageIDSuffixPattern.MatchString(suffix) {
+				return awserrors.Errorf(awserrors.ErrorInvalidAMIIDMalformed, "Invalid id: %q", *imageId)
+			}
 		}
 	}
 
@@ -34,11 +46,6 @@ func ValidateDescribeImagesInput(input *ec2.DescribeImagesInput) (err error) {
 // DescribeImages implements the EC2 DescribeImages action, returning the AMIs visible to
 // accountID from the NATS image service.
 func DescribeImages(ctx context.Context, input *ec2.DescribeImagesInput, natsConn *nats.Conn, accountID string) (output ec2.DescribeImagesOutput, err error) {
-	err = ValidateDescribeImagesInput(input)
-	if err != nil {
-		return output, err
-	}
-
 	imageService := ec2image.NewNATSImageService(natsConn, 0)
 	result, err := imageService.DescribeImages(ctx, input, accountID)
 	if err != nil {

spinifex/domains/ec2/awsapi/image/DescribeImages_test.go

@@ -53,8 +53,8 @@ func TestValidateDescribeImagesInput(t *testing.T) {
 			name: "MultipleValidImageIds",
 			input: &ec2.DescribeImagesInput{
 				ImageIds: []*string{
-					aws.String("ami-111"),
-					aws.String("ami-222"),
+					aws.String("ami-1a2b3c4d"),
+					aws.String("ami-0123456789abcdef1"),
 				},
 			},
 			wantErr: false,
@@ -63,7 +63,7 @@ func TestValidateDescribeImagesInput(t *testing.T) {
 			name: "MixedValidAndInvalidImageIds",
 			input: &ec2.DescribeImagesInput{
 				ImageIds: []*string{
-					aws.String("ami-valid"),
+					aws.String("ami-0123456789abcdef0"),
 					aws.String("invalid-id"),
 				},
 			},
@@ -85,11 +85,24 @@ func TestValidateDescribeImagesInput(t *testing.T) {
 			err := ValidateDescribeImagesInput(tt.input)
 
 			if tt.wantErr {
-				assert.Error(t, err)
-				assert.Equal(t, tt.errMsg, err.Error())
+				code, ok := awserrors.ResolveErrorCode(err)
+				assert.True(t, ok)
+				assert.Equal(t, tt.errMsg, code)
 			} else {
 				assert.NoError(t, err)
 			}
 		})
 	}
 }
+
+// AWS answers a well-prefixed ID whose suffix is not 8 or 17 lowercase hex
+// characters with Malformed, not NotFound.
+func TestValidateDescribeImagesInput_MalformedSuffix(t *testing.T) {
+	for _, id := range []string{"ami-zzz", "ami-0123456789ABCDEF0", "ami-0123456789abcdef", "ami-"} {
+		err := ValidateDescribeImagesInput(&ec2.DescribeImagesInput{ImageIds: []*string{aws.String(id)}})
+		code, msg, ok := awserrors.ResolveErrorDetail(err)
+		assert.True(t, ok, id)
+		assert.Equal(t, awserrors.ErrorInvalidAMIIDMalformed, code, id)
+		assert.Equal(t, `Invalid id: "`+id+`"`, msg, id)
+	}
+}

spinifex/domains/ec2/awsapi/image/handler_test.go

@@ -39,13 +39,6 @@ func TestCreateImage_NilNATS(t *testing.T) {
 	assert.Error(t, err)
 }
 
-func TestDescribeImages_ValidationErrors(t *testing.T) {
-	_, err := DescribeImages(context.Background(), &ec2.DescribeImagesInput{
-		ImageIds: []*string{aws.String("bad-id")},
-	}, nil, "")
-	assert.EqualError(t, err, awserrors.ErrorInvalidAMIIDMalformed)
-}
-
 func TestDescribeImages_NilNATS(t *testing.T) {
 	_, err := DescribeImages(context.Background(), nil, nil, "acct-123")
 	assert.Error(t, err)

spinifex/domains/ec2/awsapi/instance/DescribeInstances.go

@@ -89,6 +89,12 @@ func DescribeInstances(ctx context.Context, input *ec2.DescribeInstancesInput, n
 	return &ec2.DescribeInstancesOutput{Reservations: reservations}, nil
 }
 
+// ValidateDescribeInstancesInput rejects any InstanceIds entry that is not a
+// well-formed instance ID with InvalidInstanceID.Malformed.
+func ValidateDescribeInstancesInput(input *ec2.DescribeInstancesInput) error {
+	return validateInstanceIDFormat(input.InstanceIds)
+}
+
 // DescribeInstancesChecked is the customer-facing variant served by the
 // gateway's DescribeInstances action. On top of DescribeInstances' aggregation,
 // it asserts InvalidInstanceID.NotFound for any explicitly named instance ID

spinifex/domains/ec2/awsapi/instance/IamInstanceProfileAssociation.go

@@ -35,7 +35,7 @@ func resolveAndAuthorizeProfile(spec *ec2.IamInstanceProfileSpecification, iamSv
 	}
 	profile, err := iamSvc.ResolveInstanceProfile(accountID, nameOrARN)
 	if err != nil {
-		if err.Error() == awserrors.ErrorIAMNoSuchEntity {
+		if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
 			return nil, errors.New(awserrors.ErrorInvalidIamInstanceProfileNotFound)
 		}
 		return nil, err

spinifex/domains/ec2/awsapi/instance/IamInstanceProfileAssociation_test.go

@@ -45,7 +45,7 @@ func (f *fakeIAMService) ResolveInstanceProfile(accountID, nameOrARN string) (*i
 	if f.resolveFn != nil {
 		return f.resolveFn(accountID, nameOrARN)
 	}
-	return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+	return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", nameOrARN)
 }
 
 func (f *fakeIAMService) CanonicalResourceARN(accountID string, kind arn.IAMResourceType, name string) (string, error) {

spinifex/domains/ec2/awsapi/instance/instanceid.go

@@ -25,7 +25,7 @@ var instanceIDPattern = regexp.MustCompile(`^i-([0-9a-f]{8}|[0-9a-f]{17})$`)
 func validateInstanceIDFormat(ids []*string) error {
 	for _, id := range ids {
 		if id != nil && !instanceIDPattern.MatchString(*id) {
-			return errors.New(awserrors.ErrorInvalidInstanceIDMalformed)
+			return awserrors.Errorf(awserrors.ErrorInvalidInstanceIDMalformed, "Invalid id: %q", *id)
 		}
 	}
 	return nil

spinifex/domains/ec2/awsapi/instance/instanceid_test.go

@@ -57,8 +57,10 @@ func TestValidateInstanceIDFormat(t *testing.T) {
 	}
 	for _, id := range malformed {
 		err := validateInstanceIDFormat([]*string{aws.String(id)})
-		require.Error(t, err, id)
-		assert.Equal(t, awserrors.ErrorInvalidInstanceIDMalformed, err.Error(), id)
+		code, msg, ok := awserrors.ResolveErrorDetail(err)
+		require.True(t, ok, id)
+		assert.Equal(t, awserrors.ErrorInvalidInstanceIDMalformed, code, id)
+		assert.Equal(t, `Invalid id: "`+id+`"`, msg, id)
 	}
 	err := validateInstanceIDFormat([]*string{aws.String("i-0123456789abcdef0"), aws.String("i-abc")})
 	require.Error(t, err, "one malformed id fails the whole batch")
@@ -100,8 +102,9 @@ func TestLifecycleVerbs_MalformedIDRejectedBeforeNATS(t *testing.T) {
 	}
 	for name, call := range calls {
 		err := call()
-		require.Error(t, err, name)
-		assert.Equal(t, awserrors.ErrorInvalidInstanceIDMalformed, err.Error(), name)
+		code, ok := awserrors.ResolveErrorCode(err)
+		require.True(t, ok, name)
+		assert.Equal(t, awserrors.ErrorInvalidInstanceIDMalformed, code, name)
 	}
 	require.NoError(t, nc.Flush())
 	assert.Zero(t, requests.Load(), "no NATS request may be sent for a batch holding a malformed id")

spinifex/domains/ec2/awsapi/volume/DescribeVolumes.go

@@ -5,6 +5,7 @@ package volume
 import (
 	"context"
 	"errors"
+	"regexp"
 	"strings"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -13,8 +14,13 @@ import (
 	"github.com/nats-io/nats.go"
 )
 
-// ValidateDescribeVolumesInput accepts a nil input and rejects any VolumeIds entry without the
-// vol- prefix with InvalidVolumeID.Malformed.
+// volumeIDSuffixPattern is the AWS volume ID shape after vol-: 8 (legacy) or
+// 17 lowercase hex characters.
+var volumeIDSuffixPattern = regexp.MustCompile(`^([0-9a-f]{8}|[0-9a-f]{17})$`)
+
+// ValidateDescribeVolumesInput accepts a nil input, rejects any VolumeIds entry without the
+// vol- prefix with InvalidVolumeID.Malformed, and any with a malformed suffix with
+// InvalidParameterValue, as AWS does.
 func ValidateDescribeVolumesInput(input *ec2.DescribeVolumesInput) error {
 	if input == nil {
 		return nil
@@ -22,9 +28,17 @@ func ValidateDescribeVolumesInput(input *ec2.DescribeVolumesInput) error {
 
 	if input.VolumeIds != nil {
 		for _, volumeId := range input.VolumeIds {
-			if volumeId != nil && !strings.HasPrefix(*volumeId, "vol-") {
+			if volumeId == nil {
+				continue
+			}
+			suffix, ok := strings.CutPrefix(*volumeId, "vol-")
+			if !ok {
 				return errors.New(awserrors.ErrorInvalidVolumeIDMalformed)
 			}
+			if !volumeIDSuffixPattern.MatchString(suffix) {
+				return awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
+					"Value (%s) for parameter volumes is invalid. Expected: 'vol-...'.", *volumeId)
+			}
 		}
 	}
 
@@ -35,11 +49,6 @@ func ValidateDescribeVolumesInput(input *ec2.DescribeVolumesInput) error {
 func DescribeVolumes(ctx context.Context, input *ec2.DescribeVolumesInput, natsConn *nats.Conn, accountID string) (ec2.DescribeVolumesOutput, error) {
 	var output ec2.DescribeVolumesOutput
 
-	err := ValidateDescribeVolumesInput(input)
-	if err != nil {
-		return output, err
-	}
-
 	volumeService := ec2volume.NewNATSVolumeService(natsConn)
 	result, err := volumeService.DescribeVolumes(ctx, input, accountID)
 	if err != nil {

spinifex/domains/ec2/awsapi/volume/DescribeVolumes_test.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -36,9 +37,9 @@ func TestValidateDescribeVolumesInput(t *testing.T) {
 			name: "ValidMultipleVolumeIds",
 			input: &ec2.DescribeVolumesInput{
 				VolumeIds: []*string{
-					aws.String("vol-abc123"),
-					aws.String("vol-def456"),
-					aws.String("vol-ghi789"),
+					aws.String("vol-0123456789abcdef1"),
+					aws.String("vol-1a2b3c4d"),
+					aws.String("vol-0123456789abcdef2"),
 				},
 			},
 			wantErr: false,
@@ -71,7 +72,7 @@ func TestValidateDescribeVolumesInput(t *testing.T) {
 			name: "MixedValidAndInvalid",
 			input: &ec2.DescribeVolumesInput{
 				VolumeIds: []*string{
-					aws.String("vol-valid123"),
+					aws.String("vol-0123456789abcdef3"),
 					aws.String("invalid-id"),
 				},
 			},
@@ -81,7 +82,7 @@ func TestValidateDescribeVolumesInput(t *testing.T) {
 		{
 			name: "NilVolumeIdInList",
 			input: &ec2.DescribeVolumesInput{
-				VolumeIds: []*string{nil, aws.String("vol-abc123")},
+				VolumeIds: []*string{nil, aws.String("vol-0123456789abcdef1")},
 			},
 			wantErr: false,
 		},
@@ -100,3 +101,15 @@ func TestValidateDescribeVolumesInput(t *testing.T) {
 		})
 	}
 }
+
+// AWS answers a well-prefixed ID whose suffix is not 8 or 17 lowercase hex
+// characters with InvalidParameterValue, not NotFound.
+func TestValidateDescribeVolumesInput_MalformedSuffix(t *testing.T) {
+	for _, id := range []string{"vol-zzz", "vol-0123456789ABCDEF0", "vol-0123456789abcdef", "vol-"} {
+		err := ValidateDescribeVolumesInput(&ec2.DescribeVolumesInput{VolumeIds: []*string{aws.String(id)}})
+		code, msg, ok := awserrors.ResolveErrorDetail(err)
+		assert.True(t, ok, id)
+		assert.Equal(t, awserrors.ErrorInvalidParameterValue, code, id)
+		assert.Equal(t, "Value ("+id+") for parameter volumes is invalid. Expected: 'vol-...'.", msg, id)
+	}
+}

spinifex/domains/ec2/awsapi/volume/handler_test.go

@@ -96,13 +96,6 @@ func TestModifyVolume_NilNATS(t *testing.T) {
 	assert.Error(t, err)
 }
 
-func TestDescribeVolumes_ValidationErrors(t *testing.T) {
-	_, err := DescribeVolumes(context.Background(), &ec2.DescribeVolumesInput{
-		VolumeIds: []*string{aws.String("bad-id")},
-	}, nil, "")
-	assert.EqualError(t, err, awserrors.ErrorInvalidVolumeIDMalformed)
-}
-
 func TestDescribeVolumes_NilNATS(t *testing.T) {
 	_, err := DescribeVolumes(context.Background(), nil, nil, "acct-123")
 	assert.Error(t, err)

spinifex/domains/ec2/guestmetadata/metadata.go

@@ -738,7 +738,7 @@ func (s *IMDSServiceImpl) profileFor(ctx context.Context, eni *eniFacts) (*resol
 	}
 	profile, err := s.iam.ResolveInstanceProfile(ctx, eni.iamAccountID(), inst.iamInstanceProfileArn)
 	if err != nil {
-		if err.Error() == awserrors.ErrorIAMNoSuchEntity {
+		if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
 			return nil, roleMissProfileDeleted, nil // profile deleted; treat as no role
 		}
 		slog.ErrorContext(ctx, "IMDS: resolve instance profile failed", "account_id", eni.iamAccountID(), "arn", inst.iamInstanceProfileArn, "err", err)

spinifex/domains/ec2/guestmetadata/metadata_test.go

@@ -1152,7 +1152,7 @@ func TestHTTP_SecurityCredentialsListBackendError500(t *testing.T) {
 // genuine backend error (TestHTTP_*BackendError500) is a 500.
 func TestHTTP_IAMInfoDeletedProfile404(t *testing.T) {
 	res := &fakeResolver{eni: testENI(), inst: &instanceFacts{iamInstanceProfileArn: "arn:aws:iam::111122223333:instance-profile/gone"}}
-	svc, _ := newTestService(res, &fakeIAM{profileErr: errors.New(awserrors.ErrorIAMNoSuchEntity)}, &fakeAssumer{})
+	svc, _ := newTestService(res, &fakeIAM{profileErr: awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile gone cannot be found.")}, &fakeAssumer{})
 	h := withTapENI(svc.httpHandler(), testENI())
 	token := issueToken(t, h)
 	rec := get(t, h, prefixMetaData+"iam/info", token)

spinifex/domains/ec2/vpc/eni.go

@@ -224,7 +224,7 @@ func (s *VPCServiceImpl) CreateNetworkInterface(ctx context.Context, input *ec2.
 	groupNames := make(map[string]string)
 	s.fillSGNames(ctx, accountID, record.SecurityGroupIds, groupNames)
 	return &ec2.CreateNetworkInterfaceOutput{
-		NetworkInterface: s.eniRecordToEC2(&record, accountID, groupNames),
+		NetworkInterface: s.eniRecordToEC2(&record, accountID, groupNames, s.vpcDNSHostnames(ctx, accountID, record.VpcId)),
 	}, nil
 }
 
@@ -595,7 +595,7 @@ func (s *VPCServiceImpl) DescribeNetworkInterfaces(ctx context.Context, input *e
 		}
 
 		s.fillSGNames(ctx, accountID, record.SecurityGroupIds, groupNames)
-		enis = append(enis, s.eniRecordToEC2(&record, accountID, groupNames))
+		enis = append(enis, s.eniRecordToEC2(&record, accountID, groupNames, s.vpcDNSHostnames(ctx, accountID, record.VpcId)))
 	}
 
 	// If specific ENI IDs were requested but not found, return error
@@ -1076,9 +1076,26 @@ func (s *VPCServiceImpl) fillSGNames(ctx context.Context, accountID string, ids
 	}
 }
 
+// vpcDNSHostnames reports whether the VPC has enableDnsHostnames on. An
+// unreadable VPC reads as off, so its interfaces go unnamed.
+func (s *VPCServiceImpl) vpcDNSHostnames(ctx context.Context, accountID, vpcID string) bool {
+	entry, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, vpcID))
+	if err != nil {
+		slog.WarnContext(ctx, "vpcDNSHostnames: VPC read failed", "vpcId", vpcID, "err", err)
+		return false
+	}
+	var rec VPCRecord
+	if err := json.Unmarshal(entry.Value(), &rec); err != nil {
+		slog.WarnContext(ctx, "vpcDNSHostnames: VPC unmarshal failed", "vpcId", vpcID, "err", err)
+		return false
+	}
+	return rec.EnableDnsHostnames
+}
+
 // eniRecordToEC2 converts an ENI record to an EC2 NetworkInterface, naming
-// each security group from groupNames.
-func (s *VPCServiceImpl) eniRecordToEC2(record *ENIRecord, accountID string, groupNames map[string]string) *ec2.NetworkInterface {
+// each security group from groupNames. AWS names the interface only while its
+// VPC has enableDnsHostnames on.
+func (s *VPCServiceImpl) eniRecordToEC2(record *ENIRecord, accountID string, groupNames map[string]string, dnsHostnames bool) *ec2.NetworkInterface {
 	// ENIs with spinifex:managed-by tag are system-managed (e.g. by ELBv2)
 	requesterManaged := record.Tags["spinifex:managed-by"] != ""
 
@@ -1116,7 +1133,7 @@ func (s *VPCServiceImpl) eniRecordToEC2(record *ENIRecord, accountID string, gro
 		eni.Groups = groups
 	}
 
-	if s.privateDNSName != nil && record.PrivateIpAddress != "" {
+	if dnsHostnames && s.privateDNSName != nil && record.PrivateIpAddress != "" {
 		if name := s.privateDNSName(record.PrivateIpAddress); name != "" {
 			eni.PrivateDnsName = aws.String(name)
 			eni.PrivateIpAddresses[0].PrivateDnsName = aws.String(name)

spinifex/domains/ec2/vpc/eni_test.go

@@ -55,15 +55,11 @@ func TestNetworkInterface_NamesGroupsAndPrivateAddress(t *testing.T) {
 	vpcId := createTestVPC(t, svc, "10.0.0.0/16")
 	subnetId := createTestSubnet(t, svc, vpcId, "10.0.1.0/24")
 	sgId := createTestSG(t, svc, vpcId, "web")
+	setHostnames(t, svc, vpcId, true)
 
 	// Without a namer an interface carries no private name.
 	unnamed := createTestENI(t, svc, subnetId)
-	desc, err := svc.DescribeNetworkInterfaces(context.Background(), &ec2.DescribeNetworkInterfacesInput{
-		NetworkInterfaceIds: []*string{aws.String(unnamed)},
-	}, testAccountID)
-	require.NoError(t, err)
-	require.Len(t, desc.NetworkInterfaces, 1)
-	assert.Nil(t, desc.NetworkInterfaces[0].PrivateDnsName)
+	assert.Nil(t, describeTestENI(t, svc, unnamed).PrivateDnsName)
 
 	svc.SetPrivateDNSNamer(func(ip string) string { return "name-of-" + ip })
 	out, err := svc.CreateNetworkInterface(context.Background(), &ec2.CreateNetworkInterfaceInput{
@@ -72,13 +68,8 @@ func TestNetworkInterface_NamesGroupsAndPrivateAddress(t *testing.T) {
 	}, testAccountID)
 	require.NoError(t, err)
 
-	desc, err = svc.DescribeNetworkInterfaces(context.Background(), &ec2.DescribeNetworkInterfacesInput{
-		NetworkInterfaceIds: []*string{out.NetworkInterface.NetworkInterfaceId},
-	}, testAccountID)
-	require.NoError(t, err)
-	require.Len(t, desc.NetworkInterfaces, 1)
-
-	for name, eni := range map[string]*ec2.NetworkInterface{"create": out.NetworkInterface, "describe": desc.NetworkInterfaces[0]} {
+	desc := describeTestENI(t, svc, *out.NetworkInterface.NetworkInterfaceId)
+	for name, eni := range map[string]*ec2.NetworkInterface{"create": out.NetworkInterface, "describe": desc} {
 		want := "name-of-" + aws.StringValue(eni.PrivateIpAddress)
 		assert.Equal(t, want, aws.StringValue(eni.PrivateDnsName), name)
 		require.Len(t, eni.PrivateIpAddresses, 1, name)
@@ -89,6 +80,58 @@ func TestNetworkInterface_NamesGroupsAndPrivateAddress(t *testing.T) {
 	}
 }
 
+func TestNetworkInterface_PrivateDnsNameFollowsVpcDnsHostnames(t *testing.T) {
+	t.Parallel()
+	svc := setupTestVPCService(t)
+	svc.SetPrivateDNSNamer(func(ip string) string { return "name-of-" + ip })
+	vpcId := createTestVPC(t, svc, "10.0.0.0/16")
+	subnetId := createTestSubnet(t, svc, vpcId, "10.0.1.0/24")
+
+	// A new VPC has enableDnsHostnames off, so its interfaces go unnamed.
+	out, err := svc.CreateNetworkInterface(context.Background(), &ec2.CreateNetworkInterfaceInput{
+		SubnetId: aws.String(subnetId),
+	}, testAccountID)
+	require.NoError(t, err)
+	eniId := *out.NetworkInterface.NetworkInterfaceId
+	for name, eni := range map[string]*ec2.NetworkInterface{"create": out.NetworkInterface, "describe": describeTestENI(t, svc, eniId)} {
+		assert.Nil(t, eni.PrivateDnsName, name)
+		require.Len(t, eni.PrivateIpAddresses, 1, name)
+		assert.Nil(t, eni.PrivateIpAddresses[0].PrivateDnsName, name)
+	}
+
+	// Turning the flag on names the existing interface.
+	setHostnames(t, svc, vpcId, true)
+	eni := describeTestENI(t, svc, eniId)
+	want := "name-of-" + aws.StringValue(eni.PrivateIpAddress)
+	assert.Equal(t, want, aws.StringValue(eni.PrivateDnsName))
+	assert.Equal(t, want, aws.StringValue(eni.PrivateIpAddresses[0].PrivateDnsName))
+
+	// And turning it off again drops the name.
+	setHostnames(t, svc, vpcId, false)
+	eni = describeTestENI(t, svc, eniId)
+	assert.Nil(t, eni.PrivateDnsName)
+	assert.Nil(t, eni.PrivateIpAddresses[0].PrivateDnsName)
+}
+
+func setHostnames(t *testing.T, svc *VPCServiceImpl, vpcId string, on bool) {
+	t.Helper()
+	_, err := svc.ModifyVpcAttribute(context.Background(), &ec2.ModifyVpcAttributeInput{
+		VpcId:              aws.String(vpcId),
+		EnableDnsHostnames: &ec2.AttributeBooleanValue{Value: aws.Bool(on)},
+	}, testAccountID)
+	require.NoError(t, err)
+}
+
+func describeTestENI(t *testing.T, svc *VPCServiceImpl, eniId string) *ec2.NetworkInterface {
+	t.Helper()
+	desc, err := svc.DescribeNetworkInterfaces(context.Background(), &ec2.DescribeNetworkInterfacesInput{
+		NetworkInterfaceIds: []*string{aws.String(eniId)},
+	}, testAccountID)
+	require.NoError(t, err)
+	require.Len(t, desc.NetworkInterfaces, 1)
+	return desc.NetworkInterfaces[0]
+}
+
 func TestCreateNetworkInterface_SequentialIPs(t *testing.T) {
 	t.Parallel()
 	svc := setupTestVPCService(t)

spinifex/domains/ecs/records.go

@@ -272,11 +272,15 @@ type TaskRecord struct {
 	// Group / StartedBy mirror the AWS task fields. A service's tasks carry
 	// Group="service:{name}" so the reconciler counts them and the task-state
 	// hook resolves a RUNNING/STOPPED task back to its owning service.
-	Group                string `json:"group,omitempty"`
-	StartedBy            string `json:"startedBy,omitempty"`
-	TaskDefFamily        string `json:"taskDefFamily"`
-	TaskDefRevision      int    `json:"taskDefRevision"`
-	TaskDefARN           string `json:"taskDefArn"`
+	Group           string `json:"group,omitempty"`
+	StartedBy       string `json:"startedBy,omitempty"`
+	TaskDefFamily   string `json:"taskDefFamily"`
+	TaskDefRevision int    `json:"taskDefRevision"`
+	TaskDefARN      string `json:"taskDefArn"`
+	// CredID is the secret under which the task's containers fetch task-role
+	// credentials from the agent. Set only when the task has a role; it must
+	// never be projected into an API response or logged.
+	CredID               string `json:"credId,omitempty"`
 	ContainerInstanceID  string `json:"containerInstanceId,omitempty"`
 	ContainerInstanceARN string `json:"containerInstanceArn,omitempty"`
 	DesiredStatus        string `json:"desiredStatus"`

spinifex/domains/ecs/service.go

@@ -288,7 +288,7 @@ func (s *Service) DescribeClusters(ctx context.Context, input *ecs.DescribeClust
 		names = []string{defaultCluster}
 	}
 	includeSettings := slices.Contains(awsStringSlice(input.Include), ecs.ClusterFieldSettings)
-	out := &ecs.DescribeClustersOutput{}
+	out := &ecs.DescribeClustersOutput{Clusters: []*ecs.Cluster{}, Failures: []*ecs.Failure{}}
 	for _, name := range names {
 		name = ClusterShortName(name)
 		var rec ClusterRecord
@@ -458,7 +458,7 @@ func (s *Service) RegisterTaskDefinition(ctx context.Context, input *ecs.Registe
 	if family == "" {
 		return nil, errors.New(awserrors.ErrorECSInvalidParameter)
 	}
-	if err := validateContainerDefs(input.ContainerDefinitions); err != nil {
+	if err := validateContainerDefs(input.ContainerDefinitions, aws.StringValue(input.Memory)); err != nil {
 		return nil, err
 	}
 	warnUnsupportedLogDrivers(ctx, family, input.ContainerDefinitions)
@@ -497,12 +497,23 @@ func (s *Service) RegisterTaskDefinition(ctx context.Context, input *ecs.Registe
 // The test here is per value, not per field: an empty list or a false/unset
 // flag requests nothing and is accepted, because there is nothing to enforce
 // or refuse. Only a value that asks for the restriction is rejected.
-func validateContainerDefs(defs []*ecs.ContainerDefinition) error {
+//
+// Ahead of those, AWS's own rules: every container needs an image, and without
+// a task-level memory every container needs memory or memoryReservation.
+func validateContainerDefs(defs []*ecs.ContainerDefinition, taskMemory string) error {
 	for _, c := range defs {
 		if c == nil {
 			continue
 		}
 		name := aws.StringValue(c.Name)
+		if aws.StringValue(c.Image) == "" {
+			return awserrors.Errorf(awserrors.ErrorECSClientException,
+				"Container.image should not be null or empty.")
+		}
+		if taskMemory == "" && c.Memory == nil && c.MemoryReservation == nil {
+			return awserrors.Errorf(awserrors.ErrorECSClientException,
+				"Invalid setting for container '%s'. At least one of 'memory' or 'memoryReservation' must be specified.", name)
+		}
 		if len(c.Secrets) > 0 {
 			return errors.New(awserrors.ErrorECSInvalidParameter)
 		}

spinifex/domains/ecs/service_test.go

@@ -2,7 +2,9 @@ package ecs
 
 import (
 	"context"
+	"encoding/json"
 	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
+	"maps"
 	"strconv"
 	"testing"
 	"time"
@@ -302,6 +304,89 @@ func TestService_RunTask_AssignCarriesTaskRole(t *testing.T) {
 	assert.Equal(t, roleARN, poll.Assignments[0].TaskRoleARN)
 }
 
+// A role task's credential ID is a scheduler-minted secret: not the public task
+// ID, distinct per task, stable on re-delivery, persisted on the task record and
+// absent from every API response. A role-less task gets none.
+func TestService_TaskCredIDIsMintedSecret(t *testing.T) {
+	ctx := context.Background()
+	svc, _ := newTestService(t)
+	_, err := svc.CreateCluster(ctx, &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
+	require.NoError(t, err)
+	_, err = svc.RegisterTaskDefinition(ctx, &ecs.RegisterTaskDefinitionInput{
+		Family:      aws.String("victim"),
+		TaskRoleArn: aws.String("arn:aws:iam::123456789012:role/task-high"),
+		ContainerDefinitions: []*ecs.ContainerDefinition{{
+			Name: aws.String("app"), Image: aws.String("registry/app:1"),
+			Cpu: aws.Int64(128), Memory: aws.Int64(256), Essential: aws.Bool(true),
+		}},
+	}, testAccountID)
+	require.NoError(t, err)
+	registerTaskDef(t, svc, "attacker", 128, 256)
+	registerInstance(t, svc, "web", "i-1", 1024, 2048)
+
+	run, err := svc.RunTask(ctx, &ecs.RunTaskInput{Cluster: aws.String("web"), TaskDefinition: aws.String("victim")}, testAccountID)
+	require.NoError(t, err)
+	start, err := svc.StartTask(ctx, &ecs.StartTaskInput{
+		Cluster: aws.String("web"), TaskDefinition: aws.String("victim"), ContainerInstances: []*string{aws.String("i-1")},
+	}, testAccountID)
+	require.NoError(t, err)
+	_, err = svc.StartTask(ctx, &ecs.StartTaskInput{
+		Cluster: aws.String("web"), TaskDefinition: aws.String("attacker"), ContainerInstances: []*string{aws.String("i-1")},
+	}, testAccountID)
+	require.NoError(t, err)
+
+	poll := func() map[string]ecsv1.Assign {
+		p, perr := svc.PollAssignments(ctx, &PollAssignmentsInput{Cluster: "web", ContainerInstance: "i-1"}, testAccountID)
+		require.NoError(t, perr)
+		byTask := map[string]ecsv1.Assign{}
+		for _, as := range p.Assignments {
+			byTask[as.TaskID] = as
+		}
+		return byTask
+	}
+	first, second := poll(), poll()
+	require.Len(t, first, 3)
+
+	kv, err := svc.bucket(ctx, testAccountID)
+	require.NoError(t, err)
+	var apiJSON []byte
+	for _, out := range []any{run, start} {
+		b, merr := json.Marshal(out)
+		require.NoError(t, merr)
+		apiJSON = append(apiJSON, b...)
+	}
+	var taskARNs []*string
+	for as := range maps.Values(first) {
+		taskARNs = append(taskARNs, aws.String(as.TaskARN))
+	}
+	desc, err := svc.DescribeTasks(ctx, &ecs.DescribeTasksInput{Cluster: aws.String("web"), Tasks: taskARNs}, testAccountID)
+	require.NoError(t, err)
+	b, err := json.Marshal(desc)
+	require.NoError(t, err)
+	apiJSON = append(apiJSON, b...)
+
+	seen := map[string]bool{}
+	for taskID, as := range first {
+		if as.TaskRoleARN == "" {
+			assert.Empty(t, as.CredID, "role-less task got a credential ID")
+			continue
+		}
+		require.NotEmpty(t, as.CredID)
+		assert.NotEqual(t, taskID, as.CredID)
+		assert.False(t, seen[as.CredID], "credential ID reused across tasks")
+		seen[as.CredID] = true
+		assert.Equal(t, as.CredID, second[taskID].CredID, "re-delivered assign changed credential ID")
+
+		var rec TaskRecord
+		found, gerr := getJSON(ctx, kv, TaskKey("web", taskID), &rec)
+		require.NoError(t, gerr)
+		require.True(t, found)
+		assert.Equal(t, as.CredID, rec.CredID)
+		assert.NotContains(t, string(apiJSON), as.CredID, "credential ID leaked in an API response")
+	}
+	assert.Len(t, seen, 2)
+}
+
 // PollAssignments is at-least-once: re-poll without ack re-delivers the assign;
 // acking it (then STOPPED) drains the inbox so it is never re-delivered.
 func TestService_PollAssignments_AckAndReclaim(t *testing.T) {
@@ -812,6 +897,12 @@ func TestService_ListTasks_EmptyIsPresentNotAbsent(t *testing.T) {
 	assert.Empty(t, out.TaskArns)
 }
 
+func TestService_ListTasks_ClusterNotFound(t *testing.T) {
+	svc, _ := newTestService(t)
+	_, err := svc.ListTasks(context.Background(), &ecs.ListTasksInput{Cluster: aws.String("ghost")}, testAccountID)
+	require.EqualError(t, err, awserrors.ErrorECSClusterNotFound)
+}
+
 func TestService_ListContainerInstances_EmptyIsPresentNotAbsent(t *testing.T) {
 	svc, _ := newTestService(t)
 	_, err := svc.CreateCluster(context.Background(), &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
@@ -829,3 +920,21 @@ func TestAccountIDFromBucket(t *testing.T) {
 	_, ok = accountIDFromBucket("some-other-bucket")
 	assert.False(t, ok)
 }
+
+func TestService_DescribeClusters_ListsAlwaysPresent(t *testing.T) {
+	svc, _ := newTestService(t)
+	_, err := svc.CreateCluster(context.Background(), &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
+	require.NoError(t, err)
+
+	found, err := svc.DescribeClusters(context.Background(), &ecs.DescribeClustersInput{Clusters: []*string{aws.String("web")}}, testAccountID)
+	require.NoError(t, err)
+	require.Len(t, found.Clusters, 1)
+	assert.NotNil(t, found.Failures)
+	assert.Empty(t, found.Failures)
+
+	miss, err := svc.DescribeClusters(context.Background(), &ecs.DescribeClustersInput{Clusters: []*string{aws.String("nope")}}, testAccountID)
+	require.NoError(t, err)
+	require.Len(t, miss.Failures, 1)
+	assert.NotNil(t, miss.Clusters)
+	assert.Empty(t, miss.Clusters)
+}

spinifex/domains/ecs/services_test.go

@@ -313,7 +313,7 @@ func TestService_CreateService_ProjectsNetworkConfigurationAndManagedTags(t *tes
 		Family:      aws.String("app"),
 		NetworkMode: aws.String(NetworkModeAwsvpc),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 		}},
 	}, testAccountID)
 	require.NoError(t, err)
@@ -352,7 +352,7 @@ func TestService_UpdateService_ChangesNetworkConfiguration(t *testing.T) {
 		Family:      aws.String("app"),
 		NetworkMode: aws.String(NetworkModeAwsvpc),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 		}},
 	}, testAccountID)
 	require.NoError(t, err)

spinifex/domains/ecs/task_accounting_test.go

@@ -16,6 +16,8 @@ import (
 // agent to reach anything other than PENDING.
 func seedTasks(t *testing.T, svc *ecsdomain.Service, cluster string, recs ...*ecsdomain.TaskRecord) {
 	t.Helper()
+	_, err := svc.CreateCluster(t.Context(), &ecs.CreateClusterInput{ClusterName: aws.String(cluster)}, ecsdomain.TestAccountID)
+	require.NoError(t, err)
 	kv, err := svc.Bucket(t.Context(), ecsdomain.TestAccountID)
 	require.NoError(t, err)
 	for _, r := range recs {

spinifex/domains/ecs/taskdef_validation_test.go

@@ -19,7 +19,7 @@ func TestRegisterTaskDefinition_RejectsSecrets(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			Secrets: []*ecs.Secret{{
 				Name: aws.String("DB_PASSWORD"), ValueFrom: aws.String("arn:aws:ssm:::parameter/db"),
 			}},
@@ -37,7 +37,7 @@ func TestRegisterTaskDefinition_AcceptsUnsupportedLogDriver(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			LogConfiguration: &ecs.LogConfiguration{
 				LogDriver: aws.String("awslogs"),
 				Options:   map[string]*string{"awslogs-group": aws.String("/ecs/app")},
@@ -67,7 +67,7 @@ func TestRegisterTaskDefinition_EchoesRequiresCompatibilities(t *testing.T) {
 		Family:                  aws.String("app"),
 		RequiresCompatibilities: aws.StringSlice([]string{"EC2"}),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 		}},
 	}, testAccountID)
 	require.NoError(t, err)
@@ -91,7 +91,7 @@ func TestRegisterTaskDefinition_EchoesRuntimePlatform(t *testing.T) {
 			OperatingSystemFamily: aws.String("LINUX"),
 		},
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 		}},
 	}, testAccountID)
 	require.NoError(t, err)
@@ -113,7 +113,7 @@ func TestRegisterTaskDefinition_RuntimePlatformOmittedWhenUnset(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 		}},
 	}, testAccountID)
 	require.NoError(t, err)
@@ -253,7 +253,7 @@ func TestRegisterTaskDefinition_EchoesRuntimeFields(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			User:                   aws.String("1000:1000"),
 			ReadonlyRootFilesystem: aws.Bool(true),
 			Privileged:             aws.Bool(false),
@@ -310,7 +310,7 @@ func TestRegisterTaskDefinition_RuntimeFieldsOmittedWhenUnset(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 		}},
 	}, testAccountID)
 	require.NoError(t, err)
@@ -347,7 +347,7 @@ func TestRegisterTaskDefinition_EchoesEmptyCollections(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			Environment:     []*ecs.KeyValuePair{},
 			MountPoints:     []*ecs.MountPoint{},
 			VolumesFrom:     []*ecs.VolumeFrom{},
@@ -386,7 +386,7 @@ func TestRegisterTaskDefinition_FillsAWSDefaults(t *testing.T) {
 		Family:      aws.String("app"),
 		NetworkMode: aws.String("awsvpc"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			PortMappings: []*ecs.PortMapping{{ContainerPort: aws.Int64(80)}},
 		}},
 	}, testAccountID)
@@ -420,7 +420,7 @@ func TestRegisterTaskDefinition_BridgeLeavesHostPortUnset(t *testing.T) {
 		Family:      aws.String("app"),
 		NetworkMode: aws.String("bridge"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			PortMappings: []*ecs.PortMapping{{ContainerPort: aws.Int64(53), Protocol: aws.String("udp")}},
 		}},
 	}, testAccountID)
@@ -435,7 +435,7 @@ func TestRegisterTaskDefinition_AcceptsEmptyRuntimeFields(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			MountPoints:    []*ecs.MountPoint{},
 			VolumesFrom:    []*ecs.VolumeFrom{},
 			SystemControls: []*ecs.SystemControl{},
@@ -455,7 +455,7 @@ func TestRegisterTaskDefinition_RejectsMountPoints(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			MountPoints: []*ecs.MountPoint{{
 				SourceVolume: aws.String("data"), ContainerPath: aws.String("/data"),
 			}},
@@ -476,7 +476,7 @@ func TestRegisterTaskDefinition_RejectsVolumesFrom(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			VolumesFrom: []*ecs.VolumeFrom{{SourceContainer: aws.String("other")}},
 		}},
 	}, testAccountID)
@@ -494,7 +494,7 @@ func TestRegisterTaskDefinition_RejectsLinuxParametersDevices(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			LinuxParameters: &ecs.LinuxParameters{
 				Devices: []*ecs.Device{{HostPath: aws.String("/dev/fuse")}},
 			},
@@ -514,7 +514,7 @@ func TestRegisterTaskDefinition_RejectsInitProcessEnabled(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			LinuxParameters: &ecs.LinuxParameters{InitProcessEnabled: aws.Bool(true)},
 		}},
 	}, testAccountID)
@@ -533,7 +533,7 @@ func TestRegisterTaskDefinition_RejectsSharedMemoryAndTmpfs(t *testing.T) {
 	_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("shm"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			LinuxParameters: &ecs.LinuxParameters{SharedMemorySize: aws.Int64(64)},
 		}},
 	}, testAccountID)
@@ -545,7 +545,7 @@ func TestRegisterTaskDefinition_RejectsSharedMemoryAndTmpfs(t *testing.T) {
 	_, err = svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("tmpfs"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			LinuxParameters: &ecs.LinuxParameters{
 				Tmpfs: []*ecs.Tmpfs{{ContainerPath: aws.String("/tmp"), Size: aws.Int64(64)}},
 			},
@@ -567,7 +567,7 @@ func TestRunTask_AssignCarriesRuntimeFields(t *testing.T) {
 	_, err = svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
 		Family: aws.String("app"),
 		ContainerDefinitions: []*ecs.ContainerDefinition{{
-			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true),
+			Name: aws.String("app"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true), Memory: aws.Int64(256),
 			User:                   aws.String("1000"),
 			ReadonlyRootFilesystem: aws.Bool(true),
 			Privileged:             aws.Bool(true),
@@ -608,3 +608,42 @@ func TestRunTask_AssignCarriesRuntimeFields(t *testing.T) {
 	assert.Equal(t, "net.core.somaxconn", ac.SystemControls[0].Namespace)
 	assert.Equal(t, []string{"SYS_PTRACE"}, ac.CapAdd)
 }
+
+// AWS rejects a container with no image before checking its memory, and
+// without task-level memory requires memory or memoryReservation per container.
+func TestRegisterTaskDefinition_RequiresImageAndMemory(t *testing.T) {
+	tests := []struct {
+		name       string
+		taskMemory *string
+		container  *ecs.ContainerDefinition
+		wantErr    string
+	}{
+		{"no image or memory", nil, &ecs.ContainerDefinition{Name: aws.String("web")},
+			"Container.image should not be null or empty."},
+		{"empty image", aws.String("512"), &ecs.ContainerDefinition{Name: aws.String("web"), Image: aws.String("")},
+			"Container.image should not be null or empty."},
+		{"no memory", nil, &ecs.ContainerDefinition{Name: aws.String("web"), Image: aws.String("registry/web:1")},
+			"Invalid setting for container 'web'. At least one of 'memory' or 'memoryReservation' must be specified."},
+		{"memoryReservation only", nil, &ecs.ContainerDefinition{
+			Name: aws.String("web"), Image: aws.String("registry/web:1"), MemoryReservation: aws.Int64(128)}, ""},
+		{"task-level memory", aws.String("512"), &ecs.ContainerDefinition{
+			Name: aws.String("web"), Image: aws.String("registry/web:1")}, ""},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			svc, _ := newTestService(t)
+			_, err := svc.RegisterTaskDefinition(context.Background(), &ecs.RegisterTaskDefinitionInput{
+				Family: aws.String("app"), Memory: tc.taskMemory,
+				ContainerDefinitions: []*ecs.ContainerDefinition{tc.container},
+			}, testAccountID)
+			if tc.wantErr == "" {
+				require.NoError(t, err)
+				return
+			}
+			code, msg, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok)
+			assert.Equal(t, awserrors.ErrorECSClientException, code)
+			assert.Equal(t, tc.wantErr, msg)
+		})
+	}
+}

spinifex/domains/ecs/tasks.go

@@ -237,6 +237,9 @@ func (s *Service) newTaskRecord(accountID, cluster, taskID string, td *taskdefin
 		GPU:                  gpu,
 		CreatedAt:            now,
 	}
+	if td.TaskRoleArn != "" {
+		rec.CredID = uuid.NewV4().String()
+	}
 	for _, c := range td.Containers {
 		rec.Containers = append(rec.Containers, ContainerState{Name: c.Name, Status: TaskStatusPending})
 	}
@@ -257,6 +260,7 @@ func (s *Service) publishAssign(ctx context.Context, kv jetstream.KeyValue, acco
 		TaskDefFamily:    td.Family,
 		TaskDefRevision:  td.Revision,
 		TaskRoleARN:      td.TaskRoleArn,
+		CredID:           rec.CredID,
 		ExecutionRoleARN: td.ExecutionRoleArn,
 		ENIID:            rec.ENIID,
 		ENIMacAddress:    rec.ENIMacAddress,
@@ -303,6 +307,14 @@ func (s *Service) ListTasks(ctx context.Context, input *ecs.ListTasksInput, acco
 	if err != nil {
 		return nil, err
 	}
+	var clusterRec ClusterRecord
+	found, err := getJSON(ctx, kv, ClusterMetaKey(cluster), &clusterRec)
+	if err != nil {
+		return nil, err
+	}
+	if !found {
+		return nil, errors.New(awserrors.ErrorECSClusterNotFound)
+	}
 	keys, err := keysWithPrefix(ctx, kv, TasksPrefix(cluster))
 	if err != nil {
 		return nil, err

spinifex/domains/eks/addon/catalog.go

@@ -4,8 +4,14 @@ import (
 	"fmt"
 	"maps"
 	"slices"
+	"strconv"
+	"strings"
 )
 
+// firstBuild is the EKS build number of every bundled version; it moves per
+// version when Spinifex rebuilds a bundle.
+const firstBuild = 1
+
 // NvidiaDevicePlugin is staged by a GPU node group's launch (EnsureGPUDevicePlugin),
 // never user-requested directly.
 const NvidiaDevicePlugin = "nvidia-device-plugin"
@@ -18,6 +24,11 @@ type Spec struct {
 	// Versions lists supported versions newest-first; [0] is the default.
 	Versions       []string
 	DefaultVersion string
+	// Builds maps each upstream version to its EKS build number, the n in
+	// v<upstream>-eksbuild.<n>.
+	Builds map[string]int
+	// Type is the AWS add-on type reported by DescribeAddonVersions.
+	Type string
 	// RequiresIRSA indicates the add-on needs an IAM role for service-account binding.
 	RequiresIRSA bool
 	Description  string
@@ -28,26 +39,26 @@ type Spec struct {
 
 // catalog is the bundled add-on registry. Keep newest version first per slice.
 var catalog = buildCatalog(
-	newSpec("aws-load-balancer-controller", true,
+	typed("networking", newSpec("aws-load-balancer-controller", true,
 		"Provisions ELBv2 load balancers for Kubernetes Service/Ingress resources.",
-		"2.11.0"),
-	newSpec("argocd", false,
+		"2.11.0")),
+	typed("gitops", newSpec("argocd", false,
 		"Declarative GitOps continuous delivery for Kubernetes.",
-		"3.0.23"),
-	newSpec("aws-ebs-csi-driver", true,
+		"3.0.23")),
+	typed("storage", newSpec("aws-ebs-csi-driver", true,
 		"Container Storage Interface driver for Amazon EBS (Viperblock) volumes.",
-		"1.40.1"),
+		"1.40.1")),
 	// Staged automatically for GPU node groups, so hidden from the public
 	// catalogue like spinifex-noop but still creatable by name.
-	hidden(newSpec(NvidiaDevicePlugin, false,
+	hidden(typed("accelerator", newSpec(NvidiaDevicePlugin, false,
 		"NVIDIA device plugin exposing nvidia.com/gpu allocatable via CDI on GPU-tainted nodes.",
-		"0.17.4")),
+		"0.17.4"))),
 	// spinifex-noop is the delivery-transport fixture: a trivial bundle
 	// (Namespace + ConfigMap) the add-on e2e uses to prove stage, render,
 	// auto-deploy, ACTIVE and delete round-trip without a real workload.
-	hidden(newSpec("spinifex-noop", false,
+	hidden(typed("fixture", newSpec("spinifex-noop", false,
 		"No-op delivery-transport fixture (Namespace + ConfigMap).",
-		"0.1.0")),
+		"0.1.0"))),
 )
 
 // hidden marks a spec as internal: still creatable, but absent from the
@@ -57,6 +68,12 @@ func hidden(s Spec) Spec {
 	return s
 }
 
+// typed sets the AWS add-on type reported by DescribeAddonVersions.
+func typed(addonType string, s Spec) Spec {
+	s.Type = addonType
+	return s
+}
+
 // newSpec builds a spec with the first version as its default.
 func newSpec(name string, requiresIRSA bool, description string, versions ...string) Spec {
 	spec := Spec{
@@ -68,6 +85,10 @@ func newSpec(name string, requiresIRSA bool, description string, versions ...str
 	if len(versions) > 0 {
 		spec.DefaultVersion = versions[0]
 	}
+	spec.Builds = make(map[string]int, len(versions))
+	for _, v := range versions {
+		spec.Builds[v] = firstBuild
+	}
 	return spec
 }
 
@@ -111,6 +132,50 @@ func (s Spec) SupportsVersion(version string) bool {
 	return slices.Contains(s.Versions, version)
 }
 
+// awsForm renders a catalog version as v<upstream>-eksbuild.<build>.
+func (s Spec) awsForm(upstream string) string {
+	build, ok := s.Builds[upstream]
+	if !ok {
+		build = firstBuild
+	}
+	return "v" + upstream + "-eksbuild." + strconv.Itoa(build)
+}
+
+// AWSVersion returns the AWS form of a stored version. A version the catalogue
+// does not list, or one already in AWS form, is returned unchanged.
+func (s Spec) AWSVersion(stored string) string {
+	if !slices.Contains(s.Versions, stored) {
+		return stored
+	}
+	return s.awsForm(stored)
+}
+
+// AWSVersions returns every supported version in AWS form, newest first.
+func (s Spec) AWSVersions() []string {
+	out := make([]string, len(s.Versions))
+	for i, v := range s.Versions {
+		out[i] = s.awsForm(v)
+	}
+	return out
+}
+
+// ParseVersion maps a client-supplied version, in AWS form or bare upstream
+// form, to the upstream version stored on the record. Nothing else is accepted.
+func (s Spec) ParseVersion(input string) (string, bool) {
+	if slices.Contains(s.Versions, input) {
+		return input, true
+	}
+	if !strings.HasPrefix(input, "v") {
+		return "", false
+	}
+	for _, v := range s.Versions {
+		if input == s.awsForm(v) {
+			return v, true
+		}
+	}
+	return "", false
+}
+
 // Specs returns every catalogue entry, hidden ones included, sorted by name.
 func Specs() []Spec {
 	names := slices.Sorted(maps.Keys(catalog))

spinifex/domains/eks/addon/catalog_test.go

@@ -100,3 +100,78 @@ func TestValidateCatalog_ChecksTheBundledCatalog(t *testing.T) {
 	t.Cleanup(func() { catalog = previous })
 	require.EqualError(t, ValidateCatalog(), `add-on "broken" has no versions`)
 }
+
+// buildSpec is a two-version spec whose newest version is on EKS build 2, so
+// a test cannot pass by hardcoding build 1.
+func buildSpec() Spec {
+	return Spec{
+		Name:     "fixture",
+		Versions: []string{"2.0.0", "1.0.0"},
+		Builds:   map[string]int{"2.0.0": 2, "1.0.0": 1},
+	}
+}
+
+func TestSpec_AWSVersionUsesTheCatalogBuildNumber(t *testing.T) {
+	t.Parallel()
+	spec := buildSpec()
+	assert.Equal(t, "v2.0.0-eksbuild.2", spec.AWSVersion("2.0.0"))
+	assert.Equal(t, "v1.0.0-eksbuild.1", spec.AWSVersion("1.0.0"))
+}
+
+func TestSpec_AWSVersionIsIdempotent(t *testing.T) {
+	t.Parallel()
+	spec := buildSpec()
+	assert.Equal(t, "v2.0.0-eksbuild.2", spec.AWSVersion("v2.0.0-eksbuild.2"),
+		"a record already holding the AWS form must not be wrapped twice")
+}
+
+func TestSpec_AWSVersionsAreNewestFirst(t *testing.T) {
+	t.Parallel()
+	assert.Equal(t, []string{"v2.0.0-eksbuild.2", "v1.0.0-eksbuild.1"}, buildSpec().AWSVersions())
+}
+
+func TestSpec_ParseVersionAcceptsOnlyTheExactForms(t *testing.T) {
+	t.Parallel()
+	spec := buildSpec()
+	for input, want := range map[string]string{
+		"v2.0.0-eksbuild.2": "2.0.0",
+		"2.0.0":             "2.0.0",
+		"v1.0.0-eksbuild.1": "1.0.0",
+		"1.0.0":             "1.0.0",
+	} {
+		got, ok := spec.ParseVersion(input)
+		assert.True(t, ok, "%q must be accepted", input)
+		assert.Equal(t, want, got, "%q", input)
+	}
+}
+
+func TestSpec_ParseVersionRefusesEverythingElse(t *testing.T) {
+	t.Parallel()
+	spec := buildSpec()
+	for _, input := range []string{
+		"v2.0.0-eksbuild.1", // wrong build for 2.0.0
+		"v2.0.0-eksbuild.3",
+		"v1.0.0-eksbuild.2", // wrong build for 1.0.0
+		"v2.0.0",            // missing build
+		"2.0",               // partial
+		"v2.0.0-eksbuild.",
+		"2.0.0-eksbuild.2", // missing v
+		"V2.0.0-eksbuild.2",
+		" 2.0.0",
+		"9.9.9",
+		"",
+	} {
+		_, ok := spec.ParseVersion(input)
+		assert.False(t, ok, "%q must be refused", input)
+	}
+}
+
+func TestCatalog_EveryVersionHasAnExplicitBuild(t *testing.T) {
+	t.Parallel()
+	for _, spec := range Specs() {
+		for _, v := range spec.Versions {
+			assert.Equal(t, 1, spec.Builds[v], "%s %s needs an explicit build number", spec.Name, v)
+			assert.Equal(t, "v"+v+"-eksbuild.1", spec.AWSVersion(v), "%s %s", spec.Name, v)
+		}
+	}
+}

spinifex/domains/eks/awsapi/addon_versions_query_test.go

@@ -0,0 +1,105 @@
+package awsapi
+
+import (
+	"context"
+	"encoding/json"
+	"sync"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// captureDescribeAddonVersions answers eks.DescribeAddonVersions and returns a
+// function reading the last input the gateway sent to the service.
+func captureDescribeAddonVersions(t *testing.T, nc *nats.Conn) func() eks.DescribeAddonVersionsInput {
+	t.Helper()
+	var (
+		mu   sync.Mutex
+		last eks.DescribeAddonVersionsInput
+	)
+	sub, err := nc.Subscribe("eks.DescribeAddonVersions", func(m *nats.Msg) {
+		mu.Lock()
+		last = eks.DescribeAddonVersionsInput{}
+		_ = json.Unmarshal(m.Data, &last)
+		mu.Unlock()
+		_ = m.Respond([]byte(`{}`))
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	return func() eks.DescribeAddonVersionsInput {
+		mu.Lock()
+		defer mu.Unlock()
+		return last
+	}
+}
+
+// The gateway folds a GET query string into the body as a JSON object of
+// string lists; the model binds every field below from the query.
+func TestDescribeAddonVersions_PassesEveryModelQueryParameter(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	got := captureDescribeAddonVersions(t, nc)
+
+	query := []byte(`{"addonName":["aws-ebs-csi-driver"],"kubernetesVersion":["1.32"],` +
+		`"maxResults":["2"],"nextToken":["tok"],"types":["storage","networking"],` +
+		`"publishers":["eks"],"owners":["aws"]}`)
+	_, err := DescribeAddonVersions(context.Background(), nc, acct, query)
+	require.NoError(t, err)
+
+	in := got()
+	assert.Equal(t, "aws-ebs-csi-driver", aws.StringValue(in.AddonName))
+	assert.Equal(t, "1.32", aws.StringValue(in.KubernetesVersion))
+	assert.Equal(t, int64(2), aws.Int64Value(in.MaxResults))
+	assert.Equal(t, "tok", aws.StringValue(in.NextToken))
+	assert.Equal(t, []string{"storage", "networking"}, aws.StringValueSlice(in.Types))
+	assert.Equal(t, []string{"eks"}, aws.StringValueSlice(in.Publishers))
+	assert.Equal(t, []string{"aws"}, aws.StringValueSlice(in.Owners))
+}
+
+func TestDescribeAddonVersions_NoQueryLeavesEveryFilterUnset(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	got := captureDescribeAddonVersions(t, nc)
+
+	_, err := DescribeAddonVersions(context.Background(), nc, acct, nil)
+	require.NoError(t, err)
+
+	in := got()
+	assert.Nil(t, in.AddonName)
+	assert.Nil(t, in.KubernetesVersion)
+	assert.Nil(t, in.MaxResults)
+	assert.Nil(t, in.NextToken)
+	assert.Empty(t, in.Types)
+}
+
+func TestDescribeAddonVersions_RefusesMalformedQuery(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	captureDescribeAddonVersions(t, nc)
+
+	_, err := DescribeAddonVersions(context.Background(), nc, acct, []byte(`{"maxResults":["many"]}`))
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "maxResults")
+
+	_, err = DescribeAddonVersions(context.Background(), nc, acct, []byte(`not json`))
+	require.Error(t, err)
+}
+
+// The route must hand the folded query to the wrapper, not drop it.
+func TestDescribeAddonVersionsRoute_PassesTheQuery(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	got := captureDescribeAddonVersions(t, nc)
+
+	for _, r := range routes {
+		if r.Action != "DescribeAddonVersions" {
+			continue
+		}
+		_, err := r.Handler(context.Background(), nc, acct, "", nil, []byte(`{"addonName":["argocd"]}`))
+		require.NoError(t, err)
+		assert.Equal(t, "argocd", aws.StringValue(got().AddonName))
+		return
+	}
+	t.Fatal("no DescribeAddonVersions route")
+}

spinifex/domains/eks/awsapi/addons.go

@@ -16,8 +16,23 @@ func ListAddons(ctx context.Context, natsConn *nats.Conn, accountID, cluster str
 }
 
 // DescribeAddonVersions — GET /addons/supported-versions.
-func DescribeAddonVersions(ctx context.Context, natsConn *nats.Conn, accountID string) (*eks.DescribeAddonVersionsOutput, error) {
-	return ekscluster.NewNATSEKSService(natsConn).DescribeAddonVersions(ctx, &eks.DescribeAddonVersionsInput{}, accountID)
+func DescribeAddonVersions(ctx context.Context, natsConn *nats.Conn, accountID string, query []byte) (*eks.DescribeAddonVersionsOutput, error) {
+	q, err := parseListQuery(query)
+	if err != nil {
+		return nil, err
+	}
+	input := &eks.DescribeAddonVersionsInput{
+		AddonName:         q.str("addonName"),
+		KubernetesVersion: q.str("kubernetesVersion"),
+		NextToken:         q.str("nextToken"),
+		Types:             q.strs("types"),
+		Publishers:        q.strs("publishers"),
+		Owners:            q.strs("owners"),
+	}
+	if input.MaxResults, err = q.int64("maxResults"); err != nil {
+		return nil, err
+	}
+	return ekscluster.NewNATSEKSService(natsConn).DescribeAddonVersions(ctx, input, accountID)
 }
 
 // CreateAddon — POST /clusters/{name}/addons.

spinifex/domains/eks/awsapi/query.go

@@ -0,0 +1,61 @@
+package awsapi
+
+import (
+	"encoding/json"
+	"strconv"
+
+	"github.com/aws/aws-sdk-go/aws"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// listQuery holds the query-string inputs of a GET action, which the gateway
+// folds into the body as a JSON object of string lists.
+type listQuery map[string][]string
+
+// parseListQuery decodes the folded query of a GET action. An empty body means
+// the request carried no query string.
+func parseListQuery(body []byte) (listQuery, error) {
+	q := listQuery{}
+	if len(body) == 0 {
+		return q, nil
+	}
+	if err := json.Unmarshal(body, (*map[string][]string)(&q)); err != nil {
+		return nil, awserrors.Errorf(awserrors.ErrorEKSInvalidParameter, "malformed query string")
+	}
+	return q, nil
+}
+
+// str returns the first value of name, or nil when it is absent or empty.
+func (q listQuery) str(name string) *string {
+	if v := q[name]; len(v) > 0 && v[0] != "" {
+		return aws.String(v[0])
+	}
+	return nil
+}
+
+// strs returns every non-empty value of name, or nil when there are none.
+func (q listQuery) strs(name string) []*string {
+	var out []string
+	for _, v := range q[name] {
+		if v != "" {
+			out = append(out, v)
+		}
+	}
+	if len(out) == 0 {
+		return nil
+	}
+	return aws.StringSlice(out)
+}
+
+// int64 returns the first value of name as an integer, or nil when absent.
+func (q listQuery) int64(name string) (*int64, error) {
+	s := q.str(name)
+	if s == nil {
+		return nil, nil
+	}
+	n, err := strconv.ParseInt(*s, 10, 64)
+	if err != nil {
+		return nil, awserrors.Errorf(awserrors.ErrorEKSInvalidParameter, "%s must be an integer", name)
+	}
+	return aws.Int64(n), nil
+}

spinifex/domains/eks/awsapi/registration.go

@@ -82,17 +82,39 @@ func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 	if err == nil {
 		return nil
 	}
-	if action, _, _, ok := router.Lookup(inv.Request.Method, inv.Request.URL.EscapedPath()); ok && IsInternalAction(action) {
+	action, _, _, routed := router.Lookup(inv.Request.Method, inv.Request.URL.EscapedPath())
+	if routed && IsInternalAction(action) {
 		return err
 	}
 	code, msg, ok := awserrors.ResolveErrorDetail(err)
 	if !ok || code != awserrors.ErrorInvalidParameterValue {
 		return err
 	}
-	if msg == "" {
-		return errors.New(awserrors.ErrorEKSInvalidParameter)
+	wire := awserrors.ErrorEKSInvalidParameter
+	if override, ok := parameterRefusalCodes[action]; ok {
+		wire = override
+		if msg == "" {
+			msg = "a parameter of the request is not valid"
+		}
 	}
-	return awserrors.Errorf(awserrors.ErrorEKSInvalidParameter, "%s", msg)
+	if msg == "" {
+		return errors.New(wire)
+	}
+	return awserrors.Errorf(wire, "%s", msg)
+}
+
+// parameterRefusalCodes holds the operations whose model does not declare
+// InvalidParameterException, with the declared code a refused parameter uses.
+var parameterRefusalCodes = map[string]string{
+	"DescribeCluster":              awserrors.ErrorEKSClient,
+	"DeleteCluster":                awserrors.ErrorEKSClient,
+	"DescribeAccessEntry":          awserrors.ErrorEKSInvalidRequest,
+	"DeleteAccessEntry":            awserrors.ErrorEKSInvalidRequest,
+	"DisassociateAccessPolicy":     awserrors.ErrorEKSInvalidRequest,
+	"ListAssociatedAccessPolicies": awserrors.ErrorEKSInvalidRequest,
+	"TagResource":                  awserrors.ErrorEKSBadRequest,
+	"UntagResource":                awserrors.ErrorEKSBadRequest,
+	"ListTagsForResource":          awserrors.ErrorEKSBadRequest,
 }
 
 // serve resolves the route, authorizes the request, then runs the action.

spinifex/domains/eks/awsapi/registration_error_codes_test.go

@@ -0,0 +1,98 @@
+//test:in-package — routes is unexported, and the table must cover exactly the
+// actions the registration serves.
+
+package awsapi
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/mulgadc/bluebottle/pkg/iampolicy"
+	"github.com/mulgadc/spinifex/internal/awsmodel"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// routePath fills a route pattern with placeholder segments. The tag routes
+// take an ARN, which has to be escaped to stay one path segment.
+func routePath(pattern string) string {
+	arn := url.PathEscape("arn:aws:eks:us-east-1:111122223333:cluster/c")
+	path := strings.ReplaceAll(pattern, "*", arn)
+	for {
+		start := strings.Index(path, "{")
+		if start < 0 {
+			return path
+		}
+		end := strings.Index(path[start:], "}")
+		path = path[:start] + "x" + path[start+end+1:]
+	}
+}
+
+// A parameter refusal must reach the wire under a code the operation declares
+// in the EKS model, or SDK clients cannot match it to a typed error.
+func TestRegistration_ParameterRefusalCodeIsDeclaredByOperation(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	sub, err := nc.Subscribe("eks.>", func(m *nats.Msg) {
+		if m.Reply != "" {
+			_ = m.Respond(awserrors.GenerateErrorPayload(awserrors.ErrorInvalidParameterValue))
+		}
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+	model, err := awsmodel.Load(awsmodel.EKS)
+	require.NoError(t, err)
+	registration := NewRegistration(Deps{NATS: nc})
+
+	for _, route := range routes {
+		if IsInternalAction(route.Action) {
+			continue
+		}
+		// The handler takes no input and the model declares no client error.
+		if route.Action == "ListAccessPolicies" {
+			continue
+		}
+		t.Run(route.Action, func(t *testing.T) {
+			operation, ok := model.Operation(route.Action)
+			require.True(t, ok, "%s is not an operation in the EKS model", route.Action)
+			declared := make([]string, 0, len(operation.Errors))
+			for _, ref := range operation.Errors {
+				declared = append(declared, ref.Shape)
+			}
+
+			req := httptest.NewRequest(route.Method, routePath(route.Pattern), strings.NewReader(`{}`))
+			inv := dispatch.Invocation{
+				Request:   req,
+				AccountID: "111122223333",
+				Region:    "us-east-1",
+				Authorize: func(string, string, []string, iampolicy.ConditionKeys) error { return nil },
+				Actor: func() (dispatch.Actor, error) {
+					return dispatch.Actor{AccountID: "111122223333", PrincipalType: "user", UserName: "alice"}, nil
+				},
+			}
+
+			err := registration.Dispatch(httptest.NewRecorder(), inv)
+
+			require.Error(t, err)
+			code, ok := awserrors.ResolveErrorCode(err)
+			require.True(t, ok, "unresolvable error code: %v", err)
+			assert.Contains(t, declared, code,
+				"%s %s refuses a parameter with %s, which the operation does not declare", route.Method, route.Pattern, code)
+		})
+	}
+}
+
+// The access operations declare InvalidRequestException, not a parameter code.
+func TestEKSInvalidRequestCodeIsRegistered(t *testing.T) {
+	assert.Equal(t, "InvalidRequestException", awserrors.ErrorEKSInvalidRequest)
+	assert.Equal(t, http.StatusBadRequest, awserrors.HTTPStatusForError(awserrors.Errorf(awserrors.ErrorEKSInvalidRequest, "x")))
+	assert.Equal(t, "BadRequestException", awserrors.ErrorEKSBadRequest)
+	assert.Equal(t, http.StatusBadRequest, awserrors.HTTPStatusForError(awserrors.Errorf(awserrors.ErrorEKSBadRequest, "x")))
+}

spinifex/domains/eks/awsapi/routes.go

@@ -131,7 +131,7 @@ var routes = []route{
 	// Addons
 	{Method: "GET", Pattern: "/addons/supported-versions", Action: "DescribeAddonVersions",
 		Handler: func(ctx context.Context, nc *nats.Conn, acct, callerARN string, p []string, b []byte) (any, error) {
-			return DescribeAddonVersions(ctx, nc, acct)
+			return DescribeAddonVersions(ctx, nc, acct, b)
 		}},
 	{Method: "GET", Pattern: "/clusters/{clusterName}/addons", Action: "ListAddons",
 		Handler: func(ctx context.Context, nc *nats.Conn, acct, callerARN string, p []string, b []byte) (any, error) {

spinifex/domains/eks/awsapi/wrappers_test.go

@@ -176,7 +176,7 @@ func TestGatewayWrappers_Addons(t *testing.T) {
 	require.NoError(t, err)
 	assert.NotNil(t, out1)
 
-	out2, err := DescribeAddonVersions(context.Background(), nc, acct)
+	out2, err := DescribeAddonVersions(context.Background(), nc, acct, nil)
 	require.NoError(t, err)
 	assert.NotNil(t, out2)
 

spinifex/domains/eks/cluster/addon_describe_versions_test.go

@@ -0,0 +1,171 @@
+package cluster
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/mulgadc/spinifex/spinifex/domains/eks/addon"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func describeVersions(t *testing.T, svc *EKSServiceImpl, in *eks.DescribeAddonVersionsInput) *eks.DescribeAddonVersionsOutput {
+	t.Helper()
+	out, err := svc.DescribeAddonVersions(context.Background(), in, testAccountID)
+	require.NoError(t, err)
+	return out
+}
+
+// The terraform aws_eks_addon_version data source asks for one add-on at the
+// cluster's Kubernetes version, then takes the version whose compatibility for
+// that cluster version is the default (or, with most_recent, the first one).
+func TestDescribeAddonVersions_TerraformProviderCall(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	out := describeVersions(t, svc, &eks.DescribeAddonVersionsInput{
+		AddonName:         aws.String(csiAddon),
+		KubernetesVersion: aws.String(defaultK8sVersion),
+	})
+	require.Len(t, out.Addons, 1)
+	info := out.Addons[0]
+	assert.Equal(t, csiAddon, aws.StringValue(info.AddonName))
+	assert.Nil(t, out.NextToken)
+
+	spec, ok := addon.Lookup(csiAddon)
+	require.True(t, ok)
+	require.Equal(t, spec.AWSVersions(), versionStrings(info), "newest first, AWS form")
+
+	defaults := 0
+	for _, v := range info.AddonVersions {
+		require.Len(t, v.Compatibilities, 1, "one entry, for the requested cluster version")
+		c := v.Compatibilities[0]
+		assert.Equal(t, defaultK8sVersion, aws.StringValue(c.ClusterVersion))
+		assert.Equal(t, []string{"*"}, aws.StringValueSlice(c.PlatformVersions))
+		if aws.BoolValue(c.DefaultVersion) {
+			defaults++
+			assert.Equal(t, spec.AWSVersion(spec.DefaultVersion), aws.StringValue(v.AddonVersion))
+		}
+	}
+	assert.Equal(t, 1, defaults, "exactly one defaultVersion true")
+}
+
+// AWS observed (EKS-01): compatibilities holds only the requested version. The
+// response for a version no add-on supports is not captured; this pins the
+// shape #61 chose, an empty list with no error.
+func TestDescribeAddonVersions_UnsupportedKubernetesVersionIsEmpty(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	out := describeVersions(t, svc, &eks.DescribeAddonVersionsInput{
+		AddonName:         aws.String(csiAddon),
+		KubernetesVersion: aws.String("1.29"),
+	})
+	assert.Empty(t, out.Addons)
+	assert.Nil(t, out.NextToken)
+}
+
+func TestDescribeAddonVersions_FiltersByNameAndHidesFixturesUnlessNamed(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	for _, name := range []string{csiAddon, "argocd", albController} {
+		out := describeVersions(t, svc, &eks.DescribeAddonVersionsInput{AddonName: aws.String(name)})
+		require.Len(t, out.Addons, 1, name)
+		assert.Equal(t, name, aws.StringValue(out.Addons[0].AddonName))
+	}
+	assert.Empty(t, describeVersions(t, svc, &eks.DescribeAddonVersionsInput{AddonName: aws.String("vpc-cni")}).Addons)
+}
+
+// maxResults counts add-ons, not versions; a page boundary never splits an
+// add-on's version list, and the last page carries no token.
+func TestDescribeAddonVersions_PagesByAddon(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	all := describeVersions(t, svc, &eks.DescribeAddonVersionsInput{})
+	require.Greater(t, len(all.Addons), 1)
+	assert.Nil(t, all.NextToken)
+
+	var names []string
+	in := &eks.DescribeAddonVersionsInput{MaxResults: aws.Int64(1)}
+	for range len(all.Addons) {
+		out := describeVersions(t, svc, in)
+		require.Len(t, out.Addons, 1)
+		names = append(names, aws.StringValue(out.Addons[0].AddonName))
+		assert.Equal(t, versionStrings(all.Addons[len(names)-1]), versionStrings(out.Addons[0]))
+		in.NextToken = out.NextToken
+	}
+	assert.Nil(t, in.NextToken, "last page carries no token")
+	var want []string
+	for _, a := range all.Addons {
+		want = append(want, aws.StringValue(a.AddonName))
+	}
+	assert.Equal(t, want, names)
+}
+
+// Every add-on carries the descriptive fields the data source and console read.
+func TestDescribeAddonVersions_CarriesAddonAndVersionMetadata(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	for _, a := range describeVersions(t, svc, &eks.DescribeAddonVersionsInput{}).Addons {
+		name := aws.StringValue(a.AddonName)
+		assert.NotEmpty(t, aws.StringValue(a.Type), name)
+		assert.NotEmpty(t, aws.StringValue(a.Owner), name)
+		assert.NotEmpty(t, aws.StringValue(a.Publisher), name)
+		assert.Nil(t, a.MarketplaceInformation, name)
+		for _, v := range a.AddonVersions {
+			assert.NotEmpty(t, v.Architecture, name)
+			assert.NotNil(t, v.RequiresConfiguration, name)
+			assert.NotNil(t, v.RequiresIamPermissions, name)
+		}
+	}
+}
+
+// types, publishers and owners are model filters; each keeps add-ons whose
+// field matches one of the given values.
+func TestDescribeAddonVersions_FiltersByTypePublisherOwner(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	first := describeVersions(t, svc, &eks.DescribeAddonVersionsInput{}).Addons[0]
+	cases := []struct {
+		name  string
+		match func(*eks.DescribeAddonVersionsInput, []*string)
+		value *string
+	}{
+		{"types", func(in *eks.DescribeAddonVersionsInput, v []*string) { in.Types = v }, first.Type},
+		{"publishers", func(in *eks.DescribeAddonVersionsInput, v []*string) { in.Publishers = v }, first.Publisher},
+		{"owners", func(in *eks.DescribeAddonVersionsInput, v []*string) { in.Owners = v }, first.Owner},
+	}
+	for _, c := range cases {
+		hit := &eks.DescribeAddonVersionsInput{}
+		c.match(hit, aws.StringSlice([]string{"nobody", aws.StringValue(c.value)}))
+		out := describeVersions(t, svc, hit)
+		require.NotEmpty(t, out.Addons, c.name)
+		assert.Contains(t, addonNames(out), aws.StringValue(first.AddonName), c.name)
+
+		miss := &eks.DescribeAddonVersionsInput{}
+		c.match(miss, aws.StringSlice([]string{"nobody"}))
+		assert.Empty(t, describeVersions(t, svc, miss).Addons, c.name)
+	}
+}
+
+func versionStrings(a *eks.AddonInfo) []string {
+	out := make([]string, 0, len(a.AddonVersions))
+	for _, v := range a.AddonVersions {
+		out = append(out, aws.StringValue(v.AddonVersion))
+	}
+	return out
+}
+
+func addonNames(o *eks.DescribeAddonVersionsOutput) []string {
+	out := make([]string, 0, len(o.Addons))
+	for _, a := range o.Addons {
+		out = append(out, aws.StringValue(a.AddonName))
+	}
+	return out
+}

spinifex/domains/eks/cluster/addon_version_format_test.go

@@ -0,0 +1,220 @@
+package cluster
+
+import (
+	"bytes"
+	"context"
+	"log/slog"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/mulgadc/spinifex/spinifex/domains/eks/addon"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	csiAddon       = "aws-ebs-csi-driver"
+	csiBare        = "1.40.1"
+	csiAWSVersion  = "v1.40.1-eksbuild.1"
+	csiWrongBuild  = "v1.40.1-eksbuild.2"
+	versionParam   = "addonVersion"
+	notBeforeAWSOK = "an AWS-format version must be accepted"
+)
+
+func TestDescribeAddonVersions_ReturnsAWSFormNewestFirst(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+
+	out, err := svc.DescribeAddonVersions(context.Background(), &eks.DescribeAddonVersionsInput{}, testAccountID)
+	require.NoError(t, err)
+	require.NotEmpty(t, out.Addons)
+	for _, a := range out.Addons {
+		spec, ok := addon.Lookup(aws.StringValue(a.AddonName))
+		require.True(t, ok)
+		require.Len(t, a.AddonVersions, len(spec.Versions))
+		for i, v := range a.AddonVersions {
+			assert.Equal(t, "v"+spec.Versions[i]+"-eksbuild.1", aws.StringValue(v.AddonVersion),
+				"%s version %d", spec.Name, i)
+		}
+	}
+}
+
+func TestCreateAddon_AcceptsBothFormsAndStoresTheBareForm(t *testing.T) {
+	t.Parallel()
+	for name, input := range map[string]string{"aws form": csiAWSVersion, "bare form": csiBare} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			svc, fake := setupAddonService(t)
+
+			out, err := svc.CreateAddon(context.Background(), &eks.CreateAddonInput{
+				ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+				AddonVersion: aws.String(input),
+			}, testAccountID)
+			require.NoError(t, err, notBeforeAWSOK)
+			assert.Equal(t, csiAWSVersion, aws.StringValue(out.Addon.AddonVersion), "response is in AWS form")
+
+			rec, err := addon.Get(t.Context(), acctKVForTest(t, svc), "c1", csiAddon)
+			require.NoError(t, err)
+			assert.Equal(t, csiBare, rec.AddonVersion, "the record keeps the upstream form")
+			require.Len(t, fake.installs, 1)
+			assert.Equal(t, csiBare, fake.installs[0].AddonVersion, "the bundle lookup sees the upstream form")
+		})
+	}
+}
+
+func TestCreateAddon_DefaultVersionResponseIsAWSForm(t *testing.T) {
+	t.Parallel()
+	svc, _ := setupAddonService(t)
+	out, err := svc.CreateAddon(context.Background(), &eks.CreateAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+	}, testAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, csiAWSVersion, aws.StringValue(out.Addon.AddonVersion))
+}
+
+func TestUpdateAddon_AcceptsBothFormsAndStoresTheBareForm(t *testing.T) {
+	t.Parallel()
+	for name, input := range map[string]string{"aws form": csiAWSVersion, "bare form": csiBare} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			svc, fake := setupAddonService(t)
+			_, err := svc.CreateAddon(context.Background(), &eks.CreateAddonInput{
+				ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+			}, testAccountID)
+			require.NoError(t, err)
+
+			_, err = svc.UpdateAddon(context.Background(), &eks.UpdateAddonInput{
+				ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+				AddonVersion: aws.String(input),
+			}, testAccountID)
+			require.NoError(t, err, notBeforeAWSOK)
+
+			rec, err := addon.Get(t.Context(), acctKVForTest(t, svc), "c1", csiAddon)
+			require.NoError(t, err)
+			assert.Equal(t, csiBare, rec.AddonVersion)
+			require.Len(t, fake.installs, 2)
+			assert.Equal(t, csiBare, fake.installs[1].AddonVersion)
+		})
+	}
+}
+
+func TestDescribeAddon_ReturnsAWSFormForAnyStoredForm(t *testing.T) {
+	t.Parallel()
+	for name, stored := range map[string]string{
+		"record stored bare before the change": csiBare,
+		"record already in AWS form":           csiAWSVersion,
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			svc, _ := setupAddonService(t)
+			_, err := svc.addons().Create(context.Background(), acctKVForTest(t, svc), testAccountID, addon.Desired{
+				Cluster: "c1", Name: csiAddon, Version: stored,
+			})
+			require.NoError(t, err)
+
+			desc, err := svc.DescribeAddon(context.Background(), &eks.DescribeAddonInput{
+				ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+			}, testAccountID)
+			require.NoError(t, err)
+			assert.Equal(t, csiAWSVersion, aws.StringValue(desc.Addon.AddonVersion), "no double wrap")
+
+			list, err := svc.ListAddons(context.Background(), &eks.ListAddonsInput{ClusterName: aws.String("c1")}, testAccountID)
+			require.NoError(t, err)
+			assert.Equal(t, []string{csiAddon}, aws.StringValueSlice(list.Addons))
+		})
+	}
+}
+
+func TestUpdateAddon_ResponseAndRecordAfterBareRecord(t *testing.T) {
+	t.Parallel()
+	svc, _ := setupAddonService(t)
+	_, err := svc.addons().Create(context.Background(), acctKVForTest(t, svc), testAccountID, addon.Desired{
+		Cluster: "c1", Name: csiAddon, Version: csiBare,
+	})
+	require.NoError(t, err)
+
+	_, err = svc.UpdateAddon(context.Background(), &eks.UpdateAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+		ConfigurationValues: aws.String(`{"a":1}`),
+	}, testAccountID)
+	require.NoError(t, err)
+	desc, err := svc.DescribeAddon(context.Background(), &eks.DescribeAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+	}, testAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, csiAWSVersion, aws.StringValue(desc.Addon.AddonVersion))
+}
+
+func TestAddonVersionRefusal_NamesTheParameterAndListsSupportedValues(t *testing.T) {
+	t.Parallel()
+	svc, _ := setupAddonService(t)
+	_, err := svc.CreateAddon(context.Background(), &eks.CreateAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String("argocd"),
+	}, testAccountID)
+	require.NoError(t, err)
+
+	refusals := map[string]func() error{
+		"create": func() error {
+			_, e := svc.CreateAddon(context.Background(), &eks.CreateAddonInput{
+				ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+				AddonVersion: aws.String(csiWrongBuild),
+			}, testAccountID)
+			return e
+		},
+		"update": func() error {
+			_, e := svc.UpdateAddon(context.Background(), &eks.UpdateAddonInput{
+				ClusterName: aws.String("c1"), AddonName: aws.String("argocd"),
+				AddonVersion: aws.String("v3.0.23-eksbuild.2"),
+			}, testAccountID)
+			return e
+		},
+	}
+	want := map[string]string{"create": csiAWSVersion, "update": "v3.0.23-eksbuild.1"}
+	for name, refuse := range refusals {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			err := refuse()
+			require.Error(t, err)
+			code, message, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok)
+			assert.Equal(t, awserrors.ErrorInvalidParameterValue, code, "this change leaves the error code alone")
+			assert.Contains(t, message, versionParam)
+			assert.Contains(t, message, want[name], "the supported AWS-form value is listed")
+		})
+	}
+}
+
+func TestAddonBareVersionInput_LogsItsUseAtInfo(t *testing.T) {
+	var buf bytes.Buffer
+	restore := slog.Default()
+	slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelInfo})))
+	t.Cleanup(func() { slog.SetDefault(restore) })
+
+	svc, _ := setupAddonService(t)
+	_, err := svc.CreateAddon(context.Background(), &eks.CreateAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+	}, testAccountID)
+	require.NoError(t, err)
+
+	buf.Reset()
+	_, err = svc.UpdateAddon(context.Background(), &eks.UpdateAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+		AddonVersion: aws.String(csiBare),
+	}, testAccountID)
+	require.NoError(t, err)
+	logged := buf.String()
+	assert.Contains(t, logged, "level=INFO")
+	assert.Contains(t, logged, "bare")
+	assert.Contains(t, logged, csiAddon)
+	assert.Contains(t, logged, "c1")
+
+	buf.Reset()
+	_, err = svc.UpdateAddon(context.Background(), &eks.UpdateAddonInput{
+		ClusterName: aws.String("c1"), AddonName: aws.String(csiAddon),
+		AddonVersion: aws.String(csiAWSVersion),
+	}, testAccountID)
+	require.NoError(t, err, notBeforeAWSOK)
+	assert.NotContains(t, buf.String(), "bare", "the AWS form is the intended form and logs nothing")
+}

spinifex/domains/eks/cluster/addons.go

@@ -2,8 +2,12 @@ package cluster
 
 import (
 	"context"
+	"encoding/base64"
 	"errors"
 	"fmt"
+	"log/slog"
+	"slices"
+	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/eks"
@@ -35,15 +39,20 @@ func (s *EKSServiceImpl) ListAddons(ctx context.Context, input *eks.ListAddonsIn
 	return &eks.ListAddonsOutput{Addons: aws.StringSlice(names)}, nil
 }
 
-// DescribeAddonVersions returns the static add-on catalog, optionally filtered by name.
-func (s *EKSServiceImpl) DescribeAddonVersions(ctx context.Context, input *eks.DescribeAddonVersionsInput, _ string) (*eks.DescribeAddonVersionsOutput, error) {
-	filter := ""
-	if input != nil {
-		filter = aws.StringValue(input.AddonName)
+// DescribeAddonVersions returns the add-on catalogue filtered by name, owner,
+// publisher, type and the Kubernetes version its versions support, one page
+// of add-ons at a time.
+func (s *EKSServiceImpl) DescribeAddonVersions(_ context.Context, input *eks.DescribeAddonVersionsInput, _ string) (*eks.DescribeAddonVersionsOutput, error) {
+	if input == nil {
+		input = &eks.DescribeAddonVersionsInput{}
 	}
-	specs := addon.Specs()
-	out := make([]*eks.AddonInfo, 0, len(specs))
-	for _, spec := range specs {
+	// Every catalogue version supports only the K3s line the node image ships.
+	if v := aws.StringValue(input.KubernetesVersion); v != "" && v != defaultK8sVersion {
+		return &eks.DescribeAddonVersionsOutput{Addons: []*eks.AddonInfo{}}, nil
+	}
+	filter := aws.StringValue(input.AddonName)
+	var matched []addon.Spec
+	for _, spec := range addon.Specs() {
 		if filter != "" && spec.Name != filter {
 			continue
 		}
@@ -52,9 +61,51 @@ func (s *EKSServiceImpl) DescribeAddonVersions(ctx context.Context, input *eks.D
 		if spec.Hidden && filter == "" {
 			continue
 		}
+		if !matchesAny(input.Types, spec.Type) || !matchesAny(input.Owners, addonOwner) ||
+			!matchesAny(input.Publishers, addonPublisher) {
+			continue
+		}
+		matched = append(matched, spec)
+	}
+	page, next, err := pageAddons(matched, input.MaxResults, input.NextToken)
+	if err != nil {
+		return nil, err
+	}
+	out := make([]*eks.AddonInfo, 0, len(page))
+	for _, spec := range page {
 		out = append(out, addonSpecToAWS(spec))
 	}
-	return &eks.DescribeAddonVersionsOutput{Addons: out}, nil
+	return &eks.DescribeAddonVersionsOutput{Addons: out, NextToken: next}, nil
+}
+
+// matchesAny reports whether the filter is empty or lists value.
+func matchesAny(filter []*string, value string) bool {
+	return len(filter) == 0 || slices.Contains(aws.StringValueSlice(filter), value)
+}
+
+// maxAddonPage bounds maxResults, the model's range being 1 to 100.
+const maxAddonPage = 100
+
+// pageAddons returns the add-ons after the token, at most maxResults of them.
+// The token is the base64 name of the last add-on returned, opaque to clients.
+func pageAddons(specs []addon.Spec, maxResults *int64, token *string) ([]addon.Spec, *string, error) {
+	if t := aws.StringValue(token); t != "" {
+		raw, err := base64.RawURLEncoding.DecodeString(t)
+		if err != nil {
+			return nil, nil, awserrors.Errorf(awserrors.ErrorEKSInvalidParameter, "nextToken is not valid")
+		}
+		after := string(raw)
+		specs = slices.DeleteFunc(slices.Clone(specs), func(sp addon.Spec) bool { return sp.Name <= after })
+	}
+	limit := len(specs)
+	if maxResults != nil {
+		limit = min(max(int(*maxResults), 1), maxAddonPage)
+	}
+	if limit >= len(specs) {
+		return specs, nil, nil
+	}
+	next := base64.RawURLEncoding.EncodeToString([]byte(specs[limit-1].Name))
+	return specs[:limit], &next, nil
 }
 
 // validateAddonServiceAccountRoleArn rejects a non-empty value that is not
@@ -89,11 +140,12 @@ func (s *EKSServiceImpl) CreateAddon(ctx context.Context, input *eks.CreateAddon
 	if !ok {
 		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
 	}
-	version := aws.StringValue(input.AddonVersion)
-	if version == "" {
-		version = spec.DefaultVersion
-	} else if !spec.SupportsVersion(version) {
-		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
+	version := spec.DefaultVersion
+	if requested := aws.StringValue(input.AddonVersion); requested != "" {
+		var err error
+		if version, err = resolveAddonVersion(ctx, spec, requested, cluster); err != nil {
+			return nil, err
+		}
 	}
 	acctKV, err := s.acctKVForNewDependant(ctx, accountID, cluster)
 	if err != nil {
@@ -197,9 +249,13 @@ func (s *EKSServiceImpl) UpdateAddon(ctx context.Context, input *eks.UpdateAddon
 	version := aws.StringValue(input.AddonVersion)
 	if version != "" {
 		spec, ok := addon.Lookup(addonName)
-		if !ok || !spec.SupportsVersion(version) {
+		if !ok {
 			return nil, errors.New(awserrors.ErrorInvalidParameterValue)
 		}
+		var err error
+		if version, err = resolveAddonVersion(ctx, spec, version, cluster); err != nil {
+			return nil, err
+		}
 	}
 	rec, err := s.addons().Update(ctx, acctKV, accountID, cluster, addonName, addon.Change{
 		Version:               version,
@@ -246,7 +302,7 @@ func addonRecordToAWS(cluster string, rec *addon.Record) *eks.Addon {
 	out := &eks.Addon{
 		AddonArn:     aws.String(rec.Arn),
 		AddonName:    aws.String(rec.AddonName),
-		AddonVersion: aws.String(rec.AddonVersion),
+		AddonVersion: aws.String(awsAddonVersion(rec)),
 		ClusterName:  aws.String(cluster),
 		Status:       aws.String(string(rec.Status)),
 		CreatedAt:    aws.Time(rec.CreatedAt),
@@ -269,17 +325,36 @@ func addonRecordToAWS(cluster string, rec *addon.Record) *eks.Addon {
 	return out
 }
 
-// addonSpecToAWS converts a catalog spec to the SDK AddonInfo shape.
+// The owner and publisher Spinifex reports for every catalogue add-on, as AWS
+// does for the add-ons it publishes.
+const (
+	addonOwner     = "aws"
+	addonPublisher = "eks"
+)
+
+// addonSpecToAWS converts a catalog spec to the SDK AddonInfo shape. Every
+// version supports the one cluster version the node image ships, and exactly
+// one of them is that cluster version's default.
 func addonSpecToAWS(spec addon.Spec) *eks.AddonInfo {
 	versions := make([]*eks.AddonVersionInfo, 0, len(spec.Versions))
-	for _, v := range spec.Versions {
+	for _, v := range spec.AWSVersions() {
 		versions = append(versions, &eks.AddonVersionInfo{
 			AddonVersion:           aws.String(v),
+			Architecture:           aws.StringSlice([]string{"amd64"}),
+			RequiresConfiguration:  aws.Bool(false),
 			RequiresIamPermissions: aws.Bool(spec.RequiresIRSA),
+			Compatibilities: []*eks.Compatibility{{
+				ClusterVersion:   aws.String(defaultK8sVersion),
+				PlatformVersions: aws.StringSlice([]string{"*"}),
+				DefaultVersion:   aws.Bool(v == spec.AWSVersion(spec.DefaultVersion)),
+			}},
 		})
 	}
 	return &eks.AddonInfo{
 		AddonName:     aws.String(spec.Name),
+		Type:          aws.String(spec.Type),
+		Owner:         aws.String(addonOwner),
+		Publisher:     aws.String(addonPublisher),
 		AddonVersions: versions,
 	}
 }
@@ -308,3 +383,30 @@ func (s *EKSServiceImpl) addonBucket(ctx context.Context, accountID string) (jet
 	}
 	return GetOrCreateAccountBucket(ctx, js, accountID)
 }
+
+// resolveAddonVersion maps a client version, in AWS form or bare upstream form,
+// to the upstream form the record and the bundle lookup use. The bare form is
+// more lenient than AWS, so each use is logged.
+func resolveAddonVersion(ctx context.Context, spec addon.Spec, requested, cluster string) (string, error) {
+	upstream, ok := spec.ParseVersion(requested)
+	if !ok {
+		return "", awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
+			"addonVersion %q is not supported for add-on %s; supported versions: %s",
+			requested, spec.Name, strings.Join(spec.AWSVersions(), ", "))
+	}
+	if requested == upstream {
+		slog.InfoContext(ctx, "EKS add-on version given in bare upstream form",
+			"addon", spec.Name, "cluster", cluster, "version", requested)
+	}
+	return upstream, nil
+}
+
+// awsAddonVersion returns a stored record's version in AWS form; an add-on the
+// catalogue no longer lists keeps its stored string.
+func awsAddonVersion(rec *addon.Record) string {
+	spec, ok := addon.Lookup(rec.AddonName)
+	if !ok {
+		return rec.AddonVersion
+	}
+	return spec.AWSVersion(rec.AddonVersion)
+}

spinifex/domains/eks/cluster/addons_test.go

@@ -98,7 +98,9 @@ func TestCreateAddon_UnknownVersionRejected(t *testing.T) {
 		ClusterName: aws.String("c1"), AddonName: aws.String(albController),
 		AddonVersion: aws.String("9.9.9-nope"),
 	}, testAccountID)
-	require.EqualError(t, err, awserrors.ErrorInvalidParameterValue)
+	code, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorInvalidParameterValue, code)
 }
 
 func TestCreateAddon_CreatesStagingAndDefaultsVersion(t *testing.T) {
@@ -113,7 +115,7 @@ func TestCreateAddon_CreatesStagingAndDefaultsVersion(t *testing.T) {
 	require.NotNil(t, out.Addon)
 	// Version defaults to catalog default; status starts CREATING (honest until
 	// VM-side delivery confirms ACTIVE).
-	assert.Equal(t, spec.DefaultVersion, aws.StringValue(out.Addon.AddonVersion))
+	assert.Equal(t, spec.AWSVersion(spec.DefaultVersion), aws.StringValue(out.Addon.AddonVersion))
 	assert.Equal(t, string(addon.StatusCreating), aws.StringValue(out.Addon.Status))
 	assert.Contains(t, aws.StringValue(out.Addon.AddonArn), ":addon/c1/"+albController)
 
@@ -214,7 +216,7 @@ func TestUpdateAddon_ChangesVersionAndReinstalls(t *testing.T) {
 	}, testAccountID)
 	require.NoError(t, err)
 	assert.Equal(t, `{"replicaCount":2}`, aws.StringValue(desc.Addon.ConfigurationValues))
-	assert.Equal(t, spec.DefaultVersion, aws.StringValue(desc.Addon.AddonVersion))
+	assert.Equal(t, spec.AWSVersion(spec.DefaultVersion), aws.StringValue(desc.Addon.AddonVersion))
 	assert.Equal(t, string(addon.StatusUpdating), aws.StringValue(desc.Addon.Status))
 }
 
@@ -230,7 +232,9 @@ func TestUpdateAddon_UnknownVersionRejected(t *testing.T) {
 		ClusterName: aws.String("c1"), AddonName: aws.String("argocd"),
 		AddonVersion: aws.String("9.9.9-nope"),
 	}, testAccountID)
-	require.EqualError(t, err, awserrors.ErrorInvalidParameterValue)
+	code, ok := awserrors.ResolveErrorCode(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorInvalidParameterValue, code)
 }
 
 func TestUpdateAddon_MissingIsNotFound(t *testing.T) {

spinifex/domains/eks/cluster/cluster_security_group_test.go

@@ -0,0 +1,421 @@
+package cluster
+
+import (
+	"context"
+	"maps"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/eks"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// sgCreatesIn returns the CreateSecurityGroup calls made against vpc whose name matches.
+func sgCreatesIn(f *fakeSGProvisioner, vpc string, match func(name string) bool) []*ec2.CreateSecurityGroupInput {
+	var out []*ec2.CreateSecurityGroupInput
+	for _, c := range f.createCalls {
+		if aws.StringValue(c.VpcId) == vpc && match(aws.StringValue(c.GroupName)) {
+			out = append(out, c)
+		}
+	}
+	return out
+}
+
+func sgCreateTags(c *ec2.CreateSecurityGroupInput) map[string]string {
+	m := map[string]string{}
+	for _, spec := range c.TagSpecifications {
+		maps.Copy(m, sgTagMap(spec.Tags))
+	}
+	return m
+}
+
+func authorizedOn(f *fakeSGProvisioner, sgID string) []*ec2.IpPermission {
+	var out []*ec2.IpPermission
+	for _, c := range f.authorizeCalls {
+		if aws.StringValue(c.GroupId) == sgID {
+			out = append(out, c.IpPermissions...)
+		}
+	}
+	return out
+}
+
+func deletedSGIDs(f *fakeSGProvisioner) []string {
+	var out []string
+	for _, c := range f.deleteCalls {
+		out = append(out, aws.StringValue(c.GroupId))
+	}
+	return out
+}
+
+// seedClusterSG registers an AWS-style cluster SG the way a created one looks.
+func seedClusterSG(f *fakeSGProvisioner, cluster, vpc, id string) {
+	f.existing[clusterSGNamePrefix+cluster+"-0123456789abcdef|"+vpc] = id
+	if f.tagsByID == nil {
+		f.tagsByID = map[string][]*ec2.Tag{}
+	}
+	f.tagsByID[id] = []*ec2.Tag{
+		{Key: aws.String("kubernetes.io/cluster/" + cluster), Value: aws.String("owned")},
+		{Key: aws.String("aws:eks:cluster-name"), Value: aws.String(cluster)},
+		{Key: aws.String(clusterEKSClusterTagKey), Value: aws.String(cluster)},
+		{Key: aws.String(clusterEKSRoleTagKey), Value: aws.String("cluster")},
+		{Key: aws.String(tags.ManagedByKey), Value: aws.String(tags.ManagedByEKS)},
+	}
+}
+
+// seedCustomerSG registers a customer-made SG, optionally tagged to imitate ours.
+func seedCustomerSG(f *fakeSGProvisioner, name, vpc, id string, tg []*ec2.Tag) {
+	f.existing[name+"|"+vpc] = id
+	if f.tagsByID == nil {
+		f.tagsByID = map[string][]*ec2.Tag{}
+	}
+	f.tagsByID[id] = tg
+}
+
+// spoofTags are every tag a customer can set that the service also stamps.
+func spoofTags(cluster string) []*ec2.Tag {
+	return []*ec2.Tag{
+		{Key: aws.String(clusterEKSClusterTagKey), Value: aws.String(cluster)},
+		{Key: aws.String(clusterEKSRoleTagKey), Value: aws.String("cluster")},
+		{Key: aws.String(tags.ManagedByKey), Value: aws.String(tags.ManagedByEKS)},
+		{Key: aws.String("kubernetes.io/cluster/" + cluster), Value: aws.String("owned")},
+	}
+}
+
+func setStoredClusterSG(t *testing.T, f *deleteClusterFixture, cluster, id string) {
+	t.Helper()
+	meta, err := GetClusterMeta(t.Context(), f.kv, cluster)
+	require.NoError(t, err)
+	meta.ResourcesVpcConfig.ClusterSecurityGroupId = id
+	require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta))
+}
+
+func createAlpha(t *testing.T, mutate func(*eks.CreateClusterInput)) (*eksServiceFixture, *ClusterMeta) {
+	t.Helper()
+	f := newEKSServiceFixture(t)
+	f.sg.uniqueIDs = true
+	in := createInput("alpha")
+	if mutate != nil {
+		mutate(in)
+	}
+	_, err := f.svc.CreateCluster(context.Background(), in, testAccountID, "")
+	require.NoError(t, err)
+	f.svc.WaitLaunches()
+	meta, err := GetClusterMeta(t.Context(), f.kv, "alpha")
+	require.NoError(t, err)
+	require.NotNil(t, meta.ResourcesVpcConfig)
+	return f, meta
+}
+
+func withPrivateAccess(in *eks.CreateClusterInput) {
+	in.ResourcesVpcConfig.EndpointPrivateAccess = aws.Bool(true)
+}
+
+// CreateCluster makes the one AWS-style cluster SG in the caller's VPC and
+// reports it as clusterSecurityGroupId.
+func TestCreateCluster_ClusterSGHasAWSNameTagsAndRules(t *testing.T) {
+	t.Parallel()
+	f, meta := createAlpha(t, nil)
+
+	created := sgCreatesIn(f.sg, "vpc-aaa", func(n string) bool { return strings.HasPrefix(n, clusterSGNamePrefix+"alpha-") })
+	require.Len(t, created, 1, "exactly one eks-cluster-sg-<name>-<id> SG in the customer VPC")
+	name := aws.StringValue(created[0].GroupName)
+	assert.NotEmpty(t, strings.TrimPrefix(name, clusterSGNamePrefix+"alpha-"), "name carries a unique id suffix")
+
+	tg := sgCreateTags(created[0])
+	assert.Equal(t, "owned", tg["kubernetes.io/cluster/alpha"])
+	assert.Equal(t, "alpha", tg["aws:eks:cluster-name"])
+	assert.NotEmpty(t, tg["Name"])
+	assert.Equal(t, "alpha", tg[clusterEKSClusterTagKey])
+	assert.Equal(t, tags.ManagedByEKS, tg[tags.ManagedByKey])
+
+	id := f.sg.existing[name+"|vpc-aaa"]
+	require.NotEmpty(t, id)
+	assert.Equal(t, id, meta.ResourcesVpcConfig.ClusterSecurityGroupId)
+
+	perms := authorizedOn(f.sg, id)
+	require.Len(t, perms, 1, "inbound all from itself, nothing else")
+	assert.Equal(t, "-1", aws.StringValue(perms[0].IpProtocol))
+	require.Len(t, perms[0].UserIdGroupPairs, 1)
+	assert.Equal(t, id, aws.StringValue(perms[0].UserIdGroupPairs[0].GroupId))
+	for _, c := range f.sg.revokeEgressCall {
+		assert.NotEqual(t, id, aws.StringValue(c.GroupId), "egress stays allow-all")
+	}
+
+	out, err := f.svc.DescribeCluster(context.Background(), &eks.DescribeClusterInput{Name: aws.String("alpha")}, testAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, id, aws.StringValue(out.Cluster.ResourcesVpcConfig.ClusterSecurityGroupId))
+}
+
+// A new cluster has no customer-VPC control-plane or nodegroup SG, and exactly
+// one SG per cluster carries kubernetes.io/cluster/<name>.
+func TestCreateCluster_NoSeparateCustomerVPCControlPlaneOrNodegroupSG(t *testing.T) {
+	t.Parallel()
+	f, _ := createAlpha(t, withPrivateAccess)
+
+	retired := sgCreatesIn(f.sg, "vpc-aaa", func(n string) bool {
+		return n == ClusterControlPlaneSGName("alpha") || n == ClusterNodegroupSGName("alpha")
+	})
+	assert.Empty(t, retired, "customer VPC must not get control-plane-sg or nodegroup-sg")
+
+	tagged := 0
+	for _, c := range f.sg.createCalls {
+		if _, ok := sgCreateTags(c)["kubernetes.io/cluster/alpha"]; ok {
+			tagged++
+		}
+	}
+	assert.Equal(t, 1, tagged, "exactly one SG tagged kubernetes.io/cluster/alpha")
+}
+
+// The private-endpoint ENI carries its own SG and the cluster SG.
+func TestCreateCluster_PrivateEndpointENICarriesBothSGs(t *testing.T) {
+	t.Parallel()
+	f, meta := createAlpha(t, withPrivateAccess)
+
+	var eni *ec2.CreateNetworkInterfaceInput
+	for _, c := range f.vpc.createCalls {
+		if strings.Contains(aws.StringValue(c.Description), "private-endpoint") {
+			eni = c
+		}
+	}
+	require.NotNil(t, eni, "private-endpoint ENI created")
+	peSG := f.sg.existing[ClusterPrivateEndpointSGName("alpha")+"|vpc-aaa"]
+	require.NotEmpty(t, peSG)
+	assert.ElementsMatch(t,
+		[]string{peSG, meta.ResourcesVpcConfig.ClusterSecurityGroupId},
+		aws.StringValueSlice(eni.Groups))
+	assert.NotContains(t, sgTagMap(f.sg.tagsByID[peSG]), "kubernetes.io/cluster/alpha",
+		"only the cluster SG carries the cluster tag")
+}
+
+// The system control-plane VPC keeps its control-plane SG and rules.
+func TestCreateCluster_SystemCPVPCSecurityGroupUnchanged(t *testing.T) {
+	t.Parallel()
+	f, meta := createAlpha(t, nil)
+	require.NotNil(t, meta.ManagedCPVPC)
+	cpVPC := meta.ManagedCPVPC.VpcId
+
+	cp := sgCreatesIn(f.sg, cpVPC, func(n string) bool { return n == ClusterControlPlaneSGName("alpha") })
+	require.Len(t, cp, 1)
+	cpID := f.sg.existing[ClusterControlPlaneSGName("alpha")+"|"+cpVPC]
+	require.NotEmpty(t, cpID)
+
+	var ports []int64
+	selfRef := false
+	for _, p := range authorizedOn(f.sg, cpID) {
+		ports = append(ports, aws.Int64Value(p.FromPort))
+		for _, pair := range p.UserIdGroupPairs {
+			if aws.StringValue(pair.GroupId) == cpID {
+				selfRef = true
+			}
+		}
+	}
+	assert.Subset(t, ports, []int64{k3sAPIServerPort, konnectivityAgentPort, webhookReadyPort, 2379, 10250})
+	assert.True(t, selfRef, "HA self-referencing rules stay")
+	assert.Empty(t, sgCreatesIn(f.sg, cpVPC, func(n string) bool { return strings.HasPrefix(n, clusterSGNamePrefix) }),
+		"the cluster SG is a customer-VPC resource only")
+}
+
+// Workers launch with exactly the cluster SG and no extra SG or cross rules.
+func TestCreateNodegroup_WorkersGetExactlyTheClusterSG(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	f.sg.uniqueIDs = true
+	seedActiveClusterWithToken(t, f, "c1")
+	meta, err := GetClusterMeta(t.Context(), f.kv, "c1")
+	require.NoError(t, err)
+	meta.ResourcesVpcConfig.ClusterSecurityGroupId = "sg-cluster"
+	require.NoError(t, PutClusterMeta(t.Context(), f.kv, meta))
+	seedClusterSG(f.sg, "c1", "vpc-aaa", "sg-cluster")
+	markWorkersReady(t, f, "c1", "ng1", 2)
+
+	_, err = f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 2), testAccountID)
+	require.NoError(t, err)
+	f.svc.WaitLaunches()
+
+	require.Len(t, f.worker.runCalls, 2)
+	for _, c := range f.worker.runCalls {
+		assert.Equal(t, []string{"sg-cluster"}, aws.StringValueSlice(c.SecurityGroupIds))
+	}
+	assert.Empty(t, f.sg.createCalls, "no nodegroup or control-plane SG is created for a worker")
+	assert.Empty(t, f.sg.authorizeCalls, "no CP<->nodegroup cross rules")
+}
+
+// A launch template that names security groups gets exactly those.
+func TestLaunchOneWorker_TemplateSecurityGroupsReplaceClusterSG(t *testing.T) {
+	t.Parallel()
+	reader := &fakeLaunchTemplates{out: &ec2.DescribeLaunchTemplateVersionsOutput{
+		LaunchTemplateVersions: []*ec2.LaunchTemplateVersion{{
+			LaunchTemplateData: &ec2.ResponseLaunchTemplateData{
+				SecurityGroupIds: aws.StringSlice([]string{"sg-lt-1", "sg-lt-2"}),
+			},
+		}},
+	}}
+	f, err := launchWorkerWith(t, &NodegroupLaunchTemplate{ID: "lt-0123", Version: "1"}, reader)
+	require.NoError(t, err)
+
+	require.Len(t, f.worker.runCalls, 1)
+	assert.Equal(t, []string{"sg-lt-1", "sg-lt-2"}, aws.StringValueSlice(f.worker.runCalls[0].SecurityGroupIds))
+}
+
+// A template naming no security groups falls back to the cluster SG alone.
+func TestLaunchOneWorker_TemplateWithoutSecurityGroupsUsesClusterSG(t *testing.T) {
+	t.Parallel()
+	f, err := launchWorkerWith(t, &NodegroupLaunchTemplate{ID: "lt-0123", Version: "1"},
+		&fakeLaunchTemplates{out: templateVersionWith(nil)})
+	require.NoError(t, err)
+
+	require.Len(t, f.worker.runCalls, 1)
+	assert.Equal(t, []string{"sg-ng"}, aws.StringValueSlice(f.worker.runCalls[0].SecurityGroupIds))
+}
+
+// Delete removes the SG recorded on the cluster, whatever its name, and leaves
+// another cluster's SG in the same VPC alone.
+func TestDeleteCluster_RemovesClusterSGFoundByTag(t *testing.T) {
+	t.Parallel()
+	f := newDeleteClusterFixture(t, "alpha")
+	seedClusterSG(f.sg, "alpha", "vpc-aaa", "sg-new")
+	seedClusterSG(f.sg, "beta", "vpc-aaa", "sg-beta")
+	setStoredClusterSG(t, f, "alpha", "sg-new")
+	f.sg.perms = map[string][]*ec2.IpPermission{"sg-new": {{
+		IpProtocol:       aws.String("-1"),
+		UserIdGroupPairs: []*ec2.UserIdGroupPair{{GroupId: aws.String("sg-new")}},
+	}}}
+	f.sg.enforceDepViolation = true
+
+	deleteAndSettle(t, f.svc, "alpha")
+
+	deleted := deletedSGIDs(f.sg)
+	assert.Contains(t, deleted, "sg-new")
+	assert.NotContains(t, deleted, "sg-beta")
+	_, err := GetClusterMeta(t.Context(), f.kv, "alpha")
+	assert.Error(t, err, "teardown completed and removed the record")
+}
+
+// An old-layout cluster (control-plane-sg + nodegroup-sg, no cluster SG) still
+// has both SGs removed.
+func TestDeleteCluster_OldLayoutStillRemovesBothNamedSGs(t *testing.T) {
+	t.Parallel()
+	f := newDeleteClusterFixture(t, "alpha")
+	f.sg.existing[ClusterControlPlaneSGName("alpha")+"|vpc-aaa"] = "sg-old-cp"
+	f.sg.existing[ClusterNodegroupSGName("alpha")+"|vpc-aaa"] = "sg-old-ng"
+
+	deleteAndSettle(t, f.svc, "alpha")
+
+	deleted := deletedSGIDs(f.sg)
+	assert.Contains(t, deleted, "sg-old-cp")
+	assert.Contains(t, deleted, "sg-old-ng")
+}
+
+// A cluster SG that cannot be deleted yet keeps the cluster DELETING; the
+// deleting reaper removes it once the dependency clears, not the request.
+func TestDeleteCluster_ClusterSGConvergesInTheReaper(t *testing.T) {
+	origBudget, origInterval := sgDeleteWaitBudget, sgDeleteWaitInterval
+	sgDeleteWaitBudget, sgDeleteWaitInterval = 10*time.Millisecond, time.Millisecond
+	t.Cleanup(func() { sgDeleteWaitBudget, sgDeleteWaitInterval = origBudget, origInterval })
+
+	f := newDeleteClusterFixture(t, "alpha")
+	seedClusterSG(f.sg, "alpha", "vpc-aaa", "sg-new")
+	setStoredClusterSG(t, f, "alpha", "sg-new")
+	f.sg.deleteFailsBefore = map[string]int{"sg-new": 1000}
+
+	deleteAndSettle(t, f.svc, "alpha")
+	meta := requireTeardownRetained(t, f.kv, "alpha")
+	assert.Contains(t, meta.DeleteFailedStep, stepSecurityGroups)
+
+	f.sg.mu.Lock()
+	f.sg.deleteFailsBefore["sg-new"] = 0
+	f.sg.mu.Unlock()
+	redrive(t, f.svc)
+
+	_, err := GetClusterMeta(t.Context(), f.kv, "alpha")
+	assert.Error(t, err, "the reaper finished the teardown")
+}
+
+// A customer-made SG carrying the tags the service stamps is neither adopted by
+// a create retry nor deleted with the cluster.
+func TestEnsureClusterSG_DoesNotAdoptACustomerTaggedSG(t *testing.T) {
+	t.Parallel()
+	sg := newFakeSGProvisioner()
+	sg.uniqueIDs = true
+	seedCustomerSG(sg, "customer-sg", "vpc-aaa", "sg-spoof", spoofTags("c1"))
+
+	id, err := EnsureClusterSG(context.Background(), sg, testAccountID, "c1", "vpc-aaa")
+	require.NoError(t, err)
+
+	assert.NotEqual(t, "sg-spoof", id)
+	assert.Len(t, sg.createCalls, 1)
+	for _, p := range sg.authorizeCalls {
+		assert.NotEqual(t, "sg-spoof", aws.StringValue(p.GroupId))
+	}
+}
+
+func TestDeleteCluster_DoesNotDeleteACustomerTaggedSG(t *testing.T) {
+	t.Parallel()
+	f := newDeleteClusterFixture(t, "alpha")
+	seedClusterSG(f.sg, "alpha", "vpc-aaa", "sg-new")
+	setStoredClusterSG(t, f, "alpha", "sg-new")
+	seedCustomerSG(f.sg, "customer-sg", "vpc-aaa", "sg-spoof", spoofTags("alpha"))
+	f.sg.perms = map[string][]*ec2.IpPermission{"sg-spoof": {{
+		IpProtocol: aws.String("tcp"), FromPort: aws.Int64(22), ToPort: aws.Int64(22),
+		IpRanges: []*ec2.IpRange{{CidrIp: aws.String("10.0.0.0/8")}},
+	}}}
+
+	deleteAndSettle(t, f.svc, "alpha")
+
+	assert.Contains(t, deletedSGIDs(f.sg), "sg-new")
+	assert.NotContains(t, deletedSGIDs(f.sg), "sg-spoof")
+	for _, r := range f.sg.revokeCalls {
+		assert.NotEqual(t, "sg-spoof", aws.StringValue(r.GroupId), "no rule is revoked on it")
+	}
+}
+
+// Deleting a cluster revokes, on an unrelated SG in the VPC, only the rules that
+// reference a cluster-owned SG.
+func TestDeleteClusterSGs_UnrelatedSGKeepsItsOtherRules(t *testing.T) {
+	t.Parallel()
+	sgp := newFakeSGProvisioner()
+	seedClusterSG(sgp, "alpha", "vpc-aaa", "sg-new")
+	seedCustomerSG(sgp, "customer-sg", "vpc-aaa", "sg-other", nil)
+	ssh := &ec2.IpPermission{
+		IpProtocol: aws.String("tcp"), FromPort: aws.Int64(22), ToPort: aws.Int64(22),
+		IpRanges: []*ec2.IpRange{{CidrIp: aws.String("10.0.0.0/8")}},
+	}
+	mixed := &ec2.IpPermission{
+		IpProtocol: aws.String("tcp"), FromPort: aws.Int64(443), ToPort: aws.Int64(443),
+		IpRanges: []*ec2.IpRange{{CidrIp: aws.String("192.168.0.0/16")}},
+		UserIdGroupPairs: []*ec2.UserIdGroupPair{
+			{GroupId: aws.String("sg-new")},
+			{GroupId: aws.String("sg-peer")},
+		},
+	}
+	sgp.perms = map[string][]*ec2.IpPermission{"sg-other": {ssh, mixed}}
+	sgp.permsEgress = map[string][]*ec2.IpPermission{"sg-other": {groupRefPerm("tcp", 6443, "sg-new"), groupRefPerm("tcp", 80, "sg-peer")}}
+
+	require.NoError(t, DeleteClusterSGs(context.Background(), sgp, testAccountID, "alpha", "vpc-aaa", "sg-new"))
+
+	var revoked []*ec2.IpPermission
+	for _, r := range sgp.revokeCalls {
+		if aws.StringValue(r.GroupId) == "sg-other" {
+			revoked = append(revoked, r.IpPermissions...)
+		}
+	}
+	require.Len(t, revoked, 1, "only the rule referencing the cluster SG goes")
+	require.Len(t, revoked[0].UserIdGroupPairs, 1)
+	assert.Equal(t, "sg-new", aws.StringValue(revoked[0].UserIdGroupPairs[0].GroupId))
+	assert.Empty(t, revoked[0].IpRanges, "its CIDR sources are not part of the revoke")
+
+	var revokedEgress []*ec2.IpPermission
+	for _, r := range sgp.revokeEgressCall {
+		if aws.StringValue(r.GroupId) == "sg-other" {
+			revokedEgress = append(revokedEgress, r.IpPermissions...)
+		}
+	}
+	require.Len(t, revokedEgress, 1)
+	assert.Equal(t, "sg-new", aws.StringValue(revokedEgress[0].UserIdGroupPairs[0].GroupId))
+	assert.NotContains(t, deletedSGIDs(sgp), "sg-other")
+}

spinifex/domains/eks/cluster/create_cluster_test.go

@@ -419,37 +419,6 @@ func TestCreateCluster_BuildsManagedCPVPCUnderSystemAccount(t *testing.T) {
 	assert.True(t, tagged, "NAT gateway EIP must carry the ManagedBy=eks tag")
 }
 
-// The primary cluster SG (clusterSecurityGroupId) must be created in the
-// caller's own VPC, not the system-managed CP VPC the control-plane VM
-// actually lives in — otherwise the tenant's own DescribeSecurityGroups can
-// never resolve it. fakeSubnetResolver resolves every subnet to "vpc-aaa".
-func TestCreateCluster_ClusterSecurityGroupCreatedInCallerVPC(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-
-	_, err := f.svc.CreateCluster(context.Background(), createInput("alpha"), testAccountID, "")
-	require.NoError(t, err)
-	f.svc.WaitLaunches()
-
-	meta, err := GetClusterMeta(t.Context(), f.kv, "alpha")
-	require.NoError(t, err)
-	require.NotNil(t, meta.ResourcesVpcConfig)
-	assert.NotEmpty(t, meta.ResourcesVpcConfig.ClusterSecurityGroupId)
-
-	found := false
-	for _, call := range f.sg.createCalls {
-		if aws.StringValue(call.GroupName) == ClusterControlPlaneSGName("alpha") && aws.StringValue(call.VpcId) == "vpc-aaa" {
-			found = true
-		}
-	}
-	assert.True(t, found, "control-plane SG must be created in the caller's VPC (vpc-aaa)")
-
-	out, err := f.svc.DescribeCluster(context.Background(), &eks.DescribeClusterInput{Name: aws.String("alpha")}, testAccountID)
-	require.NoError(t, err)
-	require.NotNil(t, out.Cluster.ResourcesVpcConfig)
-	assert.Equal(t, meta.ResourcesVpcConfig.ClusterSecurityGroupId, aws.StringValue(out.Cluster.ResourcesVpcConfig.ClusterSecurityGroupId))
-}
-
 // Security groups the caller explicitly passes on CreateCluster must come back
 // unchanged from DescribeCluster — never substituted with platform-created ids.
 func TestCreateCluster_CallerSecurityGroupIdsEchoedNotOverwritten(t *testing.T) {

spinifex/domains/eks/cluster/nodegroup.go

@@ -472,13 +472,9 @@ type nodegroupLaunchCtx struct {
 func (s *EKSServiceImpl) launchNodegroupInfra(ctx context.Context, lc nodegroupLaunchCtx) {
 	acctKV, accountID, cluster, ng, meta, rec := lc.acctKV, lc.accountID, lc.cluster, lc.ng, lc.meta, lc.rec
 
-	cpSGID, ngSGID, err := EnsureClusterSGs(ctx, s.deps.VPCSG, accountID, cluster, meta.ResourcesVpcConfig.VpcId)
+	ngSGID, err := EnsureWorkerSG(ctx, s.deps.VPCSG, accountID, cluster, meta.ResourcesVpcConfig.VpcId, meta.ResourcesVpcConfig.ClusterSecurityGroupId)
 	if err != nil {
-		s.markNodegroupFailed(ctx, acctKV, cluster, ng, "ensure cluster SGs: "+err.Error())
-		return
-	}
-	if err := EnsureNodegroupSGRules(ctx, s.deps.VPCSG, accountID, cluster, cpSGID, ngSGID); err != nil {
-		s.markNodegroupFailed(ctx, acctKV, cluster, ng, "ensure nodegroup SG rules: "+err.Error())
+		s.markNodegroupFailed(ctx, acctKV, cluster, ng, "ensure cluster SG: "+err.Error())
 		return
 	}
 
@@ -707,11 +703,15 @@ func (s *EKSServiceImpl) launchOneWorker(ctx context.Context, rec *NodegroupReco
 		}}
 	}
 
-	metadataOptions, err := s.workerMetadataOptions(ctx, rec, accountID)
+	metadataOptions, templateSGs, err := s.workerTemplateSettings(ctx, rec, accountID)
 	if err != nil {
 		return "", err
 	}
 	runInput.MetadataOptions = metadataOptions
+	// As on AWS, security groups named by the launch template replace the cluster SG.
+	if len(templateSGs) > 0 {
+		runInput.SecurityGroupIds = aws.StringSlice(templateSGs)
+	}
 
 	// Attach the node role's instance profile so IMDS serves the role to the ECR
 	// credential provider; without it worker pulls from the internal ECR get 401.
@@ -743,21 +743,23 @@ func (s *EKSServiceImpl) launchOneWorker(ctx context.Context, rec *NodegroupReco
 	return "", errors.New("run worker returned no instance id")
 }
 
-// workerMetadataOptions chooses a worker's IMDS options: fields a launch template
-// version sets win as given, and every unset field gets the managed nodegroup
-// default (IMDSv2 required, hop limit 2) so pods behind the node can reach IMDS.
-func (s *EKSServiceImpl) workerMetadataOptions(ctx context.Context, rec *NodegroupRecord, accountID string) (*ec2.InstanceMetadataOptionsRequest, error) {
+// workerTemplateSettings chooses a worker's IMDS options and security groups.
+// Metadata fields a launch template version sets win as given, and every unset
+// field gets the managed nodegroup default (IMDSv2 required, hop limit 2) so pods
+// behind the node can reach IMDS. The template's security group IDs, if any, are
+// returned for the caller to use in place of the cluster SG.
+func (s *EKSServiceImpl) workerTemplateSettings(ctx context.Context, rec *NodegroupRecord, accountID string) (*ec2.InstanceMetadataOptionsRequest, []string, error) {
 	managedDefault := &ec2.InstanceMetadataOptionsRequest{
 		HttpTokens:              aws.String(ec2.HttpTokensStateRequired),
 		HttpPutResponseHopLimit: aws.Int64(workerMetadataHopLimit),
 	}
 	if rec.LaunchTemplate == nil {
-		return managedDefault, nil
+		return managedDefault, nil, nil
 	}
 	if s.deps.LaunchTemplates == nil {
 		slog.ErrorContext(ctx, "EKS nodegroup: launch template named but no launch template service wired",
 			"nodegroup", rec.Name, "launchTemplate", rec.LaunchTemplate.ID+rec.LaunchTemplate.Name)
-		return nil, errors.New(awserrors.ErrorServerInternal)
+		return nil, nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
 	version := rec.LaunchTemplate.Version
@@ -772,14 +774,16 @@ func (s *EKSServiceImpl) workerMetadataOptions(ctx context.Context, rec *Nodegro
 	}
 	out, err := s.deps.LaunchTemplates.DescribeLaunchTemplateVersions(ctx, in, accountID)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	if len(out.LaunchTemplateVersions) == 0 || out.LaunchTemplateVersions[0].LaunchTemplateData == nil {
-		return nil, errors.New(awserrors.ErrorInvalidLaunchTemplateIdVersionNotFound)
+		return nil, nil, errors.New(awserrors.ErrorInvalidLaunchTemplateIdVersionNotFound)
 	}
-	opts := out.LaunchTemplateVersions[0].LaunchTemplateData.MetadataOptions
+	data := out.LaunchTemplateVersions[0].LaunchTemplateData
+	sgIDs := aws.StringValueSlice(data.SecurityGroupIds)
+	opts := data.MetadataOptions
 	if opts == nil {
-		return managedDefault, nil
+		return managedDefault, sgIDs, nil
 	}
 	req := &ec2.InstanceMetadataOptionsRequest{
 		HttpTokens:              opts.HttpTokens,
@@ -796,7 +800,7 @@ func (s *EKSServiceImpl) workerMetadataOptions(ctx context.Context, rec *Nodegro
 	if req.HttpPutResponseHopLimit == nil {
 		req.HttpPutResponseHopLimit = managedDefault.HttpPutResponseHopLimit
 	}
-	return req, nil
+	return req, sgIDs, nil
 }
 
 // selectWorkerHost picks the host for the next worker: the eligible host (one
@@ -886,7 +890,7 @@ func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string
 		return s.attachRoleToProfile(accountID, profileName, roleName,
 			aws.StringValue(out.InstanceProfile.Arn), len(out.InstanceProfile.Roles) > 0)
 	}
-	if err.Error() != awserrors.ErrorIAMNoSuchEntity {
+	if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMNoSuchEntity {
 		return "", fmt.Errorf("get instance profile %q: %w", profileName, err)
 	}
 
@@ -895,7 +899,7 @@ func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string
 	})
 	if err != nil {
 		// A racing worker launch may have created it first; re-read and converge.
-		if err.Error() != awserrors.ErrorIAMEntityAlreadyExists {
+		if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMEntityAlreadyExists {
 			return "", fmt.Errorf("create instance profile %q: %w", profileName, err)
 		}
 		got, gerr := s.iamEnsurer().GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{
@@ -922,7 +926,7 @@ func (s *EKSServiceImpl) attachRoleToProfile(accountID, profileName, roleName, p
 		InstanceProfileName: aws.String(profileName),
 		RoleName:            aws.String(roleName),
 	})
-	if err != nil && err.Error() != awserrors.ErrorIAMLimitExceeded {
+	if code, _ := awserrors.ResolveErrorCode(err); err != nil && code != awserrors.ErrorIAMLimitExceeded {
 		return "", fmt.Errorf("add role %q to instance profile %q: %w", roleName, profileName, err)
 	}
 	return profileARN, nil
@@ -1092,9 +1096,9 @@ func (s *EKSServiceImpl) launchWorkersCAS(ctx context.Context, acctKV jetstream.
 		}
 		return nil, fmt.Errorf("resolve eks-node AMI: %w", err)
 	}
-	_, ngSGID, err := EnsureClusterSGs(ctx, s.deps.VPCSG, accountID, cluster, meta.ResourcesVpcConfig.VpcId)
+	ngSGID, err := EnsureWorkerSG(ctx, s.deps.VPCSG, accountID, cluster, meta.ResourcesVpcConfig.VpcId, meta.ResourcesVpcConfig.ClusterSecurityGroupId)
 	if err != nil {
-		return nil, fmt.Errorf("ensure cluster SGs: %w", err)
+		return nil, fmt.Errorf("ensure cluster SG: %w", err)
 	}
 
 	// Each iteration records at most one worker, so bound by the gap plus CAS slack.

spinifex/domains/eks/cluster/nodegroup_test.go

@@ -338,6 +338,25 @@ func TestEnsureNodegroupSGRules_DuplicateRuleTolerated(t *testing.T) {
 		"duplicate-rule error must be treated as success")
 }
 
+// A cluster created before the cluster SG keeps launching workers into its
+// nodegroup SG with the control-plane cross rules.
+func TestCreateNodegroup_OldLayoutClusterKeepsNodegroupSG(t *testing.T) {
+	t.Parallel()
+	f := newEKSServiceFixture(t)
+	f.sg.createIDs = []string{"sg-old-cp", "sg-old-ng"}
+	seedActiveClusterWithToken(t, f, "c1")
+	f.sg.existing[ClusterNodegroupSGName("c1")+"|vpc-aaa"] = "sg-old-ng"
+	markWorkersReady(t, f, "c1", "ng1", 1)
+
+	_, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 1), testAccountID)
+	require.NoError(t, err)
+	f.svc.WaitLaunches()
+
+	require.Len(t, f.worker.runCalls, 1)
+	assert.Equal(t, []string{"sg-old-ng"}, aws.StringValueSlice(f.worker.runCalls[0].SecurityGroupIds))
+	assert.Len(t, f.sg.authorizeCalls, 6, "the cp<->ng cross rules are still ensured")
+}
+
 func TestCreateNodegroup_HappyPath(t *testing.T) {
 	t.Parallel()
 	f := newEKSServiceFixture(t)
@@ -382,8 +401,14 @@ func TestCreateNodegroup_HappyPath(t *testing.T) {
 		}
 	}
 
-	// SG ingress rules were authorized.
+	// A cluster with no SG yet gets the cluster SG, created once with its self
+	// rule, and every worker carries it.
+	require.Len(t, f.sg.createCalls, 1)
+	clusterSG := f.sg.existing[aws.StringValue(f.sg.createCalls[0].GroupName)+"|vpc-aaa"]
 	assert.NotEmpty(t, f.sg.authorizeCalls)
+	for _, call := range f.worker.runCalls {
+		assert.Equal(t, []string{clusterSG}, aws.StringValueSlice(call.SecurityGroupIds))
+	}
 
 	// Duplicate create → ResourceInUseException.
 	_, err = f.svc.CreateNodegroup(context.Background(), createNGInput("c1", "ng1", 2), testAccountID)

spinifex/domains/eks/cluster/private_endpoint.go

@@ -30,7 +30,7 @@ func EnsurePrivateEndpointENI(ctx context.Context,
 	vpcSvc k3sVPCProvisioner,
 	sgp sgProvisioner,
 	resolver SubnetVPCResolver,
-	accountID, clusterName, subnetID, vpcID string,
+	accountID, clusterName, subnetID, vpcID, clusterSGID string,
 ) (*PrivateEndpointENI, error) {
 	if accountID == "" {
 		return nil, errors.New("eks: EnsurePrivateEndpointENI empty account id")
@@ -41,6 +41,9 @@ func EnsurePrivateEndpointENI(ctx context.Context,
 	if vpcID == "" {
 		return nil, errors.New("eks: EnsurePrivateEndpointENI empty vpc id")
 	}
+	if clusterSGID == "" {
+		return nil, errors.New("eks: EnsurePrivateEndpointENI empty cluster SG id")
+	}
 
 	vpcCIDR, err := resolver.GetVPCCIDR(ctx, accountID, vpcID)
 	if err != nil {
@@ -55,7 +58,7 @@ func EnsurePrivateEndpointENI(ctx context.Context,
 	eniOut, err := vpcSvc.CreateNetworkInterface(ctx, &ec2.CreateNetworkInterfaceInput{
 		SubnetId:    aws.String(subnetID),
 		Description: aws.String("EKS private-endpoint ENI for " + clusterName),
-		Groups:      aws.StringSlice([]string{sgID}),
+		Groups:      aws.StringSlice([]string{sgID, clusterSGID}),
 		TagSpecifications: []*ec2.TagSpecification{{
 			ResourceType: aws.String("network-interface"),
 			Tags: []*ec2.Tag{

spinifex/domains/eks/cluster/private_endpoint_test.go

@@ -132,30 +132,32 @@ func TestEnsurePrivateEndpointENI_HappyPath(t *testing.T) {
 	sgp := newFakeSGProvisioner()
 	sgp.createIDs = []string{"sg-pe-001"}
 
-	pe, err := EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "111122223333", "alpha", "subnet-aaa", "vpc-aaa")
+	pe, err := EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "111122223333", "alpha", "subnet-aaa", "vpc-aaa", "sg-cluster")
 	require.NoError(t, err)
 	assert.Equal(t, "eni-pe-001", pe.ENIID)
 	assert.Equal(t, "10.0.1.50", pe.ENIIP)
 	assert.Equal(t, "02:0a:01:23:45:67", pe.ENIMac)
 	assert.Equal(t, "sg-pe-001", pe.SGID)
 
-	// ENI must be created in the customer subnet, bound to the private-endpoint SG.
+	// ENI must be created in the customer subnet, bound to the private-endpoint SG
+	// and the cluster SG.
 	require.Len(t, vpcSvc.createCalls, 1)
 	createIn := vpcSvc.createCalls[0]
 	assert.Equal(t, "subnet-aaa", aws.StringValue(createIn.SubnetId))
-	require.Len(t, createIn.Groups, 1)
-	assert.Equal(t, "sg-pe-001", aws.StringValue(createIn.Groups[0]))
+	assert.Equal(t, []string{"sg-pe-001", "sg-cluster"}, aws.StringValueSlice(createIn.Groups))
 }
 
 func TestEnsurePrivateEndpointENI_EmptyInputsRejected(t *testing.T) {
 	t.Parallel()
 	vpcSvc := &fakeK3sVPC{}
 	sgp := newFakeSGProvisioner()
-	_, err := EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "", "alpha", "subnet-aaa", "vpc-aaa")
+	_, err := EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "", "alpha", "subnet-aaa", "vpc-aaa", "sg-cluster")
 	require.Error(t, err)
-	_, err = EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "111122223333", "alpha", "", "vpc-aaa")
+	_, err = EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "111122223333", "alpha", "", "vpc-aaa", "sg-cluster")
 	require.Error(t, err)
-	_, err = EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "111122223333", "alpha", "subnet-aaa", "")
+	_, err = EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "111122223333", "alpha", "subnet-aaa", "", "sg-cluster")
+	require.Error(t, err)
+	_, err = EnsurePrivateEndpointENI(context.Background(), vpcSvc, sgp, fakeSubnetResolver{}, "111122223333", "alpha", "subnet-aaa", "vpc-aaa", "")
 	require.Error(t, err)
 	assert.Empty(t, vpcSvc.createCalls)
 }

spinifex/domains/eks/cluster/security_groups.go

@@ -2,6 +2,8 @@ package cluster
 
 import (
 	"context"
+	"crypto/rand"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"log/slog"
@@ -40,6 +42,7 @@ const clusterEKSAccountTagKey = "spinifex:eks-cluster-account"
 const clusterEKSNodegroupTagKey = "spinifex:eks-nodegroup"
 
 const (
+	clusterEKSRoleCluster         = "cluster"
 	clusterEKSRoleControlPlane    = "control-plane"
 	clusterEKSRoleNodegroup       = "nodegroup"
 	clusterEKSRolePrivateEndpoint = "private-endpoint"
@@ -62,7 +65,101 @@ func ClusterNodegroupSGName(clusterName string) string {
 	return fmt.Sprintf("eks-cluster-%s-nodegroup-sg", clusterName)
 }
 
-// EnsureClusterSGs creates or reuses the control-plane and nodegroup SGs in the customer VPC.
+// clusterSGNamePrefix starts the name of the AWS-style cluster security group,
+// eks-cluster-sg-<cluster>-<id>.
+const clusterSGNamePrefix = "eks-cluster-sg-"
+
+// EnsureClusterSG creates or reuses the cluster security group in the customer
+// VPC: tagged as EKS tags its primary SG, inbound all from itself, outbound all
+// (the VPC service's default egress). It is found by tag, so the random name
+// suffix never has to be stored. Workers and the private-endpoint ENI carry it.
+func EnsureClusterSG(ctx context.Context, sgp sgProvisioner, accountID, clusterName, vpcID string) (string, error) {
+	if clusterName == "" {
+		return "", errors.New("eks: EnsureClusterSG empty cluster name")
+	}
+	if vpcID == "" {
+		return "", errors.New("eks: EnsureClusterSG empty vpc id")
+	}
+	id, err := lookupClusterSG(ctx, sgp, accountID, vpcID, clusterName)
+	if err != nil {
+		return "", err
+	}
+	if id == "" {
+		suffix := make([]byte, 8)
+		if _, err := rand.Read(suffix); err != nil {
+			return "", fmt.Errorf("generate cluster SG name: %w", err)
+		}
+		name := clusterSGNamePrefix + clusterName + "-" + hex.EncodeToString(suffix)
+		out, err := sgp.CreateSecurityGroup(ctx, &ec2.CreateSecurityGroupInput{
+			GroupName:   aws.String(name),
+			Description: aws.String(fmt.Sprintf("EKS created security group applied to ENI that is attached to EKS Control Plane master nodes, as well as any managed workloads (cluster %s)", clusterName)),
+			VpcId:       aws.String(vpcID),
+			TagSpecifications: []*ec2.TagSpecification{{
+				ResourceType: aws.String("security-group"),
+				Tags: []*ec2.Tag{
+					{Key: aws.String("Name"), Value: aws.String(name)},
+					{Key: aws.String("kubernetes.io/cluster/" + clusterName), Value: aws.String("owned")},
+					{Key: aws.String("aws:eks:cluster-name"), Value: aws.String(clusterName)},
+					{Key: aws.String(tags.ManagedByKey), Value: aws.String(tags.ManagedByEKS)},
+					{Key: aws.String(clusterEKSClusterTagKey), Value: aws.String(clusterName)},
+					{Key: aws.String(clusterEKSRoleTagKey), Value: aws.String(clusterEKSRoleCluster)},
+				},
+			}},
+		}, accountID)
+		if err != nil {
+			return "", fmt.Errorf("create cluster SG %s: %w", name, err)
+		}
+		if out == nil || aws.StringValue(out.GroupId) == "" {
+			return "", fmt.Errorf("eks: CreateSecurityGroup returned empty GroupId for %s", name)
+		}
+		id = aws.StringValue(out.GroupId)
+	}
+	_, err = sgp.AuthorizeSecurityGroupIngress(ctx, &ec2.AuthorizeSecurityGroupIngressInput{
+		GroupId: aws.String(id),
+		IpPermissions: []*ec2.IpPermission{{
+			IpProtocol: aws.String("-1"),
+			UserIdGroupPairs: []*ec2.UserIdGroupPair{{
+				GroupId:     aws.String(id),
+				Description: aws.String("EKS cluster security group, all traffic from itself"),
+			}},
+		}},
+	}, accountID)
+	if err != nil && !awserrors.IsErrorCode(err, awserrors.ErrorInvalidPermissionDuplicate) {
+		return "", fmt.Errorf("authorize self ingress on cluster SG %s: %w", id, err)
+	}
+	return id, nil
+}
+
+// EnsureWorkerSG returns the SG nodegroup workers launch with. A cluster created
+// before the cluster SG existed keeps its nodegroup SG and control-plane rules
+// until it is recreated; every other cluster gets the SG recorded on it,
+// clusterSGID, or a freshly ensured one when none is recorded yet.
+func EnsureWorkerSG(ctx context.Context, sgp sgProvisioner, accountID, clusterName, vpcID, clusterSGID string) (string, error) {
+	if vpcID == "" {
+		return "", errors.New("eks: EnsureWorkerSG empty vpc id")
+	}
+	legacy, err := lookupSGByName(ctx, sgp, accountID, vpcID, ClusterNodegroupSGName(clusterName))
+	if err != nil {
+		return "", err
+	}
+	if legacy == "" {
+		if clusterSGID != "" {
+			return clusterSGID, nil
+		}
+		return EnsureClusterSG(ctx, sgp, accountID, clusterName, vpcID)
+	}
+	cpSGID, ngSGID, err := EnsureClusterSGs(ctx, sgp, accountID, clusterName, vpcID)
+	if err != nil {
+		return "", err
+	}
+	if err := EnsureNodegroupSGRules(ctx, sgp, accountID, clusterName, cpSGID, ngSGID); err != nil {
+		return "", err
+	}
+	return ngSGID, nil
+}
+
+// EnsureClusterSGs creates or reuses the control-plane and nodegroup SGs: the
+// control-plane SG in the system CP VPC, and both in a pre-cluster-SG customer VPC.
 // Idempotent; the two creates are independent (DeleteCluster handles partial teardown).
 func EnsureClusterSGs(ctx context.Context, sgp sgProvisioner, accountID, clusterName, vpcID string) (controlPlaneSGID, nodegroupSGID string, err error) {
 	if clusterName == "" {
@@ -91,16 +188,16 @@ func EnsureClusterSGs(ctx context.Context, sgp sgProvisioner, accountID, cluster
 	return controlPlaneSGID, nodegroupSGID, nil
 }
 
-// DeleteClusterSGs removes the cluster control-plane and nodegroup SGs. AWS
-// refuses to delete a security group still referenced by ANY other SG's rule —
-// not just the cp<->ng cross-reference (EnsureNodegroupSGRules) but the VPC's
-// LBC/ALB SG, the private-endpoint SG, and user/TF SGs, in either direction
-// (ingress or egress). To break every cycle order-independently, all ingress and
-// egress rules are revoked on every non-default SG in the VPC first, then the two
-// cluster-owned SGs are deleted. Referrer SGs are owned and deleted by their own
-// teardown — only the cluster SGs are deleted here. Missing SGs are no-ops; the
-// first error is returned after the full sweep.
-func DeleteClusterSGs(ctx context.Context, sgp sgProvisioner, accountID, clusterName, vpcID string) error {
+// DeleteClusterSGs removes the cluster's SGs: the stored cluster SG, the
+// control-plane and nodegroup SGs of a cluster created before the cluster SG, and
+// the LBC-orphaned SGs. AWS refuses to delete an SG still referenced by another
+// SG's rule, in either direction, so before any delete every rule on a
+// cluster-owned SG is revoked and, on any other SG in the VPC, only the rules that
+// reference a cluster-owned SG. Unrelated customer SGs keep everything else.
+// clusterSGID is the id recorded on the cluster; ownership never comes from a
+// customer-settable tag. Missing SGs are no-ops; the first error is returned
+// after the full sweep.
+func DeleteClusterSGs(ctx context.Context, sgp sgProvisioner, accountID, clusterName, vpcID, clusterSGID string) error {
 	if clusterName == "" {
 		return errors.New("eks: DeleteClusterSGs empty cluster name")
 	}
@@ -115,9 +212,6 @@ func DeleteClusterSGs(ctx context.Context, sgp sgProvisioner, accountID, cluster
 		}
 	}
 
-	// Break every cross-reference in the VPC before deleting any SG. Revoking
-	// both directions on every non-default SG guarantees no rule still points at
-	// a cluster SG, so the deletes below succeed regardless of order.
 	out, err := sgp.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{
 		Filters: []*ec2.Filter{
 			{Name: aws.String("vpc-id"), Values: aws.StringSlice([]string{vpcID})},
@@ -127,12 +221,49 @@ func DeleteClusterSGs(ctx context.Context, sgp sgProvisioner, accountID, cluster
 		slog.WarnContext(ctx, "DeleteClusterSGs: describe SGs failed", "vpc", vpcID, "err", err)
 		record(fmt.Errorf("describe SGs in vpc %s: %w", vpcID, err))
 	}
-	// A failed describe returns a nil out; fall back to an empty set so the two
-	// cross-reference sweeps below become no-ops while the targeted deletes still run.
+	// A failed describe returns a nil out; fall back to an empty set so the
+	// sweeps below become no-ops while the targeted deletes still run.
 	var sgs []*ec2.SecurityGroup
 	if out != nil {
 		sgs = out.SecurityGroups
 	}
+
+	// owned is every SG this cluster owns, in delete order: LBC-orphaned SGs
+	// first (they reference the cluster SG), then the stored cluster SG, then the
+	// pre-cluster-SG named ones.
+	var owned []string
+	isOwned := map[string]bool{}
+	own := func(id string) {
+		if id != "" && !isOwned[id] {
+			isOwned[id] = true
+			owned = append(owned, id)
+		}
+	}
+	for _, g := range sgs {
+		if g != nil && g.GroupId != nil && aws.StringValue(g.GroupName) != "default" && sgHasLBCClusterTag(g, clusterName) {
+			own(*g.GroupId)
+		}
+	}
+	own(clusterSGID)
+	// With no stored id (a create that died before recording it) the SG is still
+	// ours when it carries the tags only the service can set.
+	if clusterSGID == "" {
+		for _, g := range sgs {
+			if g != nil && g.GroupId != nil && sgIsClusterSG(g, clusterName) {
+				own(*g.GroupId)
+			}
+		}
+	}
+	for _, name := range []string{ClusterControlPlaneSGName(clusterName), ClusterNodegroupSGName(clusterName)} {
+		id, lookupErr := lookupSGByName(ctx, sgp, accountID, vpcID, name)
+		if lookupErr != nil {
+			slog.WarnContext(ctx, "DeleteClusterSGs: SG lookup failed", "sg", name, "err", lookupErr)
+			record(lookupErr)
+			continue
+		}
+		own(id)
+	}
+
 	for _, g := range sgs {
 		if g == nil || g.GroupId == nil {
 			continue
@@ -142,41 +273,19 @@ func DeleteClusterSGs(ctx context.Context, sgp sgProvisioner, accountID, cluster
 		if aws.StringValue(g.GroupName) == "default" {
 			continue
 		}
-		if revokeErr := revokeAllSGRules(ctx, sgp, accountID, g); revokeErr != nil {
+		var revokeErr error
+		if isOwned[*g.GroupId] {
+			revokeErr = revokeAllSGRules(ctx, sgp, accountID, g)
+		} else {
+			revokeErr = revokeRulesReferencing(ctx, sgp, accountID, g, isOwned)
+		}
+		if revokeErr != nil {
 			slog.WarnContext(ctx, "DeleteClusterSGs: revoke rules failed", "sg", aws.StringValue(g.GroupId), "err", revokeErr)
 			record(revokeErr)
 		}
 	}
 
-	// Reap LBC-orphaned SGs (k8s-traffic-toc-*): the in-cluster AWS LoadBalancer
-	// Controller creates these in the customer VPC and spinifex never tracks them,
-	// so nothing else deletes them and they pin the VPC on DependencyViolation.
-	// Match by the LBC ownership tag; rules were already revoked above. Runs after
-	// the matching ALB reap, so the LB no longer references the SG.
-	for _, g := range sgs {
-		if g == nil || g.GroupId == nil || aws.StringValue(g.GroupName) == "default" {
-			continue
-		}
-		if !sgHasLBCClusterTag(g, clusterName) {
-			continue
-		}
-		if delErr := deleteSGAwaitingDetach(ctx, sgp, accountID, aws.StringValue(g.GroupId)); delErr != nil {
-			slog.WarnContext(ctx, "DeleteClusterSGs: LBC SG delete failed", "sg", aws.StringValue(g.GroupId), "err", delErr)
-			record(fmt.Errorf("delete LBC SG %s: %w", aws.StringValue(g.GroupId), delErr))
-		}
-	}
-
-	// Delete only the cluster-owned SGs.
-	for _, name := range []string{ClusterControlPlaneSGName(clusterName), ClusterNodegroupSGName(clusterName)} {
-		id, lookupErr := lookupSGByName(ctx, sgp, accountID, vpcID, name)
-		if lookupErr != nil {
-			slog.WarnContext(ctx, "DeleteClusterSGs: SG lookup failed", "sg", name, "err", lookupErr)
-			record(lookupErr)
-			continue
-		}
-		if id == "" {
-			continue
-		}
+	for _, id := range owned {
 		if delErr := deleteSGAwaitingDetach(ctx, sgp, accountID, id); delErr != nil {
 			slog.WarnContext(ctx, "DeleteClusterSGs: SG delete failed", "sg", id, "err", delErr)
 			record(fmt.Errorf("delete SG %s: %w", id, delErr))
@@ -248,6 +357,57 @@ func revokeAllSGRules(ctx context.Context, sgp sgProvisioner, accountID string,
 	return nil
 }
 
+// revokeRulesReferencing revokes, on a SG the cluster does not own, only the
+// rules that reference an owned SG, and only those references. Every other rule
+// on the SG is left in place.
+func revokeRulesReferencing(ctx context.Context, sgp sgProvisioner, accountID string, g *ec2.SecurityGroup, owned map[string]bool) error {
+	sgID := aws.StringValue(g.GroupId)
+	if perms := referencingPerms(g.IpPermissions, owned); len(perms) > 0 {
+		if _, err := sgp.RevokeSecurityGroupIngress(ctx, &ec2.RevokeSecurityGroupIngressInput{
+			GroupId:       g.GroupId,
+			IpPermissions: perms,
+		}, accountID); err != nil && !awserrors.IsNotFound(err) {
+			return fmt.Errorf("revoke ingress on %s: %w", sgID, err)
+		}
+	}
+	if perms := referencingPerms(g.IpPermissionsEgress, owned); len(perms) > 0 {
+		if _, err := sgp.RevokeSecurityGroupEgress(ctx, &ec2.RevokeSecurityGroupEgressInput{
+			GroupId:       g.GroupId,
+			IpPermissions: perms,
+		}, accountID); err != nil && !awserrors.IsNotFound(err) {
+			return fmt.Errorf("revoke egress on %s: %w", sgID, err)
+		}
+	}
+	return nil
+}
+
+// referencingPerms returns, for each permission that names an owned SG, a copy
+// carrying only those group references.
+func referencingPerms(perms []*ec2.IpPermission, owned map[string]bool) []*ec2.IpPermission {
+	var out []*ec2.IpPermission
+	for _, p := range perms {
+		if p == nil {
+			continue
+		}
+		var pairs []*ec2.UserIdGroupPair
+		for _, pair := range p.UserIdGroupPairs {
+			if pair != nil && owned[aws.StringValue(pair.GroupId)] {
+				pairs = append(pairs, pair)
+			}
+		}
+		if len(pairs) == 0 {
+			continue
+		}
+		out = append(out, &ec2.IpPermission{
+			IpProtocol:       p.IpProtocol,
+			FromPort:         p.FromPort,
+			ToPort:           p.ToPort,
+			UserIdGroupPairs: pairs,
+		})
+	}
+	return out
+}
+
 // EnsurePrivateEndpointSG creates (or reuses) the customer-VPC SG for the
 // private-endpoint ENI and admits the customer VPC CIDR on :443 (kubectl/SDK) and
 // :6443, so in-VPC workers + kubectl reach the apiserver via the Set A NIC. :6443
@@ -486,6 +646,42 @@ func sgHasLBCClusterTag(g *ec2.SecurityGroup, clusterName string) bool {
 	})
 }
 
+// sgIsClusterSG reports whether g carries every tag the service stamps on this
+// cluster's SG. aws:eks:cluster-name is reserved, so a customer cannot set it.
+func sgIsClusterSG(g *ec2.SecurityGroup, clusterName string) bool {
+	have := map[string]string{}
+	for _, tag := range g.Tags {
+		have[aws.StringValue(tag.Key)] = aws.StringValue(tag.Value)
+	}
+	return have[tags.ManagedByKey] == tags.ManagedByEKS &&
+		have["aws:eks:cluster-name"] == clusterName &&
+		have[clusterEKSClusterTagKey] == clusterName &&
+		have[clusterEKSRoleTagKey] == clusterEKSRoleCluster
+}
+
+// lookupClusterSG finds a cluster SG that an earlier, interrupted create made
+// before its id was recorded. Only the tags the service alone can set count.
+func lookupClusterSG(ctx context.Context, sgp sgProvisioner, accountID, vpcID, clusterName string) (string, error) {
+	out, err := sgp.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{
+		Filters: []*ec2.Filter{
+			{Name: aws.String("vpc-id"), Values: aws.StringSlice([]string{vpcID})},
+			{Name: aws.String("tag:" + tags.ManagedByKey), Values: aws.StringSlice([]string{tags.ManagedByEKS})},
+			{Name: aws.String("tag:aws:eks:cluster-name"), Values: aws.StringSlice([]string{clusterName})},
+			{Name: aws.String("tag:" + clusterEKSClusterTagKey), Values: aws.StringSlice([]string{clusterName})},
+			{Name: aws.String("tag:" + clusterEKSRoleTagKey), Values: aws.StringSlice([]string{clusterEKSRoleCluster})},
+		},
+	}, accountID)
+	if err != nil {
+		return "", fmt.Errorf("describe cluster SG for %s: %w", clusterName, err)
+	}
+	for _, g := range out.SecurityGroups {
+		if g != nil && g.GroupId != nil && sgIsClusterSG(g, clusterName) {
+			return *g.GroupId, nil
+		}
+	}
+	return "", nil
+}
+
 func lookupSGByName(ctx context.Context, sgp sgProvisioner, accountID, vpcID, name string) (string, error) {
 	out, err := sgp.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{
 		Filters: []*ec2.Filter{

spinifex/domains/eks/cluster/security_groups_test.go

@@ -3,6 +3,7 @@ package cluster
 import (
 	"context"
 	"errors"
+	"strconv"
 	"strings"
 	"sync"
 	"testing"
@@ -52,6 +53,11 @@ type fakeSGProvisioner struct {
 	// nodegroup SG IDs.
 	createIDs []string
 
+	// uniqueIDs makes CreateSecurityGroup hand out sg-1, sg-2, ... once createIDs
+	// is exhausted, so a flow that creates several SGs can tell them apart.
+	uniqueIDs bool
+	nextSeq   int
+
 	createErr    error
 	describeErr  error
 	deleteErr    error
@@ -72,13 +78,23 @@ func (f *fakeSGProvisioner) CreateSecurityGroup(_ context.Context, input *ec2.Cr
 		return nil, f.createErr
 	}
 	id := "sg-default"
-	if len(f.createIDs) > 0 {
+	switch {
+	case len(f.createIDs) > 0:
 		id = f.createIDs[0]
 		f.createIDs = f.createIDs[1:]
+	case f.uniqueIDs:
+		f.nextSeq++
+		id = "sg-" + strconv.Itoa(f.nextSeq)
 	}
 	if input.GroupName != nil && input.VpcId != nil {
 		f.existing[*input.GroupName+"|"+*input.VpcId] = id
 	}
+	for _, spec := range input.TagSpecifications {
+		if f.tagsByID == nil {
+			f.tagsByID = map[string][]*ec2.Tag{}
+		}
+		f.tagsByID[id] = append(f.tagsByID[id], spec.Tags...)
+	}
 	return &ec2.CreateSecurityGroupOutput{GroupId: aws.String(id)}, nil
 }
 
@@ -104,15 +120,18 @@ func (f *fakeSGProvisioner) DescribeSecurityGroups(_ context.Context, input *ec2
 		return out, nil
 	}
 	var name, vpc string
+	tagFilters := map[string]string{}
 	for _, filt := range input.Filters {
 		if filt == nil || filt.Name == nil || len(filt.Values) == 0 || filt.Values[0] == nil {
 			continue
 		}
-		switch *filt.Name {
-		case "group-name":
+		switch {
+		case *filt.Name == "group-name":
 			name = *filt.Values[0]
-		case "vpc-id":
+		case *filt.Name == "vpc-id":
 			vpc = *filt.Values[0]
+		case strings.HasPrefix(*filt.Name, "tag:"):
+			tagFilters[strings.TrimPrefix(*filt.Name, "tag:")] = *filt.Values[0]
 		}
 	}
 	out := &ec2.DescribeSecurityGroupsOutput{}
@@ -123,6 +142,9 @@ func (f *fakeSGProvisioner) DescribeSecurityGroups(_ context.Context, input *ec2
 			if len(parts) != 2 || parts[1] != vpc {
 				continue
 			}
+			if !sgTagsMatch(f.tagsByID[id], tagFilters) {
+				continue
+			}
 			out.SecurityGroups = append(out.SecurityGroups, &ec2.SecurityGroup{
 				GroupId:             aws.String(id),
 				GroupName:           aws.String(parts[0]),
@@ -267,7 +289,7 @@ func TestDeleteClusterSGs_DeletesBothExisting(t *testing.T) {
 	sgp.existing["eks-cluster-alpha-control-plane-sg|vpc-aaa"] = "sg-existing-cp"
 	sgp.existing["eks-cluster-alpha-nodegroup-sg|vpc-aaa"] = "sg-existing-ng"
 
-	err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa")
+	err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa", "")
 	require.NoError(t, err)
 
 	require.Len(t, sgp.deleteCalls, 2)
@@ -291,7 +313,7 @@ func TestDeleteClusterSGs_ReapsLBCTaggedSGs(t *testing.T) {
 		"sg-lbc-other": {{Key: aws.String(lbcClusterOwnershipTagKey), Value: aws.String("beta")}},
 	}
 
-	err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa")
+	err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa", "")
 	require.NoError(t, err)
 
 	deleted := map[string]bool{}
@@ -309,7 +331,7 @@ func TestDeleteClusterSGs_MissingSGsNoOp(t *testing.T) {
 	t.Parallel()
 	sgp := newFakeSGProvisioner()
 
-	err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa")
+	err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa", "")
 	require.NoError(t, err)
 	assert.Empty(t, sgp.deleteCalls, "no delete calls expected when SGs already absent")
 }
@@ -323,7 +345,7 @@ func TestDeleteClusterSGs_FirstErrorSurfacedSweepContinues(t *testing.T) {
 	// after a single attempt per SG (the retry path is covered separately).
 	sgp.deleteErr = errors.New("vpcd unavailable")
 
-	err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa")
+	err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa", "")
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "sg-existing-cp")
 	assert.Len(t, sgp.deleteCalls, 2, "both delete calls should be attempted despite first error")
@@ -348,7 +370,7 @@ func TestDeleteClusterSGs_RetriesDependencyViolation(t *testing.T) {
 		sgp.existing["eks-cluster-alpha-nodegroup-sg|vpc-aaa"] = "sg-ng"
 		sgp.deleteFailsBefore = map[string]int{"sg-ng": 2} // 2 DepViolations, then ok
 
-		err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa")
+		err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa", "")
 		require.NoError(t, err, "delete must succeed once the ENI detaches")
 		assert.GreaterOrEqual(t, len(sgp.deleteCalls), 3, "retried past the transient violations")
 	})
@@ -359,7 +381,7 @@ func TestDeleteClusterSGs_RetriesDependencyViolation(t *testing.T) {
 		sgp.existing["eks-cluster-alpha-nodegroup-sg|vpc-aaa"] = "sg-ng"
 		sgp.deleteErr = errors.New("DependencyViolation: still referenced")
 
-		err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa")
+		err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa", "")
 		require.Error(t, err, "a persistent DependencyViolation must surface, not leak the SG")
 		assert.Contains(t, err.Error(), "sg-ng")
 	})
@@ -382,7 +404,7 @@ func TestDeleteClusterSGs_RevokesCrossRefsBeforeDelete(t *testing.T) {
 	// AWS refuses to delete an SG still referenced by a sibling's rule.
 	sgp.enforceDepViolation = true
 
-	err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa")
+	err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa", "")
 	require.NoError(t, err, "revoke-then-delete must clear the cp<->ng cycle so both SGs delete")
 
 	require.Len(t, sgp.revokeCalls, 2, "ingress on both SGs must be revoked before delete")
@@ -418,7 +440,7 @@ func TestDeleteClusterSGs_RevokesEgressAndNonClusterReferrers(t *testing.T) {
 	}
 	sgp.enforceDepViolation = true
 
-	err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa")
+	err := DeleteClusterSGs(context.Background(), sgp, "111122223333", "alpha", "vpc-aaa", "")
 	require.NoError(t, err, "revoking both directions on all non-default SGs must clear every cycle")
 
 	revokedEgress := map[string]bool{}
@@ -591,3 +613,64 @@ func assertSGCreateTagged(t *testing.T, in *ec2.CreateSecurityGroupInput, name,
 	assert.Equal(t, clusterName, got[clusterEKSClusterTagKey])
 	assert.Equal(t, role, got[clusterEKSRoleTagKey])
 }
+
+// sgTagsMatch reports whether tags carry every key=value in want.
+func sgTagsMatch(tags []*ec2.Tag, want map[string]string) bool {
+	have := sgTagMap(tags)
+	for k, v := range want {
+		if have[k] != v {
+			return false
+		}
+	}
+	return true
+}
+
+func sgTagMap(tags []*ec2.Tag) map[string]string {
+	m := map[string]string{}
+	for _, tg := range tags {
+		m[aws.StringValue(tg.Key)] = aws.StringValue(tg.Value)
+	}
+	return m
+}
+
+func TestEnsureClusterSG_EmptyInputsRejected(t *testing.T) {
+	t.Parallel()
+	sg := newFakeSGProvisioner()
+	_, err := EnsureClusterSG(context.Background(), sg, testAccountID, "", "vpc-aaa")
+	require.Error(t, err)
+	_, err = EnsureClusterSG(context.Background(), sg, testAccountID, "c1", "")
+	require.Error(t, err)
+	assert.Empty(t, sg.createCalls)
+}
+
+func TestEnsureClusterSG_SecondCallReusesTheTaggedSG(t *testing.T) {
+	t.Parallel()
+	sg := newFakeSGProvisioner()
+	sg.uniqueIDs = true
+
+	first, err := EnsureClusterSG(context.Background(), sg, testAccountID, "c1", "vpc-aaa")
+	require.NoError(t, err)
+	second, err := EnsureClusterSG(context.Background(), sg, testAccountID, "c1", "vpc-aaa")
+	require.NoError(t, err)
+
+	assert.Equal(t, first, second)
+	assert.Len(t, sg.createCalls, 1)
+}
+
+func TestEnsureClusterSG_DuplicateSelfRuleTolerated(t *testing.T) {
+	t.Parallel()
+	sg := newFakeSGProvisioner()
+	sg.authorizeErr = errors.New(awserrors.ErrorInvalidPermissionDuplicate)
+
+	_, err := EnsureClusterSG(context.Background(), sg, testAccountID, "c1", "vpc-aaa")
+	require.NoError(t, err)
+}
+
+func TestEnsureClusterSG_AuthorizeErrorSurfaced(t *testing.T) {
+	t.Parallel()
+	sg := newFakeSGProvisioner()
+	sg.authorizeErr = errors.New("boom")
+
+	_, err := EnsureClusterSG(context.Background(), sg, testAccountID, "c1", "vpc-aaa")
+	require.Error(t, err)
+}

spinifex/domains/eks/cluster/service_impl.go

@@ -977,13 +977,13 @@ func (s *EKSServiceImpl) launchClusterInfra(ctx context.Context, lc clusterLaunc
 		return
 	}
 
-	// The customer-facing "cluster security group" AWS auto-creates lives in the
-	// caller's own VPC, not the system-managed CP VPC above — otherwise the
-	// tenant's own DescribeSecurityGroups can never resolve it. Idempotent with
-	// launchNodegroupInfra's own EnsureClusterSGs call, which reuses this same id.
-	custClusterSG, _, err := EnsureClusterSGs(ctx, s.deps.VPCSG, accountID, name, lc.vpcID)
+	// The customer-facing cluster security group lives in the caller's own VPC,
+	// not the system-managed CP VPC above, so the tenant's own
+	// DescribeSecurityGroups resolves it. Workers and the private-endpoint ENI
+	// carry it. Idempotent with the nodegroup launch, which finds it by tag.
+	custClusterSG, err := EnsureClusterSG(ctx, s.deps.VPCSG, accountID, name, lc.vpcID)
 	if err != nil {
-		s.failClusterLaunch(ctx, acctKV, name, accountID, meta, "ensure customer-vpc cluster SGs", err)
+		s.failClusterLaunch(ctx, acctKV, name, accountID, meta, "ensure customer-vpc cluster SG", err)
 		return
 	}
 	meta.ResourcesVpcConfig.ClusterSecurityGroupId = custClusterSG
@@ -1002,7 +1002,7 @@ func (s *EKSServiceImpl) launchClusterInfra(ctx context.Context, lc clusterLaunc
 	// NIC and proxies to the CP target group with no data-plane change.
 	var crossAccountENIs []systeminstance.ExtraENIInput
 	if privateAccess {
-		pe, perr := EnsurePrivateEndpointENI(ctx, s.deps.VPCK3s, s.deps.VPCSG, s.deps.VPCSubnet, accountID, name, lc.subnetIDs[0], lc.vpcID)
+		pe, perr := EnsurePrivateEndpointENI(ctx, s.deps.VPCK3s, s.deps.VPCSG, s.deps.VPCSubnet, accountID, name, lc.subnetIDs[0], lc.vpcID, custClusterSG)
 		if perr != nil {
 			s.failClusterLaunch(ctx, acctKV, name, accountID, meta, "ensure private endpoint ENI", perr)
 			return
@@ -1621,7 +1621,7 @@ func (s *EKSServiceImpl) teardownClusterInfra(ctx context.Context, accountID, na
 		// New topology: SGs + IGW + subnets + route tables + NAT GW + VPC all live
 		// in the managed CP VPC under the system account. DeleteClusterCPVPC removes
 		// the IGW + VPC after the SGs, so SG cleanup must precede it.
-		cpSGErr := DeleteClusterSGs(ctx, s.deps.VPCSG, infraAcct, name, meta.ManagedCPVPC.VpcId)
+		cpSGErr := DeleteClusterSGs(ctx, s.deps.VPCSG, infraAcct, name, meta.ManagedCPVPC.VpcId, "")
 		if cpSGErr != nil {
 			fail(stepSecurityGroups, fmt.Errorf("delete cluster SGs: %w", cpSGErr))
 		}
@@ -1629,7 +1629,7 @@ func (s *EKSServiceImpl) teardownClusterInfra(ctx context.Context, accountID, na
 		// worker<->control-plane networking; reclaim them here too, or they orphan
 		// (cross-referencing each other) and pin the customer VPC on destroy.
 		if meta.ResourcesVpcConfig != nil && meta.ResourcesVpcConfig.VpcId != "" {
-			if err := DeleteClusterSGs(ctx, s.deps.VPCSG, accountID, name, meta.ResourcesVpcConfig.VpcId); err != nil {
+			if err := DeleteClusterSGs(ctx, s.deps.VPCSG, accountID, name, meta.ResourcesVpcConfig.VpcId, meta.ResourcesVpcConfig.ClusterSecurityGroupId); err != nil {
 				fail(stepSecurityGroups, fmt.Errorf("delete customer-VPC cluster SGs: %w", err))
 			}
 		}
@@ -1653,7 +1653,7 @@ func (s *EKSServiceImpl) teardownClusterInfra(ctx context.Context, accountID, na
 		// Legacy topology: CP lived in the customer VPC; reclaim its SGs + the
 		// cluster-owned IGW (ownership-scoped — a reused customer IGW is left
 		// intact). IGW delete must precede any VPC delete to avoid DependencyViolation.
-		if err := DeleteClusterSGs(ctx, s.deps.VPCSG, accountID, name, meta.ResourcesVpcConfig.VpcId); err != nil {
+		if err := DeleteClusterSGs(ctx, s.deps.VPCSG, accountID, name, meta.ResourcesVpcConfig.VpcId, meta.ResourcesVpcConfig.ClusterSecurityGroupId); err != nil {
 			fail(stepSecurityGroups, fmt.Errorf("delete cluster SGs: %w", err))
 		}
 		if s.deps.IGW != nil {

spinifex/domains/elbv2/haproxy.go

@@ -46,7 +46,7 @@ backend {{.Name}}
 {{- if .Redirect}}
     http-request redirect {{if .Redirect.Scheme}}scheme {{.Redirect.Scheme}}{{else}}location {{.Redirect.Location}}{{end}} code {{.Redirect.Code}}
 {{- else if .FixedResponse}}
-    http-request return status {{.FixedResponse.StatusCode}}{{if .FixedResponse.ContentType}} content-type "{{.FixedResponse.ContentType}}"{{end}}{{if .FixedResponse.Body}} string "{{.FixedResponse.Body}}"{{end}}
+    http-request return status {{.FixedResponse.StatusCode}}{{if .FixedResponse.ContentType}} content-type {{squote .FixedResponse.ContentType}}{{end}}{{if .FixedResponse.Body}} string {{squote .FixedResponse.Body}}{{end}}
 {{- else}}
     balance roundrobin{{if .HealthCheckPath}}
     option httpchk
@@ -65,7 +65,14 @@ backend {{.Name}}
 {{end}}
 `
 
-var haproxyHTTPTmpl = template.Must(template.New("haproxy").Parse(haproxyConfigTemplate))
+var haproxyHTTPTmpl = template.Must(template.New("haproxy").Funcs(template.FuncMap{"squote": haproxySingleQuote}).Parse(haproxyConfigTemplate))
+
+// haproxySingleQuote renders s as one literal HAProxy word. HAProxy expands
+// environment variables inside double quotes but not single quotes, so tenant
+// text is single-quoted with each embedded quote closed, escaped and reopened.
+func haproxySingleQuote(s string) string {
+	return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
+}
 
 // HAProxyConfig holds the full configuration for rendering the HAProxy template.
 type HAProxyConfig struct {
@@ -566,17 +573,10 @@ func validHTTPStatusCode(s string) bool {
 	return true
 }
 
-// validFixedResponseBody rejects bytes that would terminate or inject into the HAProxy directive.
+// validFixedResponseBody rejects bytes that would end the single-line HAProxy
+// directive the single-quoted body is rendered into.
 func validFixedResponseBody(s string) bool {
-	if s == "" {
-		return false
-	}
-	for _, r := range s {
-		if r == '\n' || r == '\r' || r == '"' || r == '\\' || r == '#' {
-			return false
-		}
-	}
-	return true
+	return s != "" && !strings.ContainsAny(s, "\n\r\x00")
 }
 
 // buildRedirect translates a RedirectAction into an HAProxy redirect directive.

spinifex/domains/elbv2/haproxy_test.go

@@ -279,7 +279,7 @@ func TestGenerateHAProxyConfig_FixedResponseDefault(t *testing.T) {
 	// Synthetic default backend named off the listener ARN, returning the fixed response.
 	assert.Contains(t, config, "default_backend bkdefault_lst-616760baf6a3031c3")
 	assert.Contains(t, config, "backend bkdefault_lst-616760baf6a3031c3")
-	assert.Contains(t, config, `http-request return status 404 content-type "text/plain" string "no route"`)
+	assert.Contains(t, config, `http-request return status 404 content-type 'text/plain' string 'no route'`)
 	// Host-header rule still routes to the app backend.
 	assert.Contains(t, config, "use_backend bk_tg-app1 if")
 	assert.Contains(t, config, "10.42.0.9:8080")
@@ -309,7 +309,59 @@ func TestGenerateHAProxyConfig_FixedResponseRejectsUnsafeBody(t *testing.T) {
 
 	assert.Contains(t, config, "http-request return status 200")
 	assert.NotContains(t, config, "always_true")
-	assert.NotContains(t, config, `string "evil`)
+	assert.NotContains(t, config, `string 'evil`)
+}
+
+// TestGenerateHAProxyConfig_FixedResponseIsLiteral renders the content type and
+// body in single quotes, where HAProxy performs no environment expansion, so a
+// body referencing the lb-agent's environment is served verbatim.
+func TestGenerateHAProxyConfig_FixedResponseIsLiteral(t *testing.T) {
+	t.Parallel()
+	lb := &LoadBalancerRecord{LoadBalancerID: "lb-env"}
+	listeners := []*ListenerRecord{
+		{
+			ListenerArn: "arn:lst-env",
+			Port:        80,
+			DefaultActions: []ListenerAction{
+				{Type: ActionTypeFixedResponse, FixedResponse: &FixedResponseAction{
+					StatusCode:  "200",
+					ContentType: "text/plain",
+					MessageBody: `sk=${LB_SECRET_KEY} ak=$LB_ACCESS_KEY it's "q" \ #x`,
+				}},
+			},
+		},
+	}
+
+	config, _, err := GenerateHAProxyConfigWithCerts(lb, listeners, nil, nil, "0.0.0.0", nil)
+	require.NoError(t, err)
+
+	assert.Contains(t, config, `http-request return status 200 content-type 'text/plain' string 'sk=${LB_SECRET_KEY} ak=$LB_ACCESS_KEY it'\''s "q" \ #x'`+"\n")
+	assert.NotContains(t, config, `"${LB_SECRET_KEY}`)
+}
+
+// TestGenerateHAProxyConfig_FixedResponseKeepsStoredTab keeps a stored body with
+// a tab, which is literal inside single quotes and rendered before API checks existed.
+func TestGenerateHAProxyConfig_FixedResponseKeepsStoredTab(t *testing.T) {
+	t.Parallel()
+	lb := &LoadBalancerRecord{LoadBalancerID: "lb-tab"}
+	listeners := []*ListenerRecord{
+		{
+			ListenerArn: "arn:lst-tab",
+			Port:        80,
+			DefaultActions: []ListenerAction{
+				{Type: ActionTypeFixedResponse, FixedResponse: &FixedResponseAction{
+					StatusCode:  "503",
+					ContentType: "text/html",
+					MessageBody: "<pre>\tdown</pre>",
+				}},
+			},
+		},
+	}
+
+	config, _, err := GenerateHAProxyConfigWithCerts(lb, listeners, nil, nil, "0.0.0.0", nil)
+	require.NoError(t, err)
+
+	assert.Contains(t, config, "http-request return status 503 content-type 'text/html' string '<pre>\tdown</pre>'\n")
 }
 
 // TestGenerateHAProxyConfig_RedirectSchemeDefault renders the canonical

spinifex/domains/elbv2/service_impl.go

@@ -17,6 +17,8 @@ import (
 	"strings"
 	"sync"
 	"time"
+	"unicode"
+	"unicode/utf8"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -2701,8 +2703,9 @@ func listenerActionFromSDK(a *elbv2.Action) ListenerAction {
 	return action
 }
 
-// validateListenerAction enforces the per-type action contract. Redirect fields
-// are validated here so invalid input fails at the API, not at render time.
+// validateListenerAction enforces the per-type action contract. Redirect and
+// fixed-response fields are validated here so invalid input fails at the API,
+// not at render time.
 func validateListenerAction(a ListenerAction) error {
 	if a.Type == ActionTypeRedirect {
 		if a.Redirect == nil {
@@ -2710,6 +2713,39 @@ func validateListenerAction(a ListenerAction) error {
 		}
 		return validateRedirectAction(a.Redirect)
 	}
+	if a.Type == ActionTypeFixedResponse {
+		return validateFixedResponseAction(a.FixedResponse)
+	}
+	return nil
+}
+
+// fixedResponseContentTypes are the only ContentType values ELBv2 accepts.
+var fixedResponseContentTypes = map[string]bool{
+	"text/plain":             true,
+	"text/css":               true,
+	"text/html":              true,
+	"application/javascript": true,
+	"application/json":       true,
+}
+
+// maxFixedResponseBodyLen is ELBv2's MessageBody length limit in characters.
+const maxFixedResponseBodyLen = 1024
+
+// validateFixedResponseAction rejects a ContentType ELBv2 does not allow, or a
+// MessageBody that is too long or holds control characters.
+func validateFixedResponseAction(fr *FixedResponseAction) error {
+	if fr == nil {
+		return errors.New(awserrors.ErrorMissingParameter)
+	}
+	if fr.ContentType != "" && !fixedResponseContentTypes[fr.ContentType] {
+		return errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	if utf8.RuneCountInString(fr.MessageBody) > maxFixedResponseBodyLen {
+		return errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	if strings.ContainsFunc(fr.MessageBody, unicode.IsControl) {
+		return errors.New(awserrors.ErrorInvalidParameterValue)
+	}
 	return nil
 }
 
@@ -3503,14 +3539,18 @@ func (s *ELBv2ServiceImpl) tgRecordToSDK(r *TargetGroupRecord, lbArns []string)
 		HealthCheckEnabled:         aws.Bool(r.HealthCheck.Enabled),
 		HealthCheckProtocol:        aws.String(r.HealthCheck.Protocol),
 		HealthCheckPort:            aws.String(r.HealthCheck.Port),
-		HealthCheckPath:            aws.String(r.HealthCheck.Path),
 		HealthCheckIntervalSeconds: aws.Int64(r.HealthCheck.IntervalSeconds),
 		HealthCheckTimeoutSeconds:  aws.Int64(r.HealthCheck.TimeoutSeconds),
 		HealthyThresholdCount:      aws.Int64(r.HealthCheck.HealthyThreshold),
 		UnhealthyThresholdCount:    aws.Int64(r.HealthCheck.UnhealthyThreshold),
-		Matcher: &elbv2.Matcher{
-			HttpCode: aws.String(r.HealthCheck.Matcher),
-		},
+	}
+
+	// AWS omits both keys on TCP target groups, which have no path or matcher.
+	if r.HealthCheck.Path != "" {
+		tg.HealthCheckPath = aws.String(r.HealthCheck.Path)
+	}
+	if r.HealthCheck.Matcher != "" {
+		tg.Matcher = &elbv2.Matcher{HttpCode: aws.String(r.HealthCheck.Matcher)}
 	}
 
 	// Omitted for NLB (TCP/UDP/TLS) target groups, matching AWS, which returns

spinifex/domains/elbv2/service_impl_rules.go

@@ -591,8 +591,8 @@ func (s *ELBv2ServiceImpl) validateAndConvertRuleActions(ctx context.Context, in
 				return nil, err
 			}
 		case ActionTypeFixedResponse:
-			if action.FixedResponse == nil {
-				return nil, errors.New(awserrors.ErrorMissingParameter)
+			if err := validateFixedResponseAction(action.FixedResponse); err != nil {
+				return nil, err
 			}
 		default:
 			return nil, errors.New(awserrors.ErrorELBv2InvalidConfigurationRequest)

spinifex/domains/elbv2/service_impl_rules_test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/elbv2"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -660,3 +661,53 @@ func sortByPriority(rs []*RuleRecord) []*RuleRecord {
 	}
 	return out
 }
+
+func TestValidateListenerAction_FixedResponse(t *testing.T) {
+	t.Parallel()
+	fixed := func(ct, body string) ListenerAction {
+		return ListenerAction{Type: ActionTypeFixedResponse, FixedResponse: &FixedResponseAction{
+			StatusCode: "200", ContentType: ct, MessageBody: body,
+		}}
+	}
+
+	for _, ct := range []string{"", "text/plain", "text/css", "text/html", "application/javascript", "application/json"} {
+		assert.NoError(t, validateListenerAction(fixed(ct, "ok")), ct)
+	}
+	assert.NoError(t, validateListenerAction(fixed("text/plain", "")))
+	assert.NoError(t, validateListenerAction(fixed("text/plain", "sk=${LB_SECRET_KEY} it's \"q\" #x")))
+	assert.NoError(t, validateListenerAction(fixed("text/plain", strings.Repeat("é", 1024))))
+
+	bad := []ListenerAction{
+		fixed("text/$LB_SECRET_KEY", "ok"),
+		fixed("text/xml", "ok"),
+		fixed("TEXT/PLAIN", "ok"),
+		fixed("text/plain", strings.Repeat("a", 1025)),
+		fixed("text/plain", "line\nacl x always_true"),
+		fixed("text/plain", "cr\r"),
+		fixed("text/plain", "nul\x00"),
+	}
+	for _, a := range bad {
+		assert.EqualError(t, validateListenerAction(a), awserrors.ErrorInvalidParameterValue, "%+v", *a.FixedResponse)
+	}
+	assert.EqualError(t, validateListenerAction(ListenerAction{Type: ActionTypeFixedResponse}), awserrors.ErrorMissingParameter)
+}
+
+func TestCreateRule_RejectsFixedResponseContentType(t *testing.T) {
+	t.Parallel()
+	env := newRuleTestEnv(t, "cr-badfixed")
+
+	_, err := env.svc.CreateRule(context.Background(), &elbv2.CreateRuleInput{
+		ListenerArn: aws.String(env.listenerArn),
+		Priority:    aws.Int64(1),
+		Conditions:  []*elbv2.RuleCondition{{Field: aws.String("path-pattern"), Values: aws.StringSlice([]string{"/x"})}},
+		Actions: []*elbv2.Action{{
+			Type: aws.String("fixed-response"),
+			FixedResponseConfig: &elbv2.FixedResponseActionConfig{
+				StatusCode:  aws.String("200"),
+				ContentType: aws.String("text/$LB_SECRET_KEY"),
+				MessageBody: aws.String("ok"),
+			},
+		}},
+	}, testAccountID)
+	require.EqualError(t, err, awserrors.ErrorInvalidParameterValue)
+}

spinifex/domains/elbv2/service_impl_test.go

@@ -459,11 +459,12 @@ func TestCreateTargetGroup_TCPProtocol(t *testing.T) {
 	assert.Equal(t, int64(5432), *tg.Port)
 	// NLB health check defaults: TCP protocol, no path, no matcher.
 	assert.Equal(t, "TCP", *tg.HealthCheckProtocol)
-	assert.Empty(t, *tg.HealthCheckPath)
-	assert.Empty(t, *tg.Matcher.HttpCode)
+	assert.Nil(t, tg.HealthCheckPath)
+	assert.Nil(t, tg.Matcher)
+	assert.Equal(t, int64(30), *tg.HealthCheckIntervalSeconds)
 	assert.Equal(t, int64(10), *tg.HealthCheckTimeoutSeconds)
-	assert.Equal(t, int64(3), *tg.HealthyThresholdCount)
-	assert.Equal(t, int64(3), *tg.UnhealthyThresholdCount)
+	assert.Equal(t, int64(5), *tg.HealthyThresholdCount)
+	assert.Equal(t, int64(2), *tg.UnhealthyThresholdCount)
 }
 
 func TestCreateTargetGroup_NLBProtocols(t *testing.T) {

spinifex/domains/elbv2/types.go

@@ -99,8 +99,8 @@ const (
 	// Default health check values (NLB).
 	DefaultNLBHealthCheckInterval = 30
 	DefaultNLBHealthCheckTimeout  = 10
-	DefaultNLBHealthyThreshold    = 3
-	DefaultNLBUnhealthyThreshold  = 3
+	DefaultNLBHealthyThreshold    = 5
+	DefaultNLBUnhealthyThreshold  = 2
 	DefaultNLBHealthCheckProtocol = ProtocolTCP
 	DefaultNLBHealthCheckPort     = "traffic-port"
 

spinifex/domains/iam/account_generation_test.go

@@ -33,9 +33,15 @@ func TestSetAccountStatusCountsOnlyChanges(t *testing.T) {
 		{AccountStatusTerminating, 3},
 	}
 	for _, step := range steps {
-		got, err := svc.SetAccountStatus(account.AccountID, step.status)
-		require.NoError(t, err)
-		assert.Equal(t, step.want, got.Generation, "status %s", step.status)
+		if step.status == AccountStatusTerminating {
+			generation, err := svc.QuiesceAccount(account.AccountID, "op-1", 1)
+			require.NoError(t, err)
+			assert.Equal(t, step.want, generation, "status %s", step.status)
+		} else {
+			got, err := svc.SetAccountStatus(account.AccountID, step.status)
+			require.NoError(t, err)
+			assert.Equal(t, step.want, got.Generation, "status %s", step.status)
+		}
 
 		stored, err := svc.GetAccount(account.AccountID)
 		require.NoError(t, err)

spinifex/domains/iam/account_lifecycle_test.go

@@ -9,7 +9,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func newLifecycleService(t *testing.T) iamdomain.IAMService {
+func newLifecycleService(t *testing.T) *iamdomain.IAMServiceImpl {
 	t.Helper()
 	_, nc, _ := testutil.StartTestJetStream(t)
 
@@ -56,7 +56,7 @@ func TestTerminatingIsAOneWayDoor(t *testing.T) {
 	account, err := svc.CreateAccount("tenant@example.com")
 	require.NoError(t, err)
 
-	_, err = svc.SetAccountStatus(account.AccountID, iamdomain.AccountStatusTerminating)
+	_, err = svc.QuiesceAccount(account.AccountID, "op-1", 1)
 	require.NoError(t, err)
 
 	for _, status := range []string{iamdomain.AccountStatusActive, iamdomain.AccountStatusSuspended} {
@@ -92,11 +92,11 @@ func TestDeleteAccountRequiresTerminating(t *testing.T) {
 	account, err := svc.CreateAccount("tenant@example.com")
 	require.NoError(t, err)
 
-	require.Error(t, svc.DeleteAccount(account.AccountID))
+	require.Error(t, svc.DeleteAccount(account.AccountID, "op-1", 1))
 
-	_, err = svc.SetAccountStatus(account.AccountID, iamdomain.AccountStatusTerminating)
+	_, err = svc.QuiesceAccount(account.AccountID, "op-1", 1)
 	require.NoError(t, err)
-	require.NoError(t, svc.DeleteAccount(account.AccountID))
+	require.NoError(t, svc.DeleteAccount(account.AccountID, "op-1", 1))
 
 	_, err = svc.GetAccount(account.AccountID)
 	assert.Error(t, err)
@@ -112,9 +112,9 @@ func TestProtectedAccountsCanNeverBeDeleted(t *testing.T) {
 		t.Run(accountID, func(t *testing.T) {
 			t.Parallel()
 			// Even a status that would otherwise permit deletion must not.
-			_, _ = svc.SetAccountStatus(accountID, iamdomain.AccountStatusTerminating)
+			_, _ = svc.QuiesceAccount(accountID, "op-1", 1)
 
-			err := svc.DeleteAccount(accountID)
+			err := svc.DeleteAccount(accountID, "op-1", 1)
 
 			require.Error(t, err)
 			assert.ErrorIs(t, err, iamdomain.ErrAccountUndeletable)

spinifex/domains/iam/account_quiesce_test.go

@@ -0,0 +1,334 @@
+package iam
+
+import (
+	"context"
+	"encoding/json"
+	"sync/atomic"
+	"testing"
+
+	"github.com/nats-io/nats.go/jetstream"
+
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// newQuiesceAccount returns a service and a tenant account. The first account a
+// fresh service creates takes the protected admin ID, so it is burned first.
+func newQuiesceAccount(t *testing.T) (*IAMServiceImpl, string) {
+	t.Helper()
+	svc := setupTestIAMService(t)
+	_, err := svc.CreateAccount("reserved-admin")
+	require.NoError(t, err)
+	account, err := svc.CreateAccount("quiesce-target")
+	require.NoError(t, err)
+	return svc, account.AccountID
+}
+
+// writeAccountRecord rewrites the stored record directly, to produce states the
+// service itself would not, such as a record written by an older binary.
+func writeAccountRecord(t *testing.T, svc *IAMServiceImpl, id string, mutate func(*Account)) {
+	t.Helper()
+	account, err := svc.GetAccount(id)
+	require.NoError(t, err)
+	mutate(account)
+	raw, err := json.Marshal(account)
+	require.NoError(t, err)
+	_, err = svc.accountsBucket.Put(context.Background(), id, raw)
+	require.NoError(t, err)
+}
+
+func TestQuiesceAccountRuleTable(t *testing.T) {
+	t.Parallel()
+
+	type op struct {
+		id    string
+		fence uint64
+	}
+	cases := []struct {
+		name string
+		// start puts the account in its starting state.
+		start func(t *testing.T, svc *IAMServiceImpl, id string)
+		call  op
+		// wantErr is nil when the call must succeed.
+		wantErr error
+		// wantGeneration is the generation returned and stored.
+		wantGeneration uint64
+		wantStatus     string
+		wantOp         *TerminationOp
+	}{
+		{
+			name:           "active becomes terminating and records the operation",
+			start:          func(*testing.T, *IAMServiceImpl, string) {},
+			call:           op{"op-1", 1},
+			wantGeneration: 1,
+			wantStatus:     AccountStatusTerminating,
+			wantOp:         &TerminationOp{OperationID: "op-1", Fence: 1},
+		},
+		{
+			name: "suspended becomes terminating and records the operation",
+			start: func(t *testing.T, svc *IAMServiceImpl, id string) {
+				_, err := svc.SetAccountStatus(id, AccountStatusSuspended)
+				require.NoError(t, err)
+			},
+			call:           op{"op-1", 1},
+			wantGeneration: 2,
+			wantStatus:     AccountStatusTerminating,
+			wantOp:         &TerminationOp{OperationID: "op-1", Fence: 1},
+		},
+		{
+			name: "same operation and fence is idempotent",
+			start: func(t *testing.T, svc *IAMServiceImpl, id string) {
+				_, err := svc.QuiesceAccount(id, "op-1", 1)
+				require.NoError(t, err)
+			},
+			call:           op{"op-1", 1},
+			wantGeneration: 1,
+			wantStatus:     AccountStatusTerminating,
+			wantOp:         &TerminationOp{OperationID: "op-1", Fence: 1},
+		},
+		{
+			name: "same operation at a higher fence records the fence only",
+			start: func(t *testing.T, svc *IAMServiceImpl, id string) {
+				_, err := svc.QuiesceAccount(id, "op-1", 1)
+				require.NoError(t, err)
+			},
+			call:           op{"op-1", 2},
+			wantGeneration: 1,
+			wantStatus:     AccountStatusTerminating,
+			wantOp:         &TerminationOp{OperationID: "op-1", Fence: 2},
+		},
+		{
+			name: "same operation at a lower fence is refused",
+			start: func(t *testing.T, svc *IAMServiceImpl, id string) {
+				_, err := svc.QuiesceAccount(id, "op-1", 3)
+				require.NoError(t, err)
+			},
+			call:       op{"op-1", 2},
+			wantErr:    ErrQuiesceSuperseded,
+			wantStatus: AccountStatusTerminating,
+			wantOp:     &TerminationOp{OperationID: "op-1", Fence: 3},
+		},
+		{
+			name: "a different operation is refused even at a higher fence",
+			start: func(t *testing.T, svc *IAMServiceImpl, id string) {
+				_, err := svc.QuiesceAccount(id, "op-1", 1)
+				require.NoError(t, err)
+			},
+			call:       op{"op-2", 9},
+			wantErr:    ErrQuiesceSuperseded,
+			wantStatus: AccountStatusTerminating,
+			wantOp:     &TerminationOp{OperationID: "op-1", Fence: 1},
+		},
+		{
+			name: "terminating without an operation, written by an older binary, adopts it",
+			start: func(t *testing.T, svc *IAMServiceImpl, id string) {
+				writeAccountRecord(t, svc, id, func(a *Account) {
+					a.Status = AccountStatusTerminating
+					a.Generation = 4
+				})
+			},
+			call:           op{"op-1", 1},
+			wantGeneration: 4,
+			wantStatus:     AccountStatusTerminating,
+			wantOp:         &TerminationOp{OperationID: "op-1", Fence: 1},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			svc, id := newQuiesceAccount(t)
+			tc.start(t, svc, id)
+
+			generation, err := svc.QuiesceAccount(id, tc.call.id, tc.call.fence)
+			if tc.wantErr != nil {
+				require.ErrorIs(t, err, tc.wantErr)
+			} else {
+				require.NoError(t, err)
+				assert.Equal(t, tc.wantGeneration, generation)
+			}
+
+			stored, err := svc.GetAccount(id)
+			require.NoError(t, err)
+			assert.Equal(t, tc.wantStatus, stored.Status)
+			assert.Equal(t, tc.wantOp, stored.TerminationOp)
+			if tc.wantErr == nil {
+				assert.Equal(t, tc.wantGeneration, stored.Generation)
+			}
+		})
+	}
+}
+
+func TestQuiesceAccountRefusesTheProtectedAccounts(t *testing.T) {
+	t.Parallel()
+	svc := setupTestIAMService(t)
+	systemSecret, err := svc.key.EncryptBase64("system-secret")
+	require.NoError(t, err)
+	adminSecret, err := svc.key.EncryptBase64("admin-secret")
+	require.NoError(t, err)
+	require.NoError(t, svc.SeedBootstrap(&BootstrapData{
+		AccessKeyID:     "AKIASYSTEM1234567890",
+		EncryptedSecret: systemSecret,
+		AccountID:       awsidentifiers.GlobalAccountID,
+		Admin: &AdminBootstrapData{
+			AccountID:       "000000000001",
+			AccountName:     "spinifex",
+			UserName:        "admin",
+			AccessKeyID:     "AKIAADMIN12345678901",
+			EncryptedSecret: adminSecret,
+		},
+	}))
+
+	for _, id := range []string{awsidentifiers.GlobalAccountID, "000000000001"} {
+		generation, err := svc.QuiesceAccount(id, "op-1", 1)
+		require.ErrorIs(t, err, ErrAccountUndeletable, id)
+		assert.Zero(t, generation)
+	}
+
+	stored, err := svc.GetAccount(awsidentifiers.GlobalAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, AccountStatusActive, stored.Status)
+	assert.Nil(t, stored.TerminationOp)
+}
+
+func TestQuiesceAccountNeedsAnOperationAndAFence(t *testing.T) {
+	t.Parallel()
+	svc, id := newQuiesceAccount(t)
+
+	for _, call := range []struct {
+		op    string
+		fence uint64
+	}{{"", 1}, {"op-1", 0}} {
+		_, err := svc.QuiesceAccount(id, call.op, call.fence)
+		require.Error(t, err)
+	}
+
+	stored, err := svc.GetAccount(id)
+	require.NoError(t, err)
+	assert.Equal(t, AccountStatusActive, stored.Status)
+}
+
+func TestQuiesceAccountOfAMissingAccountIsNoSuchEntity(t *testing.T) {
+	t.Parallel()
+	svc := setupTestIAMService(t)
+	_, err := svc.QuiesceAccount("000000000077", "op-1", 1)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "NoSuchEntity")
+}
+
+// Entering TERMINATING without an owning operation would leave an account that
+// no job can be fenced against, so SetAccountStatus refuses it.
+func TestSetAccountStatusRefusesTerminating(t *testing.T) {
+	t.Parallel()
+	svc, id := newQuiesceAccount(t)
+
+	got, err := svc.SetAccountStatus(id, AccountStatusTerminating)
+	require.ErrorIs(t, err, ErrUseQuiesceAccount)
+	assert.Nil(t, got)
+
+	stored, err := svc.GetAccount(id)
+	require.NoError(t, err)
+	assert.Equal(t, AccountStatusActive, stored.Status)
+	assert.Zero(t, stored.Generation)
+}
+
+// A status write by SetAccountStatus must not drop the operation record.
+func TestSetAccountStatusKeepsTheTerminationOp(t *testing.T) {
+	t.Parallel()
+	svc, id := newQuiesceAccount(t)
+	_, err := svc.QuiesceAccount(id, "op-1", 2)
+	require.NoError(t, err)
+
+	_, err = svc.SetAccountStatus(id, AccountStatusSuspended)
+	require.ErrorIs(t, err, ErrAccountUndeletable)
+
+	stored, err := svc.GetAccount(id)
+	require.NoError(t, err)
+	assert.Equal(t, &TerminationOp{OperationID: "op-1", Fence: 2}, stored.TerminationOp)
+}
+
+// racingDeleteBucket lets a takeover land between the delete's read and its
+// conditional delete.
+type racingDeleteBucket struct {
+	jetstream.KeyValue
+
+	raced   atomic.Bool
+	compete func()
+}
+
+func (b *racingDeleteBucket) Delete(ctx context.Context, key string, opts ...jetstream.KVDeleteOpt) error {
+	if b.raced.CompareAndSwap(false, true) {
+		b.compete()
+	}
+	return b.KeyValue.Delete(ctx, key, opts...)
+}
+
+// The final removal is fenced: only the holder whose operation and fence are
+// on the record may remove the account.
+func TestDeleteAccountIsFenced(t *testing.T) {
+	t.Parallel()
+
+	cases := []struct {
+		name    string
+		op      string
+		fence   uint64
+		wantErr error
+	}{
+		{"the matching operation and fence removes the account", "op-1", 2, nil},
+		{"a stale fence cannot remove the account", "op-1", 1, ErrQuiesceSuperseded},
+		{"a newer fence than the record cannot remove the account", "op-1", 3, ErrQuiesceSuperseded},
+		{"a different operation cannot remove the account", "op-2", 2, ErrQuiesceSuperseded},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			svc, id := newQuiesceAccount(t)
+			_, err := svc.QuiesceAccount(id, "op-1", 2)
+			require.NoError(t, err)
+
+			err = svc.DeleteAccount(id, tc.op, tc.fence)
+
+			_, getErr := svc.GetAccount(id)
+			if tc.wantErr == nil {
+				require.NoError(t, err)
+				require.Error(t, getErr, "the account must be gone")
+				return
+			}
+			require.ErrorIs(t, err, tc.wantErr)
+			require.NoError(t, getErr, "a refused delete must remove nothing")
+		})
+	}
+}
+
+func TestDeleteAccountOfARecordWithoutAnOperationIsRefused(t *testing.T) {
+	t.Parallel()
+	svc, id := newQuiesceAccount(t)
+	writeAccountRecord(t, svc, id, func(a *Account) { a.Status = AccountStatusTerminating })
+
+	require.ErrorIs(t, svc.DeleteAccount(id, "op-1", 1), ErrQuiesceSuperseded)
+	_, err := svc.GetAccount(id)
+	require.NoError(t, err)
+}
+
+// A record that changes between the read and the delete is not removed on the
+// strength of the earlier read: the conditional delete refuses and the retry
+// re-reads, so a takeover in that window wins.
+func TestDeleteAccountDoesNotRemoveARecordTakenOverMidDelete(t *testing.T) {
+	t.Parallel()
+	svc, id := newQuiesceAccount(t)
+	_, err := svc.QuiesceAccount(id, "op-1", 1)
+	require.NoError(t, err)
+
+	underlying := svc.accountsBucket
+	svc.accountsBucket = &racingDeleteBucket{KeyValue: underlying, compete: func() {
+		_, takeoverErr := svc.QuiesceAccount(id, "op-1", 2)
+		require.NoError(t, takeoverErr)
+	}}
+
+	require.ErrorIs(t, svc.DeleteAccount(id, "op-1", 1), ErrQuiesceSuperseded)
+	svc.accountsBucket = underlying
+	stored, err := svc.GetAccount(id)
+	require.NoError(t, err)
+	assert.Equal(t, &TerminationOp{OperationID: "op-1", Fence: 2}, stored.TerminationOp)
+}

spinifex/domains/iam/account_status_reader_test.go

@@ -12,7 +12,7 @@ import (
 
 // newStatusReaderFixture starts a real IAM service (which creates and
 // migrates spinifex-accounts) and a reader attached to the same JetStream.
-func newStatusReaderFixture(t *testing.T) (iamdomain.IAMService, *iamdomain.AccountStatusReader, jetstream.JetStream) {
+func newStatusReaderFixture(t *testing.T) (*iamdomain.IAMServiceImpl, *iamdomain.AccountStatusReader, jetstream.JetStream) {
 	t.Helper()
 	_, nc, js := testutil.StartTestJetStream(t)
 
@@ -32,7 +32,7 @@ func TestAccountStatusReader_TerminatingIsTrue(t *testing.T) {
 	require.NoError(t, err)
 	account, err := svc.CreateAccount("tenant@example.com")
 	require.NoError(t, err)
-	_, err = svc.SetAccountStatus(account.AccountID, iamdomain.AccountStatusTerminating)
+	_, err = svc.QuiesceAccount(account.AccountID, "op-1", 1)
 	require.NoError(t, err)
 
 	terminating, err := reader.AccountTerminating(t.Context(), account.AccountID)

spinifex/domains/iam/awsapi/authz.go

@@ -48,6 +48,7 @@ var iamScopes = map[string]resourceScope{
 	"CreateUser":               createScope(arn.IAMUser, "UserName"),
 	"GetUser":                  existingScope(arn.IAMUser, "UserName"),
 	"DeleteUser":               existingScope(arn.IAMUser, "UserName"),
+	"UpdateUser":               existingScope(arn.IAMUser, "UserName"),
 	"CreateAccessKey":          existingScope(arn.IAMUser, "UserName"),
 	"ListAccessKeys":           existingScope(arn.IAMUser, "UserName"),
 	"DeleteAccessKey":          existingScope(arn.IAMUser, "UserName"),
@@ -284,7 +285,7 @@ func lookupResult(t lookupTarget, resource string, err error) (string, error) {
 		}
 		return resource, nil
 	}
-	if err.Error() == awserrors.ErrorIAMNoSuchEntity {
+	if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
 		// Authorization widens to "*", so a Deny scoped to the real target
 		// will not fire. Without this line that widening leaves no trace.
 		slog.Warn("IAM authz: target not found, authorizing account-wide",

spinifex/domains/iam/awsapi/authz_test.go

@@ -33,7 +33,7 @@ func (s *resolverService) CanonicalResourceARN(_ string, kind arn.IAMResourceTyp
 			return resource, nil
 		}
 	}
-	return "", errors.New(awserrors.ErrorIAMNoSuchEntity)
+	return "", awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "The %s with name %s cannot be found.", kind, name)
 }
 
 func (s *resolverService) LookupAccessKey(_ string) (*iamdomain.AccessKey, error) {

spinifex/domains/iam/awsapi/registration.go

@@ -96,6 +96,9 @@ var actions = map[string]action{
 	"DeleteUser": handler(func(accountID string, input *iam.DeleteUserInput, d Deps) (any, error) {
 		return DeleteUser(accountID, input, d.IAM)
 	}),
+	"UpdateUser": handler(func(accountID string, input *iam.UpdateUserInput, d Deps) (any, error) {
+		return UpdateUser(accountID, input, d.IAM)
+	}),
 	"CreateAccessKey": handler(func(accountID string, input *iam.CreateAccessKeyInput, d Deps) (any, error) {
 		return CreateAccessKey(accountID, input, d.IAM)
 	}),

spinifex/domains/iam/awsapi/registration_test.go

@@ -26,6 +26,7 @@ type regService struct {
 
 	getUserName   string
 	getUserCalls  int
+	updateUser    *iam.UpdateUserInput
 	addRoleCalls  int
 	canonicalARN  string
 	canonicalErr  error
@@ -41,6 +42,11 @@ func (s *regService) GetUser(_ string, in *iam.GetUserInput) (*iam.GetUserOutput
 	return &iam.GetUserOutput{}, nil
 }
 
+func (s *regService) UpdateUser(_ string, in *iam.UpdateUserInput) (*iam.UpdateUserOutput, error) {
+	s.updateUser = in
+	return &iam.UpdateUserOutput{}, nil
+}
+
 func (s *regService) AddRoleToInstanceProfile(_ string, _ *iam.AddRoleToInstanceProfileInput) (*iam.AddRoleToInstanceProfileOutput, error) {
 	s.addRoleCalls++
 	return &iam.AddRoleToInstanceProfileOutput{}, nil
@@ -268,6 +274,23 @@ func TestRegistration_AddRoleToInstanceProfilePassRole(t *testing.T) {
 	})
 }
 
+func TestRegistration_UpdateUser(t *testing.T) {
+	svc := &regService{}
+	w, calls, err := invoke(t, Deps{IAM: svc}, testAccountID, dispatch.Actor{}, nil, url.Values{
+		"Action": {"UpdateUser"}, "UserName": {"alice"}, "NewUserName": {"alicia"}, "NewPath": {"/eng/"},
+	})
+	require.NoError(t, err)
+	require.Len(t, calls, 1)
+	assert.Equal(t, []string{"arn:aws:iam::" + testAccountID + ":user/alice"}, calls[0].resources)
+	assert.Contains(t, w.Body.String(), "UpdateUserResponse")
+	require.NotNil(t, svc.updateUser)
+	assert.Equal(t, "alicia", *svc.updateUser.NewUserName)
+	assert.Equal(t, "/eng/", *svc.updateUser.NewPath)
+
+	_, _, err = invoke(t, Deps{IAM: svc}, testAccountID, dispatch.Actor{}, nil, url.Values{"Action": {"UpdateUser"}})
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorMissingParameter), "got %v", err)
+}
+
 func TestActionNames_SortedAndInventory(t *testing.T) {
 	names := ActionNames()
 	require.NotEmpty(t, names)

spinifex/domains/iam/awsapi/user.go

@@ -78,6 +78,15 @@ func DeleteUser(accountID string, input *iam.DeleteUserInput, svc iamdomain.IAMS
 	return svc.DeleteUser(accountID, input)
 }
 
+// UpdateUser implements the IAM UpdateUser action, renaming a user or changing its path. It
+// returns MissingParameter when UserName is absent, then calls the IAM service for accountID.
+func UpdateUser(accountID string, input *iam.UpdateUserInput, svc iamdomain.IAMService) (*iam.UpdateUserOutput, error) {
+	if input.UserName == nil || *input.UserName == "" {
+		return nil, errors.New(awserrors.ErrorMissingParameter)
+	}
+	return svc.UpdateUser(accountID, input)
+}
+
 // PutUserPolicy implements the IAM PutUserPolicy action, adding or replacing a user inline
 // policy. It returns MissingParameter when UserName, PolicyName or PolicyDocument is absent, then
 // calls the IAM service for accountID.

spinifex/domains/iam/error_messages_test.go

@@ -0,0 +1,186 @@
+package iam
+
+import (
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestEntityErrorMessages pins the per-case text AWS returns for NoSuchEntity,
+// EntityAlreadyExists, DeleteConflict and LimitExceeded, as recorded against
+// real IAM, so a client sees which entity is missing or in conflict.
+func TestEntityErrorMessages(t *testing.T) {
+	svc := setupTestIAMService(t)
+	acct := testAccountID
+
+	createTestUser(t, svc, "u1")
+	createTestRole(t, svc, "r1")
+	createTestRole(t, svc, "r2")
+	createTestGroup(t, svc, "g1")
+	createTestInstanceProfile(t, svc, "ip1")
+	policy := createTestPolicy(t, svc, "p1")
+	policyARN := aws.StringValue(policy.Arn)
+	missingPolicyARN := "arn:aws:iam::" + acct + ":policy/p1-nope"
+	_, err := svc.CreateOpenIDConnectProvider(acct, &iam.CreateOpenIDConnectProviderInput{
+		Url: aws.String(testIssuer), ClientIDList: aws.StringSlice([]string{"sts.amazonaws.com"}),
+	})
+	require.NoError(t, err)
+	missingOIDCARN := "arn:aws:iam::" + acct + ":oidc-provider/oidc.nope.example.com"
+
+	// Conflict fixtures: a user holding a key, a role with a managed policy, a
+	// role and profile linked to each other, and a group with a member.
+	createTestUser(t, svc, "u-key")
+	_, err = svc.CreateAccessKey(acct, &iam.CreateAccessKeyInput{UserName: aws.String("u-key")})
+	require.NoError(t, err)
+	createTestRole(t, svc, "r-pol")
+	_, err = svc.AttachRolePolicy(acct, &iam.AttachRolePolicyInput{RoleName: aws.String("r-pol"), PolicyArn: aws.String(policyARN)})
+	require.NoError(t, err)
+	createTestRole(t, svc, "r-ip")
+	createTestInstanceProfile(t, svc, "ip-r")
+	_, err = svc.AddRoleToInstanceProfile(acct, &iam.AddRoleToInstanceProfileInput{InstanceProfileName: aws.String("ip-r"), RoleName: aws.String("r-ip")})
+	require.NoError(t, err)
+	createTestGroup(t, svc, "g-mem")
+	_, err = svc.AddUserToGroup(acct, &iam.AddUserToGroupInput{GroupName: aws.String("g-mem"), UserName: aws.String("u1")})
+	require.NoError(t, err)
+
+	cases := []struct {
+		name     string
+		call     func() error
+		wantCode string
+		wantMsg  string
+	}{
+		{"GetUser missing", func() error {
+			_, err := svc.GetUser(acct, &iam.GetUserInput{UserName: aws.String("nope")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The user with name nope cannot be found."},
+		{"TagUser missing", func() error {
+			_, err := svc.TagUser(acct, &iam.TagUserInput{UserName: aws.String("nope"), Tags: []*iam.Tag{{Key: aws.String("k"), Value: aws.String("v")}}})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The user with name nope cannot be found."},
+		{"AddUserToGroup missing user", func() error {
+			_, err := svc.AddUserToGroup(acct, &iam.AddUserToGroupInput{GroupName: aws.String("g1"), UserName: aws.String("nope")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The user with name nope cannot be found."},
+		{"GetRole missing", func() error {
+			_, err := svc.GetRole(acct, &iam.GetRoleInput{RoleName: aws.String("nope")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The role with name nope cannot be found."},
+		{"TagRole missing", func() error {
+			_, err := svc.TagRole(acct, &iam.TagRoleInput{RoleName: aws.String("nope"), Tags: []*iam.Tag{{Key: aws.String("k"), Value: aws.String("v")}}})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The role with name nope cannot be found."},
+		{"DeleteGroup missing", func() error {
+			_, err := svc.DeleteGroup(acct, &iam.DeleteGroupInput{GroupName: aws.String("nope")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The group with name nope cannot be found."},
+		{"GetPolicy missing", func() error {
+			_, err := svc.GetPolicy(acct, &iam.GetPolicyInput{PolicyArn: aws.String(missingPolicyARN)})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "Policy " + missingPolicyARN + " was not found."},
+		{"DeletePolicy missing", func() error {
+			_, err := svc.DeletePolicy(acct, &iam.DeletePolicyInput{PolicyArn: aws.String(missingPolicyARN)})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "Policy " + missingPolicyARN + " was not found."},
+		{"DetachUserPolicy not attached", func() error {
+			_, err := svc.DetachUserPolicy(acct, &iam.DetachUserPolicyInput{UserName: aws.String("u1"), PolicyArn: aws.String(policyARN)})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "Policy " + policyARN + " was not found."},
+		{"AttachRolePolicy missing policy", func() error {
+			_, err := svc.AttachRolePolicy(acct, &iam.AttachRolePolicyInput{RoleName: aws.String("r1"), PolicyArn: aws.String(missingPolicyARN)})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "Policy " + missingPolicyARN + " does not exist or is not attachable."},
+		{"GetInstanceProfile missing", func() error {
+			_, err := svc.GetInstanceProfile(acct, &iam.GetInstanceProfileInput{InstanceProfileName: aws.String("nope")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "Instance Profile nope cannot be found."},
+		{"RemoveRoleFromInstanceProfile not in profile", func() error {
+			_, err := svc.RemoveRoleFromInstanceProfile(acct, &iam.RemoveRoleFromInstanceProfileInput{InstanceProfileName: aws.String("ip1"), RoleName: aws.String("r2")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "Role r2 in Instance Profile ip1 cannot be found."},
+		{"GetOpenIDConnectProvider missing", func() error {
+			_, err := svc.GetOpenIDConnectProvider(acct, &iam.GetOpenIDConnectProviderInput{OpenIDConnectProviderArn: aws.String(missingOIDCARN)})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "OpenIDConnect Provider not found for arn " + missingOIDCARN},
+		{"GetUserPolicy missing", func() error {
+			_, err := svc.GetUserPolicy(acct, &iam.GetUserPolicyInput{UserName: aws.String("u1"), PolicyName: aws.String("inl")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The user policy with name inl cannot be found."},
+		{"DeleteRolePolicy missing", func() error {
+			_, err := svc.DeleteRolePolicy(acct, &iam.DeleteRolePolicyInput{RoleName: aws.String("r1"), PolicyName: aws.String("rinl")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The role policy with name rinl cannot be found."},
+		{"DeleteAccessKey missing", func() error {
+			_, err := svc.DeleteAccessKey(acct, &iam.DeleteAccessKeyInput{UserName: aws.String("u1"), AccessKeyId: aws.String("AKIANOPE")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The Access Key with id AKIANOPE cannot be found."},
+
+		{"CreateUser duplicate", func() error {
+			_, err := svc.CreateUser(acct, &iam.CreateUserInput{UserName: aws.String("u1")})
+			return err
+		}, awserrors.ErrorIAMEntityAlreadyExists, "User with name u1 already exists."},
+		{"CreateRole duplicate", func() error {
+			_, err := svc.CreateRole(acct, &iam.CreateRoleInput{RoleName: aws.String("r1"), AssumeRolePolicyDocument: aws.String(validTrustPolicy())})
+			return err
+		}, awserrors.ErrorIAMEntityAlreadyExists, "Role with name r1 already exists."},
+		{"CreateGroup duplicate", func() error {
+			_, err := svc.CreateGroup(acct, &iam.CreateGroupInput{GroupName: aws.String("g1")})
+			return err
+		}, awserrors.ErrorIAMEntityAlreadyExists, "Group with name g1 already exists."},
+		{"CreatePolicy duplicate", func() error {
+			_, err := svc.CreatePolicy(acct, &iam.CreatePolicyInput{PolicyName: aws.String("p1"), PolicyDocument: aws.String(validPolicyDocument())})
+			return err
+		}, awserrors.ErrorIAMEntityAlreadyExists, "A policy called p1 already exists. Duplicate names are not allowed."},
+		{"CreateInstanceProfile duplicate", func() error {
+			_, err := svc.CreateInstanceProfile(acct, &iam.CreateInstanceProfileInput{InstanceProfileName: aws.String("ip1")})
+			return err
+		}, awserrors.ErrorIAMEntityAlreadyExists, "Instance Profile ip1 already exists."},
+		{"CreateOpenIDConnectProvider duplicate", func() error {
+			_, err := svc.CreateOpenIDConnectProvider(acct, &iam.CreateOpenIDConnectProviderInput{
+				Url: aws.String(testIssuer), ClientIDList: aws.StringSlice([]string{"sts.amazonaws.com"}),
+			})
+			return err
+		}, awserrors.ErrorIAMEntityAlreadyExists, "Provider with url " + testIssuer + " already exists."},
+
+		{"DeleteUser with access key", func() error {
+			_, err := svc.DeleteUser(acct, &iam.DeleteUserInput{UserName: aws.String("u-key")})
+			return err
+		}, awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must delete access keys first."},
+		{"DeleteRole with attached policy", func() error {
+			_, err := svc.DeleteRole(acct, &iam.DeleteRoleInput{RoleName: aws.String("r-pol")})
+			return err
+		}, awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must detach all policies first."},
+		{"DeleteRole in instance profile", func() error {
+			_, err := svc.DeleteRole(acct, &iam.DeleteRoleInput{RoleName: aws.String("r-ip")})
+			return err
+		}, awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must remove roles from instance profile first."},
+		{"DeleteInstanceProfile with role", func() error {
+			_, err := svc.DeleteInstanceProfile(acct, &iam.DeleteInstanceProfileInput{InstanceProfileName: aws.String("ip-r")})
+			return err
+		}, awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must remove roles from instance profile first."},
+		{"DeleteGroup with member", func() error {
+			_, err := svc.DeleteGroup(acct, &iam.DeleteGroupInput{GroupName: aws.String("g-mem")})
+			return err
+		}, awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must remove users from group first."},
+		{"DeletePolicy attached", func() error {
+			_, err := svc.DeletePolicy(acct, &iam.DeletePolicyInput{PolicyArn: aws.String(policyARN)})
+			return err
+		}, awserrors.ErrorIAMDeleteConflict, "Cannot delete a policy attached to entities."},
+
+		{"AddRoleToInstanceProfile second role", func() error {
+			_, err := svc.AddRoleToInstanceProfile(acct, &iam.AddRoleToInstanceProfileInput{InstanceProfileName: aws.String("ip-r"), RoleName: aws.String("r2")})
+			return err
+		}, awserrors.ErrorIAMLimitExceeded, "Cannot exceed quota for InstanceSessionsPerInstanceProfile: 1"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			code, msg, ok := awserrors.ResolveErrorDetail(tc.call())
+			require.True(t, ok, "error carries no AWS code")
+			assert.Equal(t, tc.wantCode, code)
+			assert.Equal(t, tc.wantMsg, msg)
+		})
+	}
+}

spinifex/domains/iam/groups.go

@@ -65,7 +65,7 @@ func (s *IAMServiceImpl) CreateGroup(accountID string, input *iam.CreateGroupInp
 	kvKey := accountID + "." + groupName
 	if _, err := s.groupsBucket.Create(ctx, kvKey, data); err != nil {
 		if errors.Is(err, jetstream.ErrKeyExists) {
-			return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists)
+			return nil, awserrors.Errorf(awserrors.ErrorIAMEntityAlreadyExists, "Group with name %s already exists.", groupName)
 		}
 		return nil, fmt.Errorf("store group: %w", err)
 	}
@@ -175,11 +175,11 @@ func (s *IAMServiceImpl) DeleteGroup(accountID string, input *iam.DeleteGroupInp
 	}
 
 	if len(group.AttachedPolicies) > 0 {
-		return nil, errors.New(awserrors.ErrorIAMDeleteConflict)
+		return nil, errDetachPoliciesFirst
 	}
 
 	if len(group.InlinePolicies) > 0 {
-		return nil, errors.New(awserrors.ErrorIAMDeleteConflict)
+		return nil, errDeletePoliciesFirst
 	}
 
 	members, err := s.findGroupMembers(ctx, accountID, groupName)
@@ -187,7 +187,7 @@ func (s *IAMServiceImpl) DeleteGroup(accountID string, input *iam.DeleteGroupInp
 		return nil, fmt.Errorf("check group members: %w", err)
 	}
 	if len(members) > 0 {
-		return nil, errors.New(awserrors.ErrorIAMDeleteConflict)
+		return nil, awserrors.Errorf(awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must remove users from group first.")
 	}
 
 	if err := s.groupsBucket.Delete(ctx, accountID+"."+groupName); err != nil {
@@ -234,19 +234,28 @@ func (s *IAMServiceImpl) RemoveUserFromGroup(accountID string, input *iam.Remove
 	groupName := *input.GroupName
 	userName := *input.UserName
 
-	// Operate purely on the membership reference so a dangling pointer to an
-	// already-deleted group is still cleanable; never fetch the group here.
+	// Remove the membership reference without fetching the group, so a dangling
+	// pointer to an already-deleted group is still cleanable.
+	removed := false
 	err := s.updateUserCAS(ctx, accountID, userName, func(u *User) (bool, error) {
 		idx := slices.Index(u.Groups, groupName)
 		if idx < 0 {
-			return false, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return false, nil
 		}
 		u.Groups = slices.Delete(u.Groups, idx, idx+1)
+		removed = true
 		return true, nil
 	})
 	if err != nil {
 		return nil, err
 	}
+	// AWS answers success for a non-member when the group exists.
+	if !removed {
+		if _, err := s.getGroup(ctx, accountID, groupName); err != nil {
+			return nil, err
+		}
+		return &iam.RemoveUserFromGroupOutput{}, nil
+	}
 
 	slog.Info("IAM user removed from group", "accountID", accountID, "userName", userName, "groupName", groupName)
 	return &iam.RemoveUserFromGroupOutput{}, nil
@@ -265,7 +274,7 @@ func (s *IAMServiceImpl) ListGroupsForUser(accountID string, input *iam.ListGrou
 	for _, name := range user.Groups {
 		group, err := s.getGroup(ctx, accountID, name)
 		if err != nil {
-			if err.Error() == awserrors.ErrorIAMNoSuchEntity {
+			if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
 				// A dangling pointer to a vanished group is inert; skip it.
 				// Mirrors GetUserPolicies — transient/corrupt errors fail closed.
 				slog.Warn("ListGroupsForUser: member references missing group; skipping",
@@ -294,7 +303,7 @@ func (s *IAMServiceImpl) AttachGroupPolicy(accountID string, input *iam.AttachGr
 
 	// AWS-managed ARNs are stored opaquely (like roles); customer-managed ARNs must exist.
 	if !iamarn.IsAWSManagedPolicyARN(policyARN) {
-		if _, err := s.getPolicyByARN(ctx, accountID, policyARN); err != nil {
+		if err := s.attachablePolicy(ctx, accountID, policyARN); err != nil {
 			return nil, err
 		}
 	}
@@ -322,7 +331,7 @@ func (s *IAMServiceImpl) DetachGroupPolicy(accountID string, input *iam.DetachGr
 	err := s.updateGroupCAS(ctx, accountID, groupName, func(g *Group) (bool, error) {
 		idx := slices.Index(g.AttachedPolicies, policyARN)
 		if idx < 0 {
-			return false, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return false, policyNotFound(policyARN)
 		}
 		g.AttachedPolicies = slices.Delete(g.AttachedPolicies, idx, idx+1)
 		return true, nil
@@ -408,7 +417,7 @@ func (s *IAMServiceImpl) GetGroupPolicy(accountID string, input *iam.GetGroupPol
 
 	doc, ok := group.InlinePolicies[policyName]
 	if !ok {
-		return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+		return nil, noSuchEntity("group policy", policyName)
 	}
 
 	return &iam.GetGroupPolicyOutput{
@@ -427,7 +436,7 @@ func (s *IAMServiceImpl) DeleteGroupPolicy(accountID string, input *iam.DeleteGr
 
 	err := s.updateGroupCAS(ctx, accountID, groupName, func(g *Group) (bool, error) {
 		if _, ok := g.InlinePolicies[policyName]; !ok {
-			return false, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return false, noSuchEntity("group policy", policyName)
 		}
 		delete(g.InlinePolicies, policyName)
 		return true, nil
@@ -470,7 +479,7 @@ func (s *IAMServiceImpl) getGroup(ctx context.Context, accountID, groupName stri
 	entry, err := s.groupsBucket.Get(ctx, accountID+"."+groupName)
 	if err != nil {
 		if errors.Is(err, jetstream.ErrKeyNotFound) {
-			return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return nil, noSuchEntity("group", groupName)
 		}
 		return nil, fmt.Errorf("get group: %w", err)
 	}

spinifex/domains/iam/groups_test.go

@@ -409,6 +409,18 @@ func TestRemoveUserFromGroup_NotMember(t *testing.T) {
 		GroupName: aws.String("team"),
 		UserName:  aws.String("erin"),
 	})
+	require.NoError(t, err)
+}
+
+func TestRemoveUserFromGroup_GroupNotFound(t *testing.T) {
+	t.Parallel()
+	svc := setupTestIAMService(t)
+	createTestUser(t, svc, "erin")
+
+	_, err := svc.RemoveUserFromGroup(testAccountID, &iam.RemoveUserFromGroupInput{
+		GroupName: aws.String("ghost"),
+		UserName:  aws.String("erin"),
+	})
 	assert.Error(t, err)
 	assert.Contains(t, err.Error(), awserrors.ErrorIAMNoSuchEntity)
 }

spinifex/domains/iam/instance_profiles.go

@@ -66,7 +66,7 @@ func (s *IAMServiceImpl) CreateInstanceProfile(accountID string, input *iam.Crea
 
 	if _, err := s.instanceProfilesBucket.Create(ctx, accountID+"."+profileName, data); err != nil {
 		if errors.Is(err, jetstream.ErrKeyExists) {
-			return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists)
+			return nil, awserrors.Errorf(awserrors.ErrorIAMEntityAlreadyExists, "Instance Profile %s already exists.", profileName)
 		}
 		return nil, fmt.Errorf("store instance profile: %w", err)
 	}
@@ -172,7 +172,7 @@ func (s *IAMServiceImpl) DeleteInstanceProfile(accountID string, input *iam.Dele
 	_, err := kvutil.DeleteIf(ctx, s.instanceProfilesBucket, accountID+"."+profileName, s.instanceProfileCASConfig(accountID, profileName),
 		func(p *InstanceProfile) error {
 			if p.RoleName != "" {
-				return errors.New(awserrors.ErrorIAMDeleteConflict)
+				return awserrors.Errorf(awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must remove roles from instance profile first.")
 			}
 			return nil
 		})
@@ -195,7 +195,7 @@ func (s *IAMServiceImpl) AddRoleToInstanceProfile(accountID string, input *iam.A
 
 	err := s.updateInstanceProfileCAS(ctx, accountID, profileName, func(p *InstanceProfile) (bool, error) {
 		if p.RoleName != "" {
-			return false, errors.New(awserrors.ErrorIAMLimitExceeded)
+			return false, awserrors.Errorf(awserrors.ErrorIAMLimitExceeded, "Cannot exceed quota for InstanceSessionsPerInstanceProfile: 1")
 		}
 		p.RoleName = roleName
 		return true, nil
@@ -216,7 +216,7 @@ func (s *IAMServiceImpl) RemoveRoleFromInstanceProfile(accountID string, input *
 
 	err := s.updateInstanceProfileCAS(ctx, accountID, profileName, func(p *InstanceProfile) (bool, error) {
 		if p.RoleName == "" || p.RoleName != roleName {
-			return false, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return false, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Role %s in Instance Profile %s cannot be found.", roleName, profileName)
 		}
 		p.RoleName = ""
 		return true, nil
@@ -357,11 +357,16 @@ func (s *IAMServiceImpl) ListInstanceProfileTags(accountID string, input *iam.Li
 	}, nil
 }
 
+// instanceProfileNotFound is NoSuchEntity with AWS's message for a missing instance profile.
+func instanceProfileNotFound(profileName string) error {
+	return awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", profileName)
+}
+
 func (s *IAMServiceImpl) getInstanceProfile(ctx context.Context, accountID, profileName string) (*InstanceProfile, error) {
 	entry, err := s.instanceProfilesBucket.Get(ctx, accountID+"."+profileName)
 	if err != nil {
 		if errors.Is(err, jetstream.ErrKeyNotFound) {
-			return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return nil, instanceProfileNotFound(profileName)
 		}
 		return nil, fmt.Errorf("get instance profile: %w", err)
 	}
@@ -385,7 +390,7 @@ func (s *IAMServiceImpl) updateInstanceProfileCAS(ctx context.Context, accountID
 func (s *IAMServiceImpl) instanceProfileCASConfig(accountID, profileName string) kvutil.CASConfig {
 	return kvutil.CASConfig{
 		Attempts: instanceProfileCASMaxRetries,
-		NotFound: errors.New(awserrors.ErrorIAMNoSuchEntity),
+		NotFound: instanceProfileNotFound(profileName),
 		Exhausted: func(string, int) error {
 			slog.Warn("IAM instance profile CAS exhausted retries",
 				"accountID", accountID, "instanceProfileName", profileName, "attempts", instanceProfileCASMaxRetries)

spinifex/domains/iam/instance_profiles_test.go

@@ -778,7 +778,7 @@ func TestResolveInstanceProfile_NonCanonicalARNRejected(t *testing.T) {
 		_, err := svc.ResolveInstanceProfile(testAccountID, arn)
 		require.Error(t, err, arn)
 		// The EC2 gateway maps this to InvalidIamInstanceProfile.NotFound by exact match.
-		assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, err.Error(), arn)
+		assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, awserrors.ValidErrorCodeFromError(err), arn)
 	}
 }
 

spinifex/domains/iam/mock/mock.go

@@ -90,7 +90,7 @@ func (e *SystemInstanceRoleEnsurer) GetRole(_ string, in *iam.GetRoleInput) (*ia
 
 	r, ok := e.Roles[aws.StringValue(in.RoleName)]
 	if !ok {
-		return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "The role with name %s cannot be found.", aws.StringValue(in.RoleName))
 	}
 	return &iam.GetRoleOutput{Role: copyRole(r)}, nil
 }
@@ -104,7 +104,7 @@ func (e *SystemInstanceRoleEnsurer) CreateRole(accountID string, in *iam.CreateR
 	e.LastRoleAcct = accountID
 	name := aws.StringValue(in.RoleName)
 	if _, ok := e.Roles[name]; ok {
-		return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists)
+		return nil, awserrors.Errorf(awserrors.ErrorIAMEntityAlreadyExists, "Role with name %s already exists.", name)
 	}
 	r := &iam.Role{
 		RoleName: aws.String(name),
@@ -132,7 +132,7 @@ func (e *SystemInstanceRoleEnsurer) GetInstanceProfile(_ string, in *iam.GetInst
 
 	p, ok := e.Profiles[aws.StringValue(in.InstanceProfileName)]
 	if !ok {
-		return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", aws.StringValue(in.InstanceProfileName))
 	}
 	return &iam.GetInstanceProfileOutput{InstanceProfile: copyProfile(p)}, nil
 }
@@ -148,7 +148,7 @@ func (e *SystemInstanceRoleEnsurer) CreateInstanceProfile(accountID string, in *
 	}
 	name := aws.StringValue(in.InstanceProfileName)
 	if _, ok := e.Profiles[name]; ok {
-		return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists)
+		return nil, awserrors.Errorf(awserrors.ErrorIAMEntityAlreadyExists, "Instance Profile %s already exists.", name)
 	}
 	arn := "arn:aws:iam::" + accountID + ":instance-profile/" + name
 	if e.EmptyInstanceProfileARN {
@@ -176,7 +176,7 @@ func (e *SystemInstanceRoleEnsurer) AddRoleToInstanceProfile(_ string, in *iam.A
 		return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
 	}
 	if len(p.Roles) > 0 {
-		return nil, errors.New(awserrors.ErrorIAMLimitExceeded)
+		return nil, awserrors.Errorf(awserrors.ErrorIAMLimitExceeded, "Cannot exceed quota for InstanceSessionsPerInstanceProfile: 1")
 	}
 	p.Roles = []*iam.Role{{RoleName: in.RoleName}}
 	return &iam.AddRoleToInstanceProfileOutput{}, nil

spinifex/domains/iam/mock/mock_test.go

@@ -13,7 +13,7 @@ import (
 func TestGetRole_MissingReturnsNoSuchEntity(t *testing.T) {
 	e := mock.New()
 	_, err := e.GetRole("acct", &iam.GetRoleInput{RoleName: aws.String("r1")})
-	if err == nil || err.Error() != awserrors.ErrorIAMNoSuchEntity {
+	if err == nil || awserrors.ValidErrorCodeFromError(err) != awserrors.ErrorIAMNoSuchEntity {
 		t.Fatalf("err = %v, want ErrorIAMNoSuchEntity", err)
 	}
 }
@@ -24,7 +24,7 @@ func TestCreateRole_DuplicateReturnsEntityAlreadyExists(t *testing.T) {
 	if _, err := e.CreateRole("acct", in); err != nil {
 		t.Fatalf("first CreateRole: %v", err)
 	}
-	if _, err := e.CreateRole("acct", in); err == nil || err.Error() != awserrors.ErrorIAMEntityAlreadyExists {
+	if _, err := e.CreateRole("acct", in); err == nil || awserrors.ValidErrorCodeFromError(err) != awserrors.ErrorIAMEntityAlreadyExists {
 		t.Fatalf("second CreateRole err = %v, want EntityAlreadyExists", err)
 	}
 	if e.CreateRoleCalls != 2 {
@@ -48,7 +48,7 @@ func TestAddRoleToInstanceProfile_SecondAttachIsLimitExceeded(t *testing.T) {
 	if _, err := e.AddRoleToInstanceProfile("acct", in); err != nil {
 		t.Fatalf("first AddRoleToInstanceProfile: %v", err)
 	}
-	if _, err := e.AddRoleToInstanceProfile("acct", in); err == nil || err.Error() != awserrors.ErrorIAMLimitExceeded {
+	if _, err := e.AddRoleToInstanceProfile("acct", in); err == nil || awserrors.ValidErrorCodeFromError(err) != awserrors.ErrorIAMLimitExceeded {
 		t.Fatalf("second AddRoleToInstanceProfile err = %v, want LimitExceeded", err)
 	}
 }

spinifex/domains/iam/oidc_providers.go

@@ -178,7 +178,7 @@ func (s *IAMServiceImpl) CreateOpenIDConnectProvider(accountID string, input *ia
 
 	if _, err := kv.Create(ctx, OIDCProviderKey(issuer), data); err != nil {
 		if errors.Is(err, jetstream.ErrKeyExists) {
-			return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists)
+			return nil, awserrors.Errorf(awserrors.ErrorIAMEntityAlreadyExists, "Provider with url %s already exists.", issuer)
 		}
 		return nil, fmt.Errorf("store OIDC provider: %w", err)
 	}
@@ -268,7 +268,7 @@ func (s *IAMServiceImpl) DeleteOpenIDConnectProvider(accountID string, input *ia
 	kv, err := s.js.KeyValue(ctx, IAMAccountBucketName(accountID))
 	if err != nil {
 		if errors.Is(err, jetstream.ErrBucketNotFound) {
-			return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return nil, oidcProviderNotFound(arn)
 		}
 		return nil, fmt.Errorf("open IAM account bucket: %w", err)
 	}
@@ -276,7 +276,7 @@ func (s *IAMServiceImpl) DeleteOpenIDConnectProvider(accountID string, input *ia
 	key := OIDCProviderKey(issuer)
 	if _, err := kv.Get(ctx, key); err != nil {
 		if errors.Is(err, jetstream.ErrKeyNotFound) {
-			return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return nil, oidcProviderNotFound(arn)
 		}
 		return nil, fmt.Errorf("get OIDC provider: %w", err)
 	}
@@ -356,7 +356,7 @@ func (s *IAMServiceImpl) updateOIDCProviderCAS(ctx context.Context, accountID, a
 	kv, err := s.js.KeyValue(ctx, IAMAccountBucketName(accountID))
 	if err != nil {
 		if errors.Is(err, jetstream.ErrBucketNotFound) {
-			return errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return oidcProviderNotFound(arn)
 		}
 		return fmt.Errorf("open IAM account bucket: %w", err)
 	}
@@ -373,14 +373,14 @@ func (s *IAMServiceImpl) getOIDCProvider(ctx context.Context, accountID, arn str
 	kv, err := s.js.KeyValue(ctx, IAMAccountBucketName(accountID))
 	if err != nil {
 		if errors.Is(err, jetstream.ErrBucketNotFound) {
-			return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return nil, oidcProviderNotFound(arn)
 		}
 		return nil, fmt.Errorf("open IAM account bucket: %w", err)
 	}
 	entry, err := kv.Get(ctx, OIDCProviderKey(issuer))
 	if err != nil {
 		if errors.Is(err, jetstream.ErrKeyNotFound) {
-			return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return nil, oidcProviderNotFound(arn)
 		}
 		return nil, fmt.Errorf("get OIDC provider: %w", err)
 	}

spinifex/domains/iam/oidc_providers_test.go

@@ -50,7 +50,7 @@ func TestCreateOpenIDConnectProvider_Idempotent(t *testing.T) {
 		t.Fatalf("first create: %v", err)
 	}
 	_, err := svc.CreateOpenIDConnectProvider(acct, in)
-	if err == nil || err.Error() != awserrors.ErrorIAMEntityAlreadyExists {
+	if err == nil || awserrors.ValidErrorCodeFromError(err) != awserrors.ErrorIAMEntityAlreadyExists {
 		t.Fatalf("duplicate create err = %v, want EntityAlreadyExists", err)
 	}
 }
@@ -132,7 +132,7 @@ func TestGetOpenIDConnectProvider_NotFound(t *testing.T) {
 	_, err := svc.GetOpenIDConnectProvider("000000000001", &iam.GetOpenIDConnectProviderInput{
 		OpenIDConnectProviderArn: aws.String(OIDCProviderARN("000000000001", "10.0.0.1:9999/oidc/eks/x/y/z")),
 	})
-	if err == nil || err.Error() != awserrors.ErrorIAMNoSuchEntity {
+	if err == nil || awserrors.ValidErrorCodeFromError(err) != awserrors.ErrorIAMNoSuchEntity {
 		t.Fatalf("err = %v, want NoSuchEntity", err)
 	}
 }
@@ -195,7 +195,7 @@ func TestDeleteOpenIDConnectProvider(t *testing.T) {
 	_, err = svc.DeleteOpenIDConnectProvider(acct, &iam.DeleteOpenIDConnectProviderInput{
 		OpenIDConnectProviderArn: created.OpenIDConnectProviderArn,
 	})
-	if err == nil || err.Error() != awserrors.ErrorIAMNoSuchEntity {
+	if err == nil || awserrors.ValidErrorCodeFromError(err) != awserrors.ErrorIAMNoSuchEntity {
 		t.Fatalf("second delete err = %v, want NoSuchEntity", err)
 	}
 }