fix(iam): name the missing or conflicting entity in NoSuchEntity, EntityAlreadyExists, DeleteConflict and LimitExceeded messages as AWS does (#1435)

Lambda code copies live in a system-account Predastore bucket reached
only through a short-lived session on a Regional role scoped to that
bucket. STS gains an in-process AssumeCodeStoreRole that derives the
Region's code-store role itself and assumes it as lambda.amazonaws.com;
it is not on the STS interface, NATS or HTTPS.

Provisioning creates or verifies the owner-tagged role, inline policy
and bucket on first use only, retries Lambda's own partial create,
and refuses an untagged or foreign resource with an operator-visible
reason. Copies are keyed by their SHA-256, so different bytes can never
replace a recorded revision. The gateway holds the root client and IAM;
the Lambda package receives only the bounded provisioning operations
and the scoped store.

docs/coverage/README.md

@@ -17,7 +17,7 @@ tags:
 
 ## Overview
 
-The platform serves **405 operations** across the AWS APIs below. Every page names the operations its service implements from the pinned model, generated from the dispatch tables on each build rather than written by hand.
+The platform serves **406 operations** across the AWS APIs below. Every page names the operations its service implements from the pinned model, generated from the dispatch tables on each build rather than written by hand.
 
 | Service | Operations |
 |---|---:|
@@ -27,9 +27,9 @@ The platform serves **405 operations** across the AWS APIs below. Every page nam
 | [ECS](/coverage/ecs) | 31 |
 | [EKS](/coverage/eks) | 27 |
 | [ELBv2](/coverage/elbv2) | 36 |
-| [IAM](/coverage/iam) | 81 |
+| [IAM](/coverage/iam) | 82 |
 | [RDS](/coverage/rds) | 27 |
 | [Route 53](/coverage/route53) | 13 |
 | [S3](/coverage/s3) | 26 |
 | [STS](/coverage/sts) | 5 |
-| **Total** | **405** |
+| **Total** | **406** |

docs/coverage/coverage.json

@@ -2938,7 +2938,8 @@
           "UpdateAccessKey",
           "UpdateAssumeRolePolicy",
           "UpdateRole",
-          "UpdateRoleDescription"
+          "UpdateRoleDescription",
+          "UpdateUser"
         ],
         "implemented": [
           "AddRoleToInstanceProfile",
@@ -3021,7 +3022,8 @@
           "UpdateAccessKey",
           "UpdateAssumeRolePolicy",
           "UpdateRole",
-          "UpdateRoleDescription"
+          "UpdateRoleDescription",
+          "UpdateUser"
         ],
         "stubbed": [],
         "unsupported": [],
@@ -3121,7 +3123,6 @@
           "UpdateServerCertificate",
           "UpdateServiceSpecificCredential",
           "UpdateSigningCertificate",
-          "UpdateUser",
           "UploadSSHPublicKey",
           "UploadServerCertificate",
           "UploadSigningCertificate"

docs/coverage/iam/README.md

@@ -18,7 +18,7 @@ tags:
 
 ## Overview
 
-Spinifex implements **81 operations** in the IAM `2010-05-08` API model.
+Spinifex implements **82 operations** in the IAM `2010-05-08` API model.
 
 ### Scope
 
@@ -109,3 +109,4 @@ All IAM operations are account-scoped. The root user of an account bypasses poli
 | `UpdateAssumeRolePolicy` |
 | `UpdateRole` |
 | `UpdateRoleDescription` |
+| `UpdateUser` |

docs/coverage/internal-report.md

@@ -987,7 +987,7 @@ spot-price: On owned hardware there is no spot-to-on-demand price differential,
 
 ## iam
 
-`2010-05-08` — 81 of 180 modelled operations implemented (45.0%), 0 stubbed, 19 not applicable, 0 registered outside the pinned model.
+`2010-05-08` — 82 of 180 modelled operations implemented (45.6%), 0 stubbed, 19 not applicable, 0 registered outside the pinned model.
 
 ### Not implemented
 
@@ -1069,7 +1069,6 @@ spot-price: On owned hardware there is no spot-to-on-demand price differential,
 - `UpdateOpenIDConnectProviderThumbprint`
 - `UpdateSAMLProvider`
 - `UpdateServerCertificate`
-- `UpdateUser`
 - `UploadServerCertificate`
 
 ### Not applicable — do not build

spinifex/domains/acm/keyalgorithm_test.go

@@ -152,3 +152,30 @@ func TestImportCertificate_SerialZeroRejected(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorValidationException, code)
 	assert.Equal(t, "The serial number in the certificate is not supported by ACM.", msg)
 }
+
+func TestImportCertificate_ExpiredRejected(t *testing.T) {
+	t.Parallel()
+	svc := setupACMService(t)
+	key, err := rsa.GenerateKey(rand.Reader, 1024)
+	require.NoError(t, err)
+	tmpl := &x509.Certificate{
+		SerialNumber: big.NewInt(time.Now().UnixNano()),
+		Subject:      pkix.Name{CommonName: "expired.example.com"},
+		NotBefore:    time.Now().Add(-48 * time.Hour),
+		NotAfter:     time.Now().Add(-24 * time.Hour),
+	}
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
+	require.NoError(t, err)
+	_, err = svc.ImportCertificate(context.Background(), &acm.ImportCertificateInput{
+		Certificate: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}),
+		PrivateKey:  pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}),
+	}, testAccountID)
+	code, msg, ok := awserrors.ResolveErrorDetail(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorValidationException, code)
+	assert.Equal(t, "The certificate is expired. You can Import a certificate only during its validity period.", msg)
+
+	list, err := svc.ListCertificates(context.Background(), &acm.ListCertificatesInput{}, testAccountID)
+	require.NoError(t, err)
+	assert.Empty(t, list.CertificateSummaryList)
+}

spinifex/domains/acm/service_impl.go

@@ -223,10 +223,13 @@ func (s *ACMServiceImpl) ImportCertificate(ctx context.Context, input *acm.Impor
 	if leaf.SerialNumber.Sign() == 0 {
 		return nil, awserrors.Errorf(awserrors.ErrorValidationException, "The serial number in the certificate is not supported by ACM.")
 	}
-
 	certArn := aws.StringValue(input.CertificateArn)
-	var inUseBy []string
 	now := time.Now().UTC()
+	if certArn == "" && now.After(leaf.NotAfter) {
+		return nil, awserrors.Errorf(awserrors.ErrorValidationException, "The certificate is expired. You can Import a certificate only during its validity period.")
+	}
+
+	var inUseBy []string
 	createdAt := now
 	tags := tagsToMap(input.Tags)
 	reimport := certArn != ""

spinifex/domains/ec2/awsapi/instance/IamInstanceProfileAssociation.go

@@ -35,7 +35,7 @@ func resolveAndAuthorizeProfile(spec *ec2.IamInstanceProfileSpecification, iamSv
 	}
 	profile, err := iamSvc.ResolveInstanceProfile(accountID, nameOrARN)
 	if err != nil {
-		if err.Error() == awserrors.ErrorIAMNoSuchEntity {
+		if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
 			return nil, errors.New(awserrors.ErrorInvalidIamInstanceProfileNotFound)
 		}
 		return nil, err

spinifex/domains/ec2/awsapi/instance/IamInstanceProfileAssociation_test.go

@@ -45,7 +45,7 @@ func (f *fakeIAMService) ResolveInstanceProfile(accountID, nameOrARN string) (*i
 	if f.resolveFn != nil {
 		return f.resolveFn(accountID, nameOrARN)
 	}
-	return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+	return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", nameOrARN)
 }
 
 func (f *fakeIAMService) CanonicalResourceARN(accountID string, kind arn.IAMResourceType, name string) (string, error) {

spinifex/domains/ec2/guestmetadata/metadata.go

@@ -738,7 +738,7 @@ func (s *IMDSServiceImpl) profileFor(ctx context.Context, eni *eniFacts) (*resol
 	}
 	profile, err := s.iam.ResolveInstanceProfile(ctx, eni.iamAccountID(), inst.iamInstanceProfileArn)
 	if err != nil {
-		if err.Error() == awserrors.ErrorIAMNoSuchEntity {
+		if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
 			return nil, roleMissProfileDeleted, nil // profile deleted; treat as no role
 		}
 		slog.ErrorContext(ctx, "IMDS: resolve instance profile failed", "account_id", eni.iamAccountID(), "arn", inst.iamInstanceProfileArn, "err", err)

spinifex/domains/ec2/guestmetadata/metadata_test.go

@@ -1152,7 +1152,7 @@ func TestHTTP_SecurityCredentialsListBackendError500(t *testing.T) {
 // genuine backend error (TestHTTP_*BackendError500) is a 500.
 func TestHTTP_IAMInfoDeletedProfile404(t *testing.T) {
 	res := &fakeResolver{eni: testENI(), inst: &instanceFacts{iamInstanceProfileArn: "arn:aws:iam::111122223333:instance-profile/gone"}}
-	svc, _ := newTestService(res, &fakeIAM{profileErr: errors.New(awserrors.ErrorIAMNoSuchEntity)}, &fakeAssumer{})
+	svc, _ := newTestService(res, &fakeIAM{profileErr: awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile gone cannot be found.")}, &fakeAssumer{})
 	h := withTapENI(svc.httpHandler(), testENI())
 	token := issueToken(t, h)
 	rec := get(t, h, prefixMetaData+"iam/info", token)

spinifex/domains/eks/cluster/nodegroup.go

@@ -886,7 +886,7 @@ func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string
 		return s.attachRoleToProfile(accountID, profileName, roleName,
 			aws.StringValue(out.InstanceProfile.Arn), len(out.InstanceProfile.Roles) > 0)
 	}
-	if err.Error() != awserrors.ErrorIAMNoSuchEntity {
+	if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMNoSuchEntity {
 		return "", fmt.Errorf("get instance profile %q: %w", profileName, err)
 	}
 
@@ -895,7 +895,7 @@ func (s *EKSServiceImpl) ensureNodeInstanceProfile(accountID, nodeRoleARN string
 	})
 	if err != nil {
 		// A racing worker launch may have created it first; re-read and converge.
-		if err.Error() != awserrors.ErrorIAMEntityAlreadyExists {
+		if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMEntityAlreadyExists {
 			return "", fmt.Errorf("create instance profile %q: %w", profileName, err)
 		}
 		got, gerr := s.iamEnsurer().GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{
@@ -922,7 +922,7 @@ func (s *EKSServiceImpl) attachRoleToProfile(accountID, profileName, roleName, p
 		InstanceProfileName: aws.String(profileName),
 		RoleName:            aws.String(roleName),
 	})
-	if err != nil && err.Error() != awserrors.ErrorIAMLimitExceeded {
+	if code, _ := awserrors.ResolveErrorCode(err); err != nil && code != awserrors.ErrorIAMLimitExceeded {
 		return "", fmt.Errorf("add role %q to instance profile %q: %w", roleName, profileName, err)
 	}
 	return profileARN, nil

spinifex/domains/iam/awsapi/authz.go

@@ -48,6 +48,7 @@ var iamScopes = map[string]resourceScope{
 	"CreateUser":               createScope(arn.IAMUser, "UserName"),
 	"GetUser":                  existingScope(arn.IAMUser, "UserName"),
 	"DeleteUser":               existingScope(arn.IAMUser, "UserName"),
+	"UpdateUser":               existingScope(arn.IAMUser, "UserName"),
 	"CreateAccessKey":          existingScope(arn.IAMUser, "UserName"),
 	"ListAccessKeys":           existingScope(arn.IAMUser, "UserName"),
 	"DeleteAccessKey":          existingScope(arn.IAMUser, "UserName"),
@@ -284,7 +285,7 @@ func lookupResult(t lookupTarget, resource string, err error) (string, error) {
 		}
 		return resource, nil
 	}
-	if err.Error() == awserrors.ErrorIAMNoSuchEntity {
+	if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
 		// Authorization widens to "*", so a Deny scoped to the real target
 		// will not fire. Without this line that widening leaves no trace.
 		slog.Warn("IAM authz: target not found, authorizing account-wide",

spinifex/domains/iam/awsapi/authz_test.go

@@ -33,7 +33,7 @@ func (s *resolverService) CanonicalResourceARN(_ string, kind arn.IAMResourceTyp
 			return resource, nil
 		}
 	}
-	return "", errors.New(awserrors.ErrorIAMNoSuchEntity)
+	return "", awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "The %s with name %s cannot be found.", kind, name)
 }
 
 func (s *resolverService) LookupAccessKey(_ string) (*iamdomain.AccessKey, error) {

spinifex/domains/iam/awsapi/registration.go

@@ -96,6 +96,9 @@ var actions = map[string]action{
 	"DeleteUser": handler(func(accountID string, input *iam.DeleteUserInput, d Deps) (any, error) {
 		return DeleteUser(accountID, input, d.IAM)
 	}),
+	"UpdateUser": handler(func(accountID string, input *iam.UpdateUserInput, d Deps) (any, error) {
+		return UpdateUser(accountID, input, d.IAM)
+	}),
 	"CreateAccessKey": handler(func(accountID string, input *iam.CreateAccessKeyInput, d Deps) (any, error) {
 		return CreateAccessKey(accountID, input, d.IAM)
 	}),

spinifex/domains/iam/awsapi/registration_test.go

@@ -26,6 +26,7 @@ type regService struct {
 
 	getUserName   string
 	getUserCalls  int
+	updateUser    *iam.UpdateUserInput
 	addRoleCalls  int
 	canonicalARN  string
 	canonicalErr  error
@@ -41,6 +42,11 @@ func (s *regService) GetUser(_ string, in *iam.GetUserInput) (*iam.GetUserOutput
 	return &iam.GetUserOutput{}, nil
 }
 
+func (s *regService) UpdateUser(_ string, in *iam.UpdateUserInput) (*iam.UpdateUserOutput, error) {
+	s.updateUser = in
+	return &iam.UpdateUserOutput{}, nil
+}
+
 func (s *regService) AddRoleToInstanceProfile(_ string, _ *iam.AddRoleToInstanceProfileInput) (*iam.AddRoleToInstanceProfileOutput, error) {
 	s.addRoleCalls++
 	return &iam.AddRoleToInstanceProfileOutput{}, nil
@@ -268,6 +274,23 @@ func TestRegistration_AddRoleToInstanceProfilePassRole(t *testing.T) {
 	})
 }
 
+func TestRegistration_UpdateUser(t *testing.T) {
+	svc := &regService{}
+	w, calls, err := invoke(t, Deps{IAM: svc}, testAccountID, dispatch.Actor{}, nil, url.Values{
+		"Action": {"UpdateUser"}, "UserName": {"alice"}, "NewUserName": {"alicia"}, "NewPath": {"/eng/"},
+	})
+	require.NoError(t, err)
+	require.Len(t, calls, 1)
+	assert.Equal(t, []string{"arn:aws:iam::" + testAccountID + ":user/alice"}, calls[0].resources)
+	assert.Contains(t, w.Body.String(), "UpdateUserResponse")
+	require.NotNil(t, svc.updateUser)
+	assert.Equal(t, "alicia", *svc.updateUser.NewUserName)
+	assert.Equal(t, "/eng/", *svc.updateUser.NewPath)
+
+	_, _, err = invoke(t, Deps{IAM: svc}, testAccountID, dispatch.Actor{}, nil, url.Values{"Action": {"UpdateUser"}})
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorMissingParameter), "got %v", err)
+}
+
 func TestActionNames_SortedAndInventory(t *testing.T) {
 	names := ActionNames()
 	require.NotEmpty(t, names)

spinifex/domains/iam/awsapi/user.go

@@ -78,6 +78,15 @@ func DeleteUser(accountID string, input *iam.DeleteUserInput, svc iamdomain.IAMS
 	return svc.DeleteUser(accountID, input)
 }
 
+// UpdateUser implements the IAM UpdateUser action, renaming a user or changing its path. It
+// returns MissingParameter when UserName is absent, then calls the IAM service for accountID.
+func UpdateUser(accountID string, input *iam.UpdateUserInput, svc iamdomain.IAMService) (*iam.UpdateUserOutput, error) {
+	if input.UserName == nil || *input.UserName == "" {
+		return nil, errors.New(awserrors.ErrorMissingParameter)
+	}
+	return svc.UpdateUser(accountID, input)
+}
+
 // PutUserPolicy implements the IAM PutUserPolicy action, adding or replacing a user inline
 // policy. It returns MissingParameter when UserName, PolicyName or PolicyDocument is absent, then
 // calls the IAM service for accountID.

spinifex/domains/iam/error_messages_test.go

@@ -0,0 +1,186 @@
+package iam
+
+import (
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestEntityErrorMessages pins the per-case text AWS returns for NoSuchEntity,
+// EntityAlreadyExists, DeleteConflict and LimitExceeded, as recorded against
+// real IAM, so a client sees which entity is missing or in conflict.
+func TestEntityErrorMessages(t *testing.T) {
+	svc := setupTestIAMService(t)
+	acct := testAccountID
+
+	createTestUser(t, svc, "u1")
+	createTestRole(t, svc, "r1")
+	createTestRole(t, svc, "r2")
+	createTestGroup(t, svc, "g1")
+	createTestInstanceProfile(t, svc, "ip1")
+	policy := createTestPolicy(t, svc, "p1")
+	policyARN := aws.StringValue(policy.Arn)
+	missingPolicyARN := "arn:aws:iam::" + acct + ":policy/p1-nope"
+	_, err := svc.CreateOpenIDConnectProvider(acct, &iam.CreateOpenIDConnectProviderInput{
+		Url: aws.String(testIssuer), ClientIDList: aws.StringSlice([]string{"sts.amazonaws.com"}),
+	})
+	require.NoError(t, err)
+	missingOIDCARN := "arn:aws:iam::" + acct + ":oidc-provider/oidc.nope.example.com"
+
+	// Conflict fixtures: a user holding a key, a role with a managed policy, a
+	// role and profile linked to each other, and a group with a member.
+	createTestUser(t, svc, "u-key")
+	_, err = svc.CreateAccessKey(acct, &iam.CreateAccessKeyInput{UserName: aws.String("u-key")})
+	require.NoError(t, err)
+	createTestRole(t, svc, "r-pol")
+	_, err = svc.AttachRolePolicy(acct, &iam.AttachRolePolicyInput{RoleName: aws.String("r-pol"), PolicyArn: aws.String(policyARN)})
+	require.NoError(t, err)
+	createTestRole(t, svc, "r-ip")
+	createTestInstanceProfile(t, svc, "ip-r")
+	_, err = svc.AddRoleToInstanceProfile(acct, &iam.AddRoleToInstanceProfileInput{InstanceProfileName: aws.String("ip-r"), RoleName: aws.String("r-ip")})
+	require.NoError(t, err)
+	createTestGroup(t, svc, "g-mem")
+	_, err = svc.AddUserToGroup(acct, &iam.AddUserToGroupInput{GroupName: aws.String("g-mem"), UserName: aws.String("u1")})
+	require.NoError(t, err)
+
+	cases := []struct {
+		name     string
+		call     func() error
+		wantCode string
+		wantMsg  string
+	}{
+		{"GetUser missing", func() error {
+			_, err := svc.GetUser(acct, &iam.GetUserInput{UserName: aws.String("nope")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The user with name nope cannot be found."},
+		{"TagUser missing", func() error {
+			_, err := svc.TagUser(acct, &iam.TagUserInput{UserName: aws.String("nope"), Tags: []*iam.Tag{{Key: aws.String("k"), Value: aws.String("v")}}})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The user with name nope cannot be found."},
+		{"AddUserToGroup missing user", func() error {
+			_, err := svc.AddUserToGroup(acct, &iam.AddUserToGroupInput{GroupName: aws.String("g1"), UserName: aws.String("nope")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The user with name nope cannot be found."},
+		{"GetRole missing", func() error {
+			_, err := svc.GetRole(acct, &iam.GetRoleInput{RoleName: aws.String("nope")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The role with name nope cannot be found."},
+		{"TagRole missing", func() error {
+			_, err := svc.TagRole(acct, &iam.TagRoleInput{RoleName: aws.String("nope"), Tags: []*iam.Tag{{Key: aws.String("k"), Value: aws.String("v")}}})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The role with name nope cannot be found."},
+		{"DeleteGroup missing", func() error {
+			_, err := svc.DeleteGroup(acct, &iam.DeleteGroupInput{GroupName: aws.String("nope")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The group with name nope cannot be found."},
+		{"GetPolicy missing", func() error {
+			_, err := svc.GetPolicy(acct, &iam.GetPolicyInput{PolicyArn: aws.String(missingPolicyARN)})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "Policy " + missingPolicyARN + " was not found."},
+		{"DeletePolicy missing", func() error {
+			_, err := svc.DeletePolicy(acct, &iam.DeletePolicyInput{PolicyArn: aws.String(missingPolicyARN)})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "Policy " + missingPolicyARN + " was not found."},
+		{"DetachUserPolicy not attached", func() error {
+			_, err := svc.DetachUserPolicy(acct, &iam.DetachUserPolicyInput{UserName: aws.String("u1"), PolicyArn: aws.String(policyARN)})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "Policy " + policyARN + " was not found."},
+		{"AttachRolePolicy missing policy", func() error {
+			_, err := svc.AttachRolePolicy(acct, &iam.AttachRolePolicyInput{RoleName: aws.String("r1"), PolicyArn: aws.String(missingPolicyARN)})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "Policy " + missingPolicyARN + " does not exist or is not attachable."},
+		{"GetInstanceProfile missing", func() error {
+			_, err := svc.GetInstanceProfile(acct, &iam.GetInstanceProfileInput{InstanceProfileName: aws.String("nope")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "Instance Profile nope cannot be found."},
+		{"RemoveRoleFromInstanceProfile not in profile", func() error {
+			_, err := svc.RemoveRoleFromInstanceProfile(acct, &iam.RemoveRoleFromInstanceProfileInput{InstanceProfileName: aws.String("ip1"), RoleName: aws.String("r2")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "Role r2 in Instance Profile ip1 cannot be found."},
+		{"GetOpenIDConnectProvider missing", func() error {
+			_, err := svc.GetOpenIDConnectProvider(acct, &iam.GetOpenIDConnectProviderInput{OpenIDConnectProviderArn: aws.String(missingOIDCARN)})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "OpenIDConnect Provider not found for arn " + missingOIDCARN},
+		{"GetUserPolicy missing", func() error {
+			_, err := svc.GetUserPolicy(acct, &iam.GetUserPolicyInput{UserName: aws.String("u1"), PolicyName: aws.String("inl")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The user policy with name inl cannot be found."},
+		{"DeleteRolePolicy missing", func() error {
+			_, err := svc.DeleteRolePolicy(acct, &iam.DeleteRolePolicyInput{RoleName: aws.String("r1"), PolicyName: aws.String("rinl")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The role policy with name rinl cannot be found."},
+		{"DeleteAccessKey missing", func() error {
+			_, err := svc.DeleteAccessKey(acct, &iam.DeleteAccessKeyInput{UserName: aws.String("u1"), AccessKeyId: aws.String("AKIANOPE")})
+			return err
+		}, awserrors.ErrorIAMNoSuchEntity, "The Access Key with id AKIANOPE cannot be found."},
+
+		{"CreateUser duplicate", func() error {
+			_, err := svc.CreateUser(acct, &iam.CreateUserInput{UserName: aws.String("u1")})
+			return err
+		}, awserrors.ErrorIAMEntityAlreadyExists, "User with name u1 already exists."},
+		{"CreateRole duplicate", func() error {
+			_, err := svc.CreateRole(acct, &iam.CreateRoleInput{RoleName: aws.String("r1"), AssumeRolePolicyDocument: aws.String(validTrustPolicy())})
+			return err
+		}, awserrors.ErrorIAMEntityAlreadyExists, "Role with name r1 already exists."},
+		{"CreateGroup duplicate", func() error {
+			_, err := svc.CreateGroup(acct, &iam.CreateGroupInput{GroupName: aws.String("g1")})
+			return err
+		}, awserrors.ErrorIAMEntityAlreadyExists, "Group with name g1 already exists."},
+		{"CreatePolicy duplicate", func() error {
+			_, err := svc.CreatePolicy(acct, &iam.CreatePolicyInput{PolicyName: aws.String("p1"), PolicyDocument: aws.String(validPolicyDocument())})
+			return err
+		}, awserrors.ErrorIAMEntityAlreadyExists, "A policy called p1 already exists. Duplicate names are not allowed."},
+		{"CreateInstanceProfile duplicate", func() error {
+			_, err := svc.CreateInstanceProfile(acct, &iam.CreateInstanceProfileInput{InstanceProfileName: aws.String("ip1")})
+			return err
+		}, awserrors.ErrorIAMEntityAlreadyExists, "Instance Profile ip1 already exists."},
+		{"CreateOpenIDConnectProvider duplicate", func() error {
+			_, err := svc.CreateOpenIDConnectProvider(acct, &iam.CreateOpenIDConnectProviderInput{
+				Url: aws.String(testIssuer), ClientIDList: aws.StringSlice([]string{"sts.amazonaws.com"}),
+			})
+			return err
+		}, awserrors.ErrorIAMEntityAlreadyExists, "Provider with url " + testIssuer + " already exists."},
+
+		{"DeleteUser with access key", func() error {
+			_, err := svc.DeleteUser(acct, &iam.DeleteUserInput{UserName: aws.String("u-key")})
+			return err
+		}, awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must delete access keys first."},
+		{"DeleteRole with attached policy", func() error {
+			_, err := svc.DeleteRole(acct, &iam.DeleteRoleInput{RoleName: aws.String("r-pol")})
+			return err
+		}, awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must detach all policies first."},
+		{"DeleteRole in instance profile", func() error {
+			_, err := svc.DeleteRole(acct, &iam.DeleteRoleInput{RoleName: aws.String("r-ip")})
+			return err
+		}, awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must remove roles from instance profile first."},
+		{"DeleteInstanceProfile with role", func() error {
+			_, err := svc.DeleteInstanceProfile(acct, &iam.DeleteInstanceProfileInput{InstanceProfileName: aws.String("ip-r")})
+			return err
+		}, awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must remove roles from instance profile first."},
+		{"DeleteGroup with member", func() error {
+			_, err := svc.DeleteGroup(acct, &iam.DeleteGroupInput{GroupName: aws.String("g-mem")})
+			return err
+		}, awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must remove users from group first."},
+		{"DeletePolicy attached", func() error {
+			_, err := svc.DeletePolicy(acct, &iam.DeletePolicyInput{PolicyArn: aws.String(policyARN)})
+			return err
+		}, awserrors.ErrorIAMDeleteConflict, "Cannot delete a policy attached to entities."},
+
+		{"AddRoleToInstanceProfile second role", func() error {
+			_, err := svc.AddRoleToInstanceProfile(acct, &iam.AddRoleToInstanceProfileInput{InstanceProfileName: aws.String("ip-r"), RoleName: aws.String("r2")})
+			return err
+		}, awserrors.ErrorIAMLimitExceeded, "Cannot exceed quota for InstanceSessionsPerInstanceProfile: 1"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			code, msg, ok := awserrors.ResolveErrorDetail(tc.call())
+			require.True(t, ok, "error carries no AWS code")
+			assert.Equal(t, tc.wantCode, code)
+			assert.Equal(t, tc.wantMsg, msg)
+		})
+	}
+}

spinifex/domains/iam/groups.go

@@ -65,7 +65,7 @@ func (s *IAMServiceImpl) CreateGroup(accountID string, input *iam.CreateGroupInp
 	kvKey := accountID + "." + groupName
 	if _, err := s.groupsBucket.Create(ctx, kvKey, data); err != nil {
 		if errors.Is(err, jetstream.ErrKeyExists) {
-			return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists)
+			return nil, awserrors.Errorf(awserrors.ErrorIAMEntityAlreadyExists, "Group with name %s already exists.", groupName)
 		}
 		return nil, fmt.Errorf("store group: %w", err)
 	}
@@ -175,11 +175,11 @@ func (s *IAMServiceImpl) DeleteGroup(accountID string, input *iam.DeleteGroupInp
 	}
 
 	if len(group.AttachedPolicies) > 0 {
-		return nil, errors.New(awserrors.ErrorIAMDeleteConflict)
+		return nil, errDetachPoliciesFirst
 	}
 
 	if len(group.InlinePolicies) > 0 {
-		return nil, errors.New(awserrors.ErrorIAMDeleteConflict)
+		return nil, errDeletePoliciesFirst
 	}
 
 	members, err := s.findGroupMembers(ctx, accountID, groupName)
@@ -187,7 +187,7 @@ func (s *IAMServiceImpl) DeleteGroup(accountID string, input *iam.DeleteGroupInp
 		return nil, fmt.Errorf("check group members: %w", err)
 	}
 	if len(members) > 0 {
-		return nil, errors.New(awserrors.ErrorIAMDeleteConflict)
+		return nil, awserrors.Errorf(awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must remove users from group first.")
 	}
 
 	if err := s.groupsBucket.Delete(ctx, accountID+"."+groupName); err != nil {
@@ -234,19 +234,28 @@ func (s *IAMServiceImpl) RemoveUserFromGroup(accountID string, input *iam.Remove
 	groupName := *input.GroupName
 	userName := *input.UserName
 
-	// Operate purely on the membership reference so a dangling pointer to an
-	// already-deleted group is still cleanable; never fetch the group here.
+	// Remove the membership reference without fetching the group, so a dangling
+	// pointer to an already-deleted group is still cleanable.
+	removed := false
 	err := s.updateUserCAS(ctx, accountID, userName, func(u *User) (bool, error) {
 		idx := slices.Index(u.Groups, groupName)
 		if idx < 0 {
-			return false, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return false, nil
 		}
 		u.Groups = slices.Delete(u.Groups, idx, idx+1)
+		removed = true
 		return true, nil
 	})
 	if err != nil {
 		return nil, err
 	}
+	// AWS answers success for a non-member when the group exists.
+	if !removed {
+		if _, err := s.getGroup(ctx, accountID, groupName); err != nil {
+			return nil, err
+		}
+		return &iam.RemoveUserFromGroupOutput{}, nil
+	}
 
 	slog.Info("IAM user removed from group", "accountID", accountID, "userName", userName, "groupName", groupName)
 	return &iam.RemoveUserFromGroupOutput{}, nil
@@ -265,7 +274,7 @@ func (s *IAMServiceImpl) ListGroupsForUser(accountID string, input *iam.ListGrou
 	for _, name := range user.Groups {
 		group, err := s.getGroup(ctx, accountID, name)
 		if err != nil {
-			if err.Error() == awserrors.ErrorIAMNoSuchEntity {
+			if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
 				// A dangling pointer to a vanished group is inert; skip it.
 				// Mirrors GetUserPolicies — transient/corrupt errors fail closed.
 				slog.Warn("ListGroupsForUser: member references missing group; skipping",
@@ -294,7 +303,7 @@ func (s *IAMServiceImpl) AttachGroupPolicy(accountID string, input *iam.AttachGr
 
 	// AWS-managed ARNs are stored opaquely (like roles); customer-managed ARNs must exist.
 	if !iamarn.IsAWSManagedPolicyARN(policyARN) {
-		if _, err := s.getPolicyByARN(ctx, accountID, policyARN); err != nil {
+		if err := s.attachablePolicy(ctx, accountID, policyARN); err != nil {
 			return nil, err
 		}
 	}
@@ -322,7 +331,7 @@ func (s *IAMServiceImpl) DetachGroupPolicy(accountID string, input *iam.DetachGr
 	err := s.updateGroupCAS(ctx, accountID, groupName, func(g *Group) (bool, error) {
 		idx := slices.Index(g.AttachedPolicies, policyARN)
 		if idx < 0 {
-			return false, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return false, policyNotFound(policyARN)
 		}
 		g.AttachedPolicies = slices.Delete(g.AttachedPolicies, idx, idx+1)
 		return true, nil
@@ -408,7 +417,7 @@ func (s *IAMServiceImpl) GetGroupPolicy(accountID string, input *iam.GetGroupPol
 
 	doc, ok := group.InlinePolicies[policyName]
 	if !ok {
-		return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+		return nil, noSuchEntity("group policy", policyName)
 	}
 
 	return &iam.GetGroupPolicyOutput{
@@ -427,7 +436,7 @@ func (s *IAMServiceImpl) DeleteGroupPolicy(accountID string, input *iam.DeleteGr
 
 	err := s.updateGroupCAS(ctx, accountID, groupName, func(g *Group) (bool, error) {
 		if _, ok := g.InlinePolicies[policyName]; !ok {
-			return false, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return false, noSuchEntity("group policy", policyName)
 		}
 		delete(g.InlinePolicies, policyName)
 		return true, nil
@@ -470,7 +479,7 @@ func (s *IAMServiceImpl) getGroup(ctx context.Context, accountID, groupName stri
 	entry, err := s.groupsBucket.Get(ctx, accountID+"."+groupName)
 	if err != nil {
 		if errors.Is(err, jetstream.ErrKeyNotFound) {
-			return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return nil, noSuchEntity("group", groupName)
 		}
 		return nil, fmt.Errorf("get group: %w", err)
 	}

spinifex/domains/iam/groups_test.go

@@ -409,6 +409,18 @@ func TestRemoveUserFromGroup_NotMember(t *testing.T) {
 		GroupName: aws.String("team"),
 		UserName:  aws.String("erin"),
 	})
+	require.NoError(t, err)
+}
+
+func TestRemoveUserFromGroup_GroupNotFound(t *testing.T) {
+	t.Parallel()
+	svc := setupTestIAMService(t)
+	createTestUser(t, svc, "erin")
+
+	_, err := svc.RemoveUserFromGroup(testAccountID, &iam.RemoveUserFromGroupInput{
+		GroupName: aws.String("ghost"),
+		UserName:  aws.String("erin"),
+	})
 	assert.Error(t, err)
 	assert.Contains(t, err.Error(), awserrors.ErrorIAMNoSuchEntity)
 }

spinifex/domains/iam/instance_profiles.go

@@ -66,7 +66,7 @@ func (s *IAMServiceImpl) CreateInstanceProfile(accountID string, input *iam.Crea
 
 	if _, err := s.instanceProfilesBucket.Create(ctx, accountID+"."+profileName, data); err != nil {
 		if errors.Is(err, jetstream.ErrKeyExists) {
-			return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists)
+			return nil, awserrors.Errorf(awserrors.ErrorIAMEntityAlreadyExists, "Instance Profile %s already exists.", profileName)
 		}
 		return nil, fmt.Errorf("store instance profile: %w", err)
 	}
@@ -172,7 +172,7 @@ func (s *IAMServiceImpl) DeleteInstanceProfile(accountID string, input *iam.Dele
 	_, err := kvutil.DeleteIf(ctx, s.instanceProfilesBucket, accountID+"."+profileName, s.instanceProfileCASConfig(accountID, profileName),
 		func(p *InstanceProfile) error {
 			if p.RoleName != "" {
-				return errors.New(awserrors.ErrorIAMDeleteConflict)
+				return awserrors.Errorf(awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must remove roles from instance profile first.")
 			}
 			return nil
 		})
@@ -195,7 +195,7 @@ func (s *IAMServiceImpl) AddRoleToInstanceProfile(accountID string, input *iam.A
 
 	err := s.updateInstanceProfileCAS(ctx, accountID, profileName, func(p *InstanceProfile) (bool, error) {
 		if p.RoleName != "" {
-			return false, errors.New(awserrors.ErrorIAMLimitExceeded)
+			return false, awserrors.Errorf(awserrors.ErrorIAMLimitExceeded, "Cannot exceed quota for InstanceSessionsPerInstanceProfile: 1")
 		}
 		p.RoleName = roleName
 		return true, nil
@@ -216,7 +216,7 @@ func (s *IAMServiceImpl) RemoveRoleFromInstanceProfile(accountID string, input *
 
 	err := s.updateInstanceProfileCAS(ctx, accountID, profileName, func(p *InstanceProfile) (bool, error) {
 		if p.RoleName == "" || p.RoleName != roleName {
-			return false, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return false, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Role %s in Instance Profile %s cannot be found.", roleName, profileName)
 		}
 		p.RoleName = ""
 		return true, nil
@@ -357,11 +357,16 @@ func (s *IAMServiceImpl) ListInstanceProfileTags(accountID string, input *iam.Li
 	}, nil
 }
 
+// instanceProfileNotFound is NoSuchEntity with AWS's message for a missing instance profile.
+func instanceProfileNotFound(profileName string) error {
+	return awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", profileName)
+}
+
 func (s *IAMServiceImpl) getInstanceProfile(ctx context.Context, accountID, profileName string) (*InstanceProfile, error) {
 	entry, err := s.instanceProfilesBucket.Get(ctx, accountID+"."+profileName)
 	if err != nil {
 		if errors.Is(err, jetstream.ErrKeyNotFound) {
-			return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return nil, instanceProfileNotFound(profileName)
 		}
 		return nil, fmt.Errorf("get instance profile: %w", err)
 	}
@@ -385,7 +390,7 @@ func (s *IAMServiceImpl) updateInstanceProfileCAS(ctx context.Context, accountID
 func (s *IAMServiceImpl) instanceProfileCASConfig(accountID, profileName string) kvutil.CASConfig {
 	return kvutil.CASConfig{
 		Attempts: instanceProfileCASMaxRetries,
-		NotFound: errors.New(awserrors.ErrorIAMNoSuchEntity),
+		NotFound: instanceProfileNotFound(profileName),
 		Exhausted: func(string, int) error {
 			slog.Warn("IAM instance profile CAS exhausted retries",
 				"accountID", accountID, "instanceProfileName", profileName, "attempts", instanceProfileCASMaxRetries)

spinifex/domains/iam/instance_profiles_test.go

@@ -778,7 +778,7 @@ func TestResolveInstanceProfile_NonCanonicalARNRejected(t *testing.T) {
 		_, err := svc.ResolveInstanceProfile(testAccountID, arn)
 		require.Error(t, err, arn)
 		// The EC2 gateway maps this to InvalidIamInstanceProfile.NotFound by exact match.
-		assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, err.Error(), arn)
+		assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, awserrors.ValidErrorCodeFromError(err), arn)
 	}
 }
 

spinifex/domains/iam/mock/mock.go

@@ -90,7 +90,7 @@ func (e *SystemInstanceRoleEnsurer) GetRole(_ string, in *iam.GetRoleInput) (*ia
 
 	r, ok := e.Roles[aws.StringValue(in.RoleName)]
 	if !ok {
-		return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "The role with name %s cannot be found.", aws.StringValue(in.RoleName))
 	}
 	return &iam.GetRoleOutput{Role: copyRole(r)}, nil
 }
@@ -104,7 +104,7 @@ func (e *SystemInstanceRoleEnsurer) CreateRole(accountID string, in *iam.CreateR
 	e.LastRoleAcct = accountID
 	name := aws.StringValue(in.RoleName)
 	if _, ok := e.Roles[name]; ok {
-		return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists)
+		return nil, awserrors.Errorf(awserrors.ErrorIAMEntityAlreadyExists, "Role with name %s already exists.", name)
 	}
 	r := &iam.Role{
 		RoleName: aws.String(name),
@@ -132,7 +132,7 @@ func (e *SystemInstanceRoleEnsurer) GetInstanceProfile(_ string, in *iam.GetInst
 
 	p, ok := e.Profiles[aws.StringValue(in.InstanceProfileName)]
 	if !ok {
-		return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", aws.StringValue(in.InstanceProfileName))
 	}
 	return &iam.GetInstanceProfileOutput{InstanceProfile: copyProfile(p)}, nil
 }
@@ -148,7 +148,7 @@ func (e *SystemInstanceRoleEnsurer) CreateInstanceProfile(accountID string, in *
 	}
 	name := aws.StringValue(in.InstanceProfileName)
 	if _, ok := e.Profiles[name]; ok {
-		return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists)
+		return nil, awserrors.Errorf(awserrors.ErrorIAMEntityAlreadyExists, "Instance Profile %s already exists.", name)
 	}
 	arn := "arn:aws:iam::" + accountID + ":instance-profile/" + name
 	if e.EmptyInstanceProfileARN {
@@ -176,7 +176,7 @@ func (e *SystemInstanceRoleEnsurer) AddRoleToInstanceProfile(_ string, in *iam.A
 		return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
 	}
 	if len(p.Roles) > 0 {
-		return nil, errors.New(awserrors.ErrorIAMLimitExceeded)
+		return nil, awserrors.Errorf(awserrors.ErrorIAMLimitExceeded, "Cannot exceed quota for InstanceSessionsPerInstanceProfile: 1")
 	}
 	p.Roles = []*iam.Role{{RoleName: in.RoleName}}
 	return &iam.AddRoleToInstanceProfileOutput{}, nil

spinifex/domains/iam/mock/mock_test.go

@@ -13,7 +13,7 @@ import (
 func TestGetRole_MissingReturnsNoSuchEntity(t *testing.T) {
 	e := mock.New()
 	_, err := e.GetRole("acct", &iam.GetRoleInput{RoleName: aws.String("r1")})
-	if err == nil || err.Error() != awserrors.ErrorIAMNoSuchEntity {
+	if err == nil || awserrors.ValidErrorCodeFromError(err) != awserrors.ErrorIAMNoSuchEntity {
 		t.Fatalf("err = %v, want ErrorIAMNoSuchEntity", err)
 	}
 }
@@ -24,7 +24,7 @@ func TestCreateRole_DuplicateReturnsEntityAlreadyExists(t *testing.T) {
 	if _, err := e.CreateRole("acct", in); err != nil {
 		t.Fatalf("first CreateRole: %v", err)
 	}
-	if _, err := e.CreateRole("acct", in); err == nil || err.Error() != awserrors.ErrorIAMEntityAlreadyExists {
+	if _, err := e.CreateRole("acct", in); err == nil || awserrors.ValidErrorCodeFromError(err) != awserrors.ErrorIAMEntityAlreadyExists {
 		t.Fatalf("second CreateRole err = %v, want EntityAlreadyExists", err)
 	}
 	if e.CreateRoleCalls != 2 {
@@ -48,7 +48,7 @@ func TestAddRoleToInstanceProfile_SecondAttachIsLimitExceeded(t *testing.T) {
 	if _, err := e.AddRoleToInstanceProfile("acct", in); err != nil {
 		t.Fatalf("first AddRoleToInstanceProfile: %v", err)
 	}
-	if _, err := e.AddRoleToInstanceProfile("acct", in); err == nil || err.Error() != awserrors.ErrorIAMLimitExceeded {
+	if _, err := e.AddRoleToInstanceProfile("acct", in); err == nil || awserrors.ValidErrorCodeFromError(err) != awserrors.ErrorIAMLimitExceeded {
 		t.Fatalf("second AddRoleToInstanceProfile err = %v, want LimitExceeded", err)
 	}
 }

spinifex/domains/iam/oidc_providers.go

@@ -178,7 +178,7 @@ func (s *IAMServiceImpl) CreateOpenIDConnectProvider(accountID string, input *ia
 
 	if _, err := kv.Create(ctx, OIDCProviderKey(issuer), data); err != nil {
 		if errors.Is(err, jetstream.ErrKeyExists) {
-			return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists)
+			return nil, awserrors.Errorf(awserrors.ErrorIAMEntityAlreadyExists, "Provider with url %s already exists.", issuer)
 		}
 		return nil, fmt.Errorf("store OIDC provider: %w", err)
 	}
@@ -268,7 +268,7 @@ func (s *IAMServiceImpl) DeleteOpenIDConnectProvider(accountID string, input *ia
 	kv, err := s.js.KeyValue(ctx, IAMAccountBucketName(accountID))
 	if err != nil {
 		if errors.Is(err, jetstream.ErrBucketNotFound) {
-			return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return nil, oidcProviderNotFound(arn)
 		}
 		return nil, fmt.Errorf("open IAM account bucket: %w", err)
 	}
@@ -276,7 +276,7 @@ func (s *IAMServiceImpl) DeleteOpenIDConnectProvider(accountID string, input *ia
 	key := OIDCProviderKey(issuer)
 	if _, err := kv.Get(ctx, key); err != nil {
 		if errors.Is(err, jetstream.ErrKeyNotFound) {
-			return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return nil, oidcProviderNotFound(arn)
 		}
 		return nil, fmt.Errorf("get OIDC provider: %w", err)
 	}
@@ -356,7 +356,7 @@ func (s *IAMServiceImpl) updateOIDCProviderCAS(ctx context.Context, accountID, a
 	kv, err := s.js.KeyValue(ctx, IAMAccountBucketName(accountID))
 	if err != nil {
 		if errors.Is(err, jetstream.ErrBucketNotFound) {
-			return errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return oidcProviderNotFound(arn)
 		}
 		return fmt.Errorf("open IAM account bucket: %w", err)
 	}
@@ -373,14 +373,14 @@ func (s *IAMServiceImpl) getOIDCProvider(ctx context.Context, accountID, arn str
 	kv, err := s.js.KeyValue(ctx, IAMAccountBucketName(accountID))
 	if err != nil {
 		if errors.Is(err, jetstream.ErrBucketNotFound) {
-			return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return nil, oidcProviderNotFound(arn)
 		}
 		return nil, fmt.Errorf("open IAM account bucket: %w", err)
 	}
 	entry, err := kv.Get(ctx, OIDCProviderKey(issuer))
 	if err != nil {
 		if errors.Is(err, jetstream.ErrKeyNotFound) {
-			return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return nil, oidcProviderNotFound(arn)
 		}
 		return nil, fmt.Errorf("get OIDC provider: %w", err)
 	}

spinifex/domains/iam/oidc_providers_test.go

@@ -50,7 +50,7 @@ func TestCreateOpenIDConnectProvider_Idempotent(t *testing.T) {
 		t.Fatalf("first create: %v", err)
 	}
 	_, err := svc.CreateOpenIDConnectProvider(acct, in)
-	if err == nil || err.Error() != awserrors.ErrorIAMEntityAlreadyExists {
+	if err == nil || awserrors.ValidErrorCodeFromError(err) != awserrors.ErrorIAMEntityAlreadyExists {
 		t.Fatalf("duplicate create err = %v, want EntityAlreadyExists", err)
 	}
 }
@@ -132,7 +132,7 @@ func TestGetOpenIDConnectProvider_NotFound(t *testing.T) {
 	_, err := svc.GetOpenIDConnectProvider("000000000001", &iam.GetOpenIDConnectProviderInput{
 		OpenIDConnectProviderArn: aws.String(OIDCProviderARN("000000000001", "10.0.0.1:9999/oidc/eks/x/y/z")),
 	})
-	if err == nil || err.Error() != awserrors.ErrorIAMNoSuchEntity {
+	if err == nil || awserrors.ValidErrorCodeFromError(err) != awserrors.ErrorIAMNoSuchEntity {
 		t.Fatalf("err = %v, want NoSuchEntity", err)
 	}
 }
@@ -195,7 +195,7 @@ func TestDeleteOpenIDConnectProvider(t *testing.T) {
 	_, err = svc.DeleteOpenIDConnectProvider(acct, &iam.DeleteOpenIDConnectProviderInput{
 		OpenIDConnectProviderArn: created.OpenIDConnectProviderArn,
 	})
-	if err == nil || err.Error() != awserrors.ErrorIAMNoSuchEntity {
+	if err == nil || awserrors.ValidErrorCodeFromError(err) != awserrors.ErrorIAMNoSuchEntity {
 		t.Fatalf("second delete err = %v, want NoSuchEntity", err)
 	}
 }

spinifex/domains/iam/policy_versions.go

@@ -111,11 +111,11 @@ func policyCASTarget(accountID, policyARN string) (string, kvutil.CASConfig, err
 	if err != nil {
 		slog.Debug("policyCASTarget: unparseable policy ARN",
 			"accountID", accountID, "policyArn", policyARN, "err", err)
-		return "", kvutil.CASConfig{}, errors.New(awserrors.ErrorIAMNoSuchEntity)
+		return "", kvutil.CASConfig{}, policyNotFound(policyARN)
 	}
 	return accountID + "." + policyName, kvutil.CASConfig{
 		Attempts: policyCASMaxRetries,
-		NotFound: errors.New(awserrors.ErrorIAMNoSuchEntity),
+		NotFound: policyNotFound(policyARN),
 		Exhausted: func(string, int) error {
 			slog.Error("IAM policy CAS retries exhausted under contention",
 				"accountID", accountID, "policyName", policyName, "attempts", policyCASMaxRetries)

spinifex/domains/iam/records_cas_test.go

@@ -224,11 +224,11 @@ func TestTagUser_DoesNotRecreateConcurrentlyDeletedUser(t *testing.T) {
 	close(gate.release)
 	err = <-done
 	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, err.Error())
+	assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, awserrors.ValidErrorCodeFromError(err))
 
 	_, err = svc.getUser(t.Context(), testAccountID, *user.UserName)
 	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, err.Error(), "the deleted user must stay deleted")
+	assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, awserrors.ValidErrorCodeFromError(err), "the deleted user must stay deleted")
 }
 
 // Every group writer that read the record before a DetachGroupPolicy and writes
@@ -376,7 +376,7 @@ func TestOIDCProviderTagWriters_DoNotRecreateDeletedProvider(t *testing.T) {
 			close(gate.release)
 			err = <-done
 			require.Error(t, err)
-			assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, err.Error())
+			assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, awserrors.ValidErrorCodeFromError(err))
 
 			_, err = kv.Get(t.Context(), OIDCProviderKey(testIssuer))
 			assert.ErrorIs(t, err, jetstream.ErrKeyNotFound, "the deleted provider must stay deleted")

spinifex/domains/iam/role_resolver.go

@@ -33,7 +33,7 @@ func (r *RoleResolver) ResolveRole(accountID, roleARN string) (passrole.Role, er
 	}
 	role, err := r.svc.getRole(context.Background(), accountID, name)
 	if err != nil {
-		if err.Error() == awserrors.ErrorIAMNoSuchEntity {
+		if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
 			return passrole.Role{}, passrole.ErrNotFound
 		}
 		return passrole.Role{}, err

spinifex/domains/iam/roles.go

@@ -106,7 +106,7 @@ func (s *IAMServiceImpl) CreateRole(accountID string, input *iam.CreateRoleInput
 	kvKey := accountID + "." + roleName
 	if _, err := s.rolesBucket.Create(ctx, kvKey, data); err != nil {
 		if errors.Is(err, jetstream.ErrKeyExists) {
-			return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists)
+			return nil, awserrors.Errorf(awserrors.ErrorIAMEntityAlreadyExists, "Role with name %s already exists.", roleName)
 		}
 		return nil, fmt.Errorf("store role: %w", err)
 	}
@@ -195,11 +195,11 @@ func (s *IAMServiceImpl) DeleteRole(accountID string, input *iam.DeleteRoleInput
 	}
 
 	if len(role.AttachedPolicies) > 0 {
-		return nil, errors.New(awserrors.ErrorIAMDeleteConflict)
+		return nil, errDetachPoliciesFirst
 	}
 
 	if len(role.InlinePolicies) > 0 {
-		return nil, errors.New(awserrors.ErrorIAMDeleteConflict)
+		return nil, errDeletePoliciesFirst
 	}
 
 	profiles, err := s.findInstanceProfilesForRole(ctx, accountID, roleName)
@@ -207,7 +207,7 @@ func (s *IAMServiceImpl) DeleteRole(accountID string, input *iam.DeleteRoleInput
 		return nil, fmt.Errorf("check role instance profiles: %w", err)
 	}
 	if len(profiles) > 0 {
-		return nil, errors.New(awserrors.ErrorIAMDeleteConflict)
+		return nil, awserrors.Errorf(awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must remove roles from instance profile first.")
 	}
 
 	if err := s.rolesBucket.Delete(ctx, accountID+"."+roleName); err != nil {
@@ -291,7 +291,7 @@ func (s *IAMServiceImpl) UpdateRoleDescription(accountID string, input *iam.Upda
 	}
 
 	slog.Info("IAM role description updated", "accountID", accountID, "roleName", roleName)
-	return &iam.UpdateRoleDescriptionOutput{Role: gotRoleToSDK(role)}, nil
+	return &iam.UpdateRoleDescriptionOutput{Role: describedRoleToSDK(role)}, nil
 }
 
 func (s *IAMServiceImpl) UpdateAssumeRolePolicy(accountID string, input *iam.UpdateAssumeRolePolicyInput) (*iam.UpdateAssumeRolePolicyOutput, error) {
@@ -326,7 +326,7 @@ func (s *IAMServiceImpl) AttachRolePolicy(accountID string, input *iam.AttachRol
 	// so ListAttachedRolePolicies / DescribeNodegroup round-trip instead of
 	// failing NoSuchEntity. Customer-managed ARNs must still exist.
 	if !iamarn.IsAWSManagedPolicyARN(policyARN) {
-		if _, err := s.getPolicyByARN(ctx, accountID, policyARN); err != nil {
+		if err := s.attachablePolicy(ctx, accountID, policyARN); err != nil {
 			return nil, err
 		}
 	}
@@ -354,7 +354,7 @@ func (s *IAMServiceImpl) DetachRolePolicy(accountID string, input *iam.DetachRol
 	err := s.updateRoleCAS(ctx, accountID, roleName, func(role *Role) (bool, error) {
 		idx := slices.Index(role.AttachedPolicies, policyARN)
 		if idx < 0 {
-			return false, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return false, policyNotFound(policyARN)
 		}
 		role.AttachedPolicies = slices.Delete(role.AttachedPolicies, idx, idx+1)
 		return true, nil
@@ -444,7 +444,7 @@ func (s *IAMServiceImpl) GetRolePolicy(accountID string, input *iam.GetRolePolic
 
 	doc, ok := role.InlinePolicies[policyName]
 	if !ok {
-		return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+		return nil, noSuchEntity("role policy", policyName)
 	}
 
 	return &iam.GetRolePolicyOutput{
@@ -463,7 +463,7 @@ func (s *IAMServiceImpl) DeleteRolePolicy(accountID string, input *iam.DeleteRol
 
 	err := s.updateRoleCAS(ctx, accountID, roleName, func(role *Role) (bool, error) {
 		if _, ok := role.InlinePolicies[policyName]; !ok {
-			return false, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return false, noSuchEntity("role policy", policyName)
 		}
 		delete(role.InlinePolicies, policyName)
 		return true, nil
@@ -600,7 +600,7 @@ func (s *IAMServiceImpl) getRole(ctx context.Context, accountID, roleName string
 	entry, err := s.rolesBucket.Get(ctx, accountID+"."+roleName)
 	if err != nil {
 		if errors.Is(err, jetstream.ErrKeyNotFound) {
-			return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return nil, noSuchEntity("role", roleName)
 		}
 		return nil, fmt.Errorf("get role: %w", err)
 	}
@@ -622,7 +622,7 @@ func (s *IAMServiceImpl) updateRoleCAS(ctx context.Context, accountID, roleName
 	defer s.recordLocks.lock("role/" + accountID + "." + roleName)()
 	_, err := kvutil.Update(ctx, s.rolesBucket, accountID+"."+roleName, kvutil.CASConfig{
 		Attempts: roleCASMaxRetries,
-		NotFound: errors.New(awserrors.ErrorIAMNoSuchEntity),
+		NotFound: noSuchEntity("role", roleName),
 		Exhausted: func(string, int) error {
 			// Only a CAS conflict reaches here, so the role is contended rather than
 			// broken; say so, or the caller's InternalError has no cause anywhere.
@@ -698,6 +698,16 @@ func createdRoleToSDK(r *Role) *iam.Role {
 	return out
 }
 
+// describedRoleToSDK is UpdateRoleDescription's shape, which carries no
+// MaxSessionDuration, Tags or RoleLastUsed.
+func describedRoleToSDK(r *Role) *iam.Role {
+	out := roleToSDK(r)
+	if r.Description != "" {
+		out.Description = aws.String(r.Description)
+	}
+	return out
+}
+
 // listedRoleToSDK is ListRoles' shape, which carries no Tags or RoleLastUsed.
 func listedRoleToSDK(r *Role) *iam.Role {
 	out := roleToSDK(r)

spinifex/domains/iam/roles_test.go

@@ -561,7 +561,13 @@ func TestUpdateRole_NotFound(t *testing.T) {
 func TestUpdateRoleDescription(t *testing.T) {
 	t.Parallel()
 	svc := setupTestIAMService(t)
-	createTestRole(t, svc, "desc-role")
+	_, err := svc.CreateRole(testAccountID, &iam.CreateRoleInput{
+		RoleName:                 aws.String("desc-role"),
+		AssumeRolePolicyDocument: aws.String(validTrustPolicy()),
+		MaxSessionDuration:       aws.Int64(7200),
+		Tags:                     []*iam.Tag{{Key: aws.String("team"), Value: aws.String("infra")}},
+	})
+	require.NoError(t, err)
 
 	out, err := svc.UpdateRoleDescription(testAccountID, &iam.UpdateRoleDescriptionInput{
 		RoleName:    aws.String("desc-role"),
@@ -571,7 +577,15 @@ func TestUpdateRoleDescription(t *testing.T) {
 	got, err := svc.GetRole(testAccountID, &iam.GetRoleInput{RoleName: aws.String("desc-role")})
 	require.NoError(t, err)
 	assert.Equal(t, "new description", *got.Role.Description)
-	assert.Equal(t, got.Role, out.Role, "output carries the role in the GetRole shape")
+	assert.Equal(t, &iam.Role{
+		Path:                     got.Role.Path,
+		RoleName:                 got.Role.RoleName,
+		RoleId:                   got.Role.RoleId,
+		Arn:                      got.Role.Arn,
+		CreateDate:               got.Role.CreateDate,
+		AssumeRolePolicyDocument: got.Role.AssumeRolePolicyDocument,
+		Description:              aws.String("new description"),
+	}, out.Role, "AWS omits MaxSessionDuration, Tags and RoleLastUsed from this output")
 
 	_, err = svc.UpdateRoleDescription(testAccountID, &iam.UpdateRoleDescriptionInput{
 		RoleName:    aws.String("desc-role"),

spinifex/domains/iam/service.go

@@ -15,6 +15,7 @@ type IAMService interface {
 	GetUser(accountID string, input *iam.GetUserInput) (*iam.GetUserOutput, error)
 	ListUsers(accountID string, input *iam.ListUsersInput) (*iam.ListUsersOutput, error)
 	DeleteUser(accountID string, input *iam.DeleteUserInput) (*iam.DeleteUserOutput, error)
+	UpdateUser(accountID string, input *iam.UpdateUserInput) (*iam.UpdateUserOutput, error)
 
 	// Access key lifecycle — account-scoped
 	CreateAccessKey(accountID string, input *iam.CreateAccessKeyInput) (*iam.CreateAccessKeyOutput, error)

spinifex/domains/iam/service_impl.go

@@ -467,7 +467,7 @@ func (s *IAMServiceImpl) CreateUser(accountID string, input *iam.CreateUserInput
 	// Atomic create — fails if key already exists (race-safe)
 	if _, err := s.usersBucket.Create(ctx, kvKey, data); err != nil {
 		if errors.Is(err, jetstream.ErrKeyExists) {
-			return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists)
+			return nil, awserrors.Errorf(awserrors.ErrorIAMEntityAlreadyExists, "User with name %s already exists.", userName)
 		}
 		return nil, fmt.Errorf("store user: %w", err)
 	}
@@ -587,7 +587,7 @@ func (s *IAMServiceImpl) DeleteUser(accountID string, input *iam.DeleteUserInput
 	// block the delete either; removing the user record drops it.
 	for _, keyID := range user.AccessKeys {
 		if _, err := s.accessKeysBucket.Get(ctx, keyID); err == nil {
-			return nil, errors.New(awserrors.ErrorIAMDeleteConflict)
+			return nil, awserrors.Errorf(awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must delete access keys first.")
 		} else if !errors.Is(err, jetstream.ErrKeyNotFound) {
 			return nil, fmt.Errorf("get access key: %w", err)
 		}
@@ -595,13 +595,13 @@ func (s *IAMServiceImpl) DeleteUser(accountID string, input *iam.DeleteUserInput
 			"accountID", accountID, "userName", userName, "accessKeyID", keyID)
 	}
 	if len(user.AttachedPolicies) > 0 {
-		return nil, errors.New(awserrors.ErrorIAMDeleteConflict)
+		return nil, errDetachPoliciesFirst
 	}
 	if len(user.Groups) > 0 {
 		return nil, errors.New(awserrors.ErrorIAMDeleteConflict)
 	}
 	if len(user.InlinePolicies) > 0 {
-		return nil, errors.New(awserrors.ErrorIAMDeleteConflict)
+		return nil, errDeletePoliciesFirst
 	}
 
 	if err := s.usersBucket.Delete(ctx, kvKey); err != nil {
@@ -625,6 +625,133 @@ func (s *IAMServiceImpl) DeleteUser(accountID string, input *iam.DeleteUserInput
 	return &iam.DeleteUserOutput{}, nil
 }
 
+// UpdateUser applies NewPath and NewUserName to a user. A rename moves the
+// record to its new key, keeping its ID, access keys, groups and policies.
+func (s *IAMServiceImpl) UpdateUser(accountID string, input *iam.UpdateUserInput) (*iam.UpdateUserOutput, error) {
+	ctx := context.Background()
+	userName := *input.UserName
+	if input.NewUserName != nil {
+		if err := validateIAMName("newUserName", *input.NewUserName, 64); err != nil {
+			return nil, err
+		}
+	}
+	if input.NewPath != nil {
+		if err := validatePath(*input.NewPath); err != nil {
+			return nil, err
+		}
+	}
+
+	newName := aws.StringValue(input.NewUserName)
+	var err error
+	if newName == "" || newName == userName {
+		err = s.updateUserCAS(ctx, accountID, userName, func(u *User) (bool, error) {
+			if input.NewPath == nil || *input.NewPath == u.Path {
+				return false, nil
+			}
+			u.Path = *input.NewPath
+			u.ARN = arn.FormatIAMPath(arn.IAMUser, accountID, u.Path, u.UserName)
+			return true, nil
+		})
+	} else {
+		err = s.renameUser(ctx, accountID, userName, newName, input.NewPath)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	slog.Info("IAM user updated", "accountID", accountID, "userName", userName, "newUserName", newName)
+	return &iam.UpdateUserOutput{}, nil
+}
+
+// renameUser writes the user under newName and repoints its access keys before
+// deleting the old record at the revision it read. A racing writer of the old
+// record makes that delete fail, so the rename is undone and retried.
+func (s *IAMServiceImpl) renameUser(ctx context.Context, accountID, oldName, newName string, newPath *string) error {
+	oldKey, newKey := accountID+"."+oldName, accountID+"."+newName
+	first, second := oldKey, newKey
+	if second < first {
+		first, second = second, first
+	}
+	defer s.recordLocks.lock("user/" + first)()
+	defer s.recordLocks.lock("user/" + second)()
+
+	for range roleCASMaxRetries {
+		entry, err := s.usersBucket.Get(ctx, oldKey)
+		if err != nil {
+			if errors.Is(err, jetstream.ErrKeyNotFound) {
+				return errors.New(awserrors.ErrorIAMNoSuchEntity)
+			}
+			return fmt.Errorf("get user: %w", err)
+		}
+		var user User
+		if err := json.Unmarshal(entry.Value(), &user); err != nil {
+			return fmt.Errorf("unmarshal user: %w", err)
+		}
+		user.UserName = newName
+		if newPath != nil {
+			user.Path = *newPath
+		}
+		user.ARN = arn.FormatIAMPath(arn.IAMUser, accountID, user.Path, newName)
+		data, err := json.Marshal(user)
+		if err != nil {
+			return fmt.Errorf("marshal user: %w", err)
+		}
+
+		newRev, err := s.usersBucket.Create(ctx, newKey, data)
+		if err != nil {
+			if errors.Is(err, jetstream.ErrKeyExists) {
+				return errors.New(awserrors.ErrorIAMEntityAlreadyExists)
+			}
+			return fmt.Errorf("store renamed user: %w", err)
+		}
+		err = s.setAccessKeysOwner(ctx, accountID, user.AccessKeys, newName)
+		if err == nil {
+			err = s.usersBucket.Delete(ctx, oldKey, jetstream.LastRevision(entry.Revision()))
+			if err == nil {
+				return nil
+			}
+		}
+		s.undoRenameUser(ctx, accountID, user.AccessKeys, oldName, newKey, newRev)
+		if !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
+			return fmt.Errorf("rename user: %w", err)
+		}
+	}
+	slog.Error("IAM user rename lost every revision race on the old record",
+		"accountID", accountID, "userName", oldName, "attempts", roleCASMaxRetries)
+	return errors.New(awserrors.ErrorServerInternal)
+}
+
+// setAccessKeysOwner points each access key at userName. A listed key with no
+// record is skipped, as ListAccessKeys skips it.
+func (s *IAMServiceImpl) setAccessKeysOwner(ctx context.Context, accountID string, keyIDs []string, userName string) error {
+	for _, keyID := range keyIDs {
+		cfg := iamRecordCASConfig("access key", accountID, keyID)
+		cfg.NotFound = nil
+		_, err := kvutil.Update(ctx, s.accessKeysBucket, keyID, cfg, func(ak *AccessKey) (bool, error) {
+			if ak.UserName == userName {
+				return false, nil
+			}
+			ak.UserName = userName
+			return true, nil
+		})
+		if err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
+			return fmt.Errorf("repoint access key %s: %w", keyID, err)
+		}
+	}
+	return nil
+}
+
+// undoRenameUser returns the access keys to oldName and drops the record
+// written under the new name, unless a writer has changed it since.
+func (s *IAMServiceImpl) undoRenameUser(ctx context.Context, accountID string, keyIDs []string, oldName, newKey string, newRev uint64) {
+	if err := s.setAccessKeysOwner(ctx, accountID, keyIDs, oldName); err != nil {
+		slog.Error("Rollback failed: access keys left on renamed user", "accountID", accountID, "userName", oldName, "err", err)
+	}
+	if err := s.usersBucket.Delete(ctx, newKey, jetstream.LastRevision(newRev)); err != nil {
+		slog.Error("Rollback failed: renamed user record left behind", "accountID", accountID, "userName", oldName, "key", newKey, "err", err)
+	}
+}
+
 // ---------------------------------------------------------------------------
 // Access Key Lifecycle
 // ---------------------------------------------------------------------------
@@ -753,7 +880,7 @@ func (s *IAMServiceImpl) DeleteAccessKey(accountID string, input *iam.DeleteAcce
 	err := s.updateUserCAS(ctx, accountID, userName, func(u *User) (bool, error) {
 		idx := slices.Index(u.AccessKeys, accessKeyID)
 		if idx < 0 {
-			return false, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return false, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "The Access Key with id %s cannot be found.", accessKeyID)
 		}
 		u.AccessKeys = slices.Delete(u.AccessKeys, idx, idx+1)
 		return true, nil
@@ -1355,7 +1482,8 @@ func (s *IAMServiceImpl) CreatePolicy(accountID string, input *iam.CreatePolicyI
 
 	if _, err := s.policiesBucket.Create(ctx, kvKey, data); err != nil {
 		if errors.Is(err, jetstream.ErrKeyExists) {
-			return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists)
+			return nil, awserrors.Errorf(awserrors.ErrorIAMEntityAlreadyExists,
+				"A policy called %s already exists. Duplicate names are not allowed.", policyName)
 		}
 		return nil, fmt.Errorf("store policy: %w", err)
 	}
@@ -1502,7 +1630,7 @@ func (s *IAMServiceImpl) DeletePolicy(accountID string, input *iam.DeletePolicyI
 	// Revision-guarded, so a version created after the checks cannot be deleted unseen.
 	policy, err := kvutil.DeleteIf(ctx, s.policiesBucket, key, cfg, func(p *Policy) error {
 		if p.ARN != policyARN {
-			return errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return policyNotFound(policyARN)
 		}
 		if len(p.OtherVersions) > 0 {
 			return awserrors.Errorf(awserrors.ErrorIAMDeleteConflict,
@@ -1513,7 +1641,7 @@ func (s *IAMServiceImpl) DeletePolicy(accountID string, input *iam.DeletePolicyI
 			return fmt.Errorf("check policy attachments: %w", err)
 		}
 		if counts[p.ARN] > 0 {
-			return errors.New(awserrors.ErrorIAMDeleteConflict)
+			return awserrors.Errorf(awserrors.ErrorIAMDeleteConflict, "Cannot delete a policy attached to entities.")
 		}
 		return nil
 	})
@@ -1612,7 +1740,7 @@ func (s *IAMServiceImpl) AttachUserPolicy(accountID string, input *iam.AttachUse
 	// grant document is modeled in builtinManagedPolicyDoc and resolved at
 	// evaluation time. Customer-managed ARNs must still exist.
 	if !iamarn.IsAWSManagedPolicyARN(policyARN) {
-		if _, err := s.getPolicyByARN(ctx, accountID, policyARN); err != nil {
+		if err := s.attachablePolicy(ctx, accountID, policyARN); err != nil {
 			return nil, err
 		}
 	}
@@ -1640,7 +1768,7 @@ func (s *IAMServiceImpl) DetachUserPolicy(accountID string, input *iam.DetachUse
 	err := s.updateUserCAS(ctx, accountID, userName, func(u *User) (bool, error) {
 		idx := slices.Index(u.AttachedPolicies, policyARN)
 		if idx < 0 {
-			return false, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return false, policyNotFound(policyARN)
 		}
 		u.AttachedPolicies = slices.Delete(u.AttachedPolicies, idx, idx+1)
 		return true, nil
@@ -1774,7 +1902,7 @@ func (s *IAMServiceImpl) GetUserPolicy(accountID string, input *iam.GetUserPolic
 
 	doc, ok := user.InlinePolicies[policyName]
 	if !ok {
-		return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+		return nil, noSuchEntity("user policy", policyName)
 	}
 
 	return &iam.GetUserPolicyOutput{
@@ -1793,7 +1921,7 @@ func (s *IAMServiceImpl) DeleteUserPolicy(accountID string, input *iam.DeleteUse
 
 	err := s.updateUserCAS(ctx, accountID, userName, func(u *User) (bool, error) {
 		if _, ok := u.InlinePolicies[policyName]; !ok {
-			return false, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return false, noSuchEntity("user policy", policyName)
 		}
 		delete(u.InlinePolicies, policyName)
 		return true, nil
@@ -1984,7 +2112,7 @@ func (s *IAMServiceImpl) GetUserPolicies(accountID, userName string) ([]PolicyDo
 	for _, groupName := range user.Groups {
 		group, err := s.getGroup(ctx, accountID, groupName)
 		if err != nil {
-			if err.Error() == awserrors.ErrorIAMNoSuchEntity {
+			if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
 				slog.Warn("GetUserPolicies: member references missing group; skipping",
 					"accountID", accountID, "user", userName, "group", groupName)
 				continue
@@ -2025,7 +2153,7 @@ func (s *IAMServiceImpl) getPolicyByARN(ctx context.Context, accountID, policyAR
 		entry, err := s.policiesBucket.Get(ctx, kvKey)
 		if err != nil {
 			if errors.Is(err, jetstream.ErrKeyNotFound) {
-				return "", errors.New(awserrors.ErrorIAMNoSuchEntity)
+				return "", policyNotFound(policyARN)
 			}
 			return "", fmt.Errorf("get policy: %w", err)
 		}
@@ -2042,9 +2170,9 @@ func (s *IAMServiceImpl) getPolicyByARN(ctx context.Context, accountID, policyAR
 		// otherwise a malformed ARN is indistinguishable from a missing policy.
 		slog.Debug("getPolicyByARN: unparseable policy ARN",
 			"accountID", accountID, "policyArn", policyARN, "err", err)
-		return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+		return nil, policyNotFound(policyARN)
 	case errors.Is(err, iamarn.ErrPolicyARNMismatch):
-		return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+		return nil, policyNotFound(policyARN)
 	case err != nil:
 		return nil, err
 	}
@@ -2166,12 +2294,45 @@ func (s *IAMServiceImpl) buildAttachmentCounts(ctx context.Context, accountID st
 	return counts, nil
 }
 
+// noSuchEntity is NoSuchEntity with AWS's message naming the missing user,
+// role, group or inline policy, e.g. "The role with name X cannot be found.".
+func noSuchEntity(kind, name string) error {
+	return awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "The %s with name %s cannot be found.", kind, name)
+}
+
+// policyNotFound is NoSuchEntity with AWS's message for a managed policy ARN
+// that names no stored policy, or none attached to the entity.
+func policyNotFound(policyARN string) error {
+	return awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Policy %s was not found.", policyARN)
+}
+
+// oidcProviderNotFound is NoSuchEntity with AWS's message for a missing OIDC provider.
+func oidcProviderNotFound(providerARN string) error {
+	return awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "OpenIDConnect Provider not found for arn %s", providerARN)
+}
+
+// The DeleteConflict messages AWS shares across users, roles and groups.
+var (
+	errDetachPoliciesFirst = awserrors.Errorf(awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must detach all policies first.")
+	errDeletePoliciesFirst = awserrors.Errorf(awserrors.ErrorIAMDeleteConflict, "Cannot delete entity, must delete policies first.")
+)
+
+// attachablePolicy refuses a customer-managed policy ARN that names no stored
+// policy, with the message AWS gives on attach.
+func (s *IAMServiceImpl) attachablePolicy(ctx context.Context, accountID, policyARN string) error {
+	_, err := s.getPolicyByARN(ctx, accountID, policyARN)
+	if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
+		return awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Policy %s does not exist or is not attachable.", policyARN)
+	}
+	return err
+}
+
 func (s *IAMServiceImpl) getUser(ctx context.Context, accountID, userName string) (*User, error) {
 	kvKey := accountID + "." + userName
 	entry, err := s.usersBucket.Get(ctx, kvKey)
 	if err != nil {
 		if errors.Is(err, jetstream.ErrKeyNotFound) {
-			return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+			return nil, noSuchEntity("user", userName)
 		}
 		return nil, fmt.Errorf("get user: %w", err)
 	}
@@ -2195,9 +2356,13 @@ func (s *IAMServiceImpl) updateUserCAS(ctx context.Context, accountID, userName
 // iamRecordCASConfig is the CAS policy for an IAM user, group or OIDC provider
 // record: a missing record is NoSuchEntity and exhausted retries are logged.
 func iamRecordCASConfig(kind, accountID, name string) kvutil.CASConfig {
+	notFound := noSuchEntity(kind, name)
+	if kind == "oidc-provider" {
+		notFound = oidcProviderNotFound(name)
+	}
 	return kvutil.CASConfig{
 		Attempts: roleCASMaxRetries,
-		NotFound: errors.New(awserrors.ErrorIAMNoSuchEntity),
+		NotFound: notFound,
 		Exhausted: func(string, int) error {
 			slog.Error("IAM record CAS retries exhausted under contention",
 				"kind", kind, "accountID", accountID, "name", name, "attempts", roleCASMaxRetries)

spinifex/domains/iam/system_role.go

@@ -48,7 +48,7 @@ func EnsureSystemInstanceProfile(e SystemInstanceRoleEnsurer, accountID, roleNam
 // there draws credentials from it, so provisioning one would be unasked-for.
 func ConvergeSystemRolePolicy(e SystemInstanceRoleEnsurer, accountID, roleName, policyName, policyDoc string) error {
 	if _, err := e.GetRole(accountID, &iam.GetRoleInput{RoleName: aws.String(roleName)}); err != nil {
-		if err.Error() == awserrors.ErrorIAMNoSuchEntity {
+		if code, _ := awserrors.ResolveErrorCode(err); code == awserrors.ErrorIAMNoSuchEntity {
 			return nil
 		}
 		return fmt.Errorf("get role %q: %w", roleName, err)
@@ -68,7 +68,7 @@ func ConvergeSystemRolePolicy(e SystemInstanceRoleEnsurer, accountID, roleName,
 func ensureSystemRole(e SystemInstanceRoleEnsurer, accountID, roleName string) error {
 	if _, err := e.GetRole(accountID, &iam.GetRoleInput{RoleName: aws.String(roleName)}); err == nil {
 		return nil
-	} else if err.Error() != awserrors.ErrorIAMNoSuchEntity {
+	} else if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMNoSuchEntity {
 		return fmt.Errorf("get role %q: %w", roleName, err)
 	}
 
@@ -77,7 +77,7 @@ func ensureSystemRole(e SystemInstanceRoleEnsurer, accountID, roleName string) e
 		AssumeRolePolicyDocument: aws.String(EC2InstanceTrustPolicy),
 		Description:              aws.String("Spinifex-managed system instance role"),
 	})
-	if err == nil || err.Error() == awserrors.ErrorIAMEntityAlreadyExists {
+	if code, _ := awserrors.ResolveErrorCode(err); err == nil || code == awserrors.ErrorIAMEntityAlreadyExists {
 		return nil
 	}
 	return fmt.Errorf("create role %q: %w", roleName, err)
@@ -94,7 +94,7 @@ func ensureSystemInstanceProfile(e SystemInstanceRoleEnsurer, accountID, roleNam
 		return attachSystemRole(e, accountID, roleName,
 			aws.StringValue(out.InstanceProfile.Arn), len(out.InstanceProfile.Roles) > 0)
 	}
-	if err.Error() != awserrors.ErrorIAMNoSuchEntity {
+	if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMNoSuchEntity {
 		return "", fmt.Errorf("get instance profile %q: %w", roleName, err)
 	}
 
@@ -102,7 +102,7 @@ func ensureSystemInstanceProfile(e SystemInstanceRoleEnsurer, accountID, roleNam
 		InstanceProfileName: aws.String(roleName),
 	})
 	if err != nil {
-		if err.Error() != awserrors.ErrorIAMEntityAlreadyExists {
+		if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMEntityAlreadyExists {
 			return "", fmt.Errorf("create instance profile %q: %w", roleName, err)
 		}
 		got, gerr := e.GetInstanceProfile(accountID, &iam.GetInstanceProfileInput{
@@ -128,7 +128,7 @@ func attachSystemRole(e SystemInstanceRoleEnsurer, accountID, roleName, profileA
 		InstanceProfileName: aws.String(roleName),
 		RoleName:            aws.String(roleName),
 	})
-	if err != nil && err.Error() != awserrors.ErrorIAMLimitExceeded {
+	if code, _ := awserrors.ResolveErrorCode(err); err != nil && code != awserrors.ErrorIAMLimitExceeded {
 		return "", fmt.Errorf("add role %q to instance profile %q: %w", roleName, roleName, err)
 	}
 	return profileARN, nil

spinifex/domains/iam/tags_test.go

@@ -460,7 +460,7 @@ func TestCreate_InvalidTagsRefusedAndNotStored(t *testing.T) {
 
 				_, err = ops.list(svc, id)
 				require.Error(t, err)
-				assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, err.Error())
+				assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, awserrors.ValidErrorCodeFromError(err))
 			})
 		}
 	}
@@ -577,15 +577,15 @@ func TestResourceTagging_MissingEntity(t *testing.T) {
 
 			err := ops.tag(svc, ops.missingID, []*iam.Tag{sdkTag("env", "prod")})
 			require.Error(t, err)
-			assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, err.Error())
+			assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, awserrors.ValidErrorCodeFromError(err))
 
 			err = ops.untag(svc, ops.missingID, []*string{aws.String("env")})
 			require.Error(t, err)
-			assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, err.Error())
+			assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, awserrors.ValidErrorCodeFromError(err))
 
 			_, err = ops.list(svc, ops.missingID)
 			require.Error(t, err)
-			assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, err.Error())
+			assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, awserrors.ValidErrorCodeFromError(err))
 		})
 	}
 }
@@ -608,7 +608,7 @@ func TestTagUser_MergedLimitExceeded(t *testing.T) {
 		Tags:     []*iam.Tag{sdkTag("overflow", "v")},
 	})
 	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorIAMLimitExceeded, err.Error())
+	assert.Equal(t, awserrors.ErrorIAMLimitExceeded, awserrors.ValidErrorCodeFromError(err))
 
 	// Upserting an existing key does not grow the set and stays allowed.
 	_, err = svc.TagUser(testAccountID, &iam.TagUserInput{

spinifex/domains/iam/user_update_test.go

@@ -0,0 +1,171 @@
+package iam
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestUpdateUser_RenameKeepsRelationships(t *testing.T) {
+	t.Parallel()
+	svc := setupTestIAMService(t)
+	before := createTestUser(t, svc, "bob")
+	policy := createTestPolicy(t, svc, "bob-managed")
+	createTestGroup(t, svc, "devs")
+
+	_, err := svc.AttachUserPolicy(testAccountID, &iam.AttachUserPolicyInput{UserName: aws.String("bob"), PolicyArn: policy.Arn})
+	require.NoError(t, err)
+	_, err = svc.PutUserPolicy(testAccountID, &iam.PutUserPolicyInput{
+		UserName: aws.String("bob"), PolicyName: aws.String("inline"), PolicyDocument: aws.String(validPolicyDocument()),
+	})
+	require.NoError(t, err)
+	_, err = svc.AddUserToGroup(testAccountID, &iam.AddUserToGroupInput{UserName: aws.String("bob"), GroupName: aws.String("devs")})
+	require.NoError(t, err)
+	key, err := svc.CreateAccessKey(testAccountID, &iam.CreateAccessKeyInput{UserName: aws.String("bob")})
+	require.NoError(t, err)
+
+	out, err := svc.UpdateUser(testAccountID, &iam.UpdateUserInput{
+		UserName: aws.String("bob"), NewUserName: aws.String("robert"), NewPath: aws.String("/eng/"),
+	})
+	require.NoError(t, err)
+	assert.Equal(t, &iam.UpdateUserOutput{}, out)
+
+	_, err = svc.GetUser(testAccountID, &iam.GetUserInput{UserName: aws.String("bob")})
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, err.Error())
+
+	got, err := svc.GetUser(testAccountID, &iam.GetUserInput{UserName: aws.String("robert")})
+	require.NoError(t, err)
+	assert.Equal(t, *before.UserId, *got.User.UserId)
+	assert.Equal(t, "/eng/", *got.User.Path)
+	assert.Equal(t, "arn:aws:iam::"+testAccountID+":user/eng/robert", *got.User.Arn)
+
+	ak, err := svc.LookupAccessKey(*key.AccessKey.AccessKeyId)
+	require.NoError(t, err)
+	assert.Equal(t, "robert", ak.UserName)
+	keys, err := svc.ListAccessKeys(testAccountID, &iam.ListAccessKeysInput{UserName: aws.String("robert")})
+	require.NoError(t, err)
+	require.Len(t, keys.AccessKeyMetadata, 1)
+	assert.Equal(t, *key.AccessKey.AccessKeyId, *keys.AccessKeyMetadata[0].AccessKeyId)
+
+	attached, err := svc.ListAttachedUserPolicies(testAccountID, &iam.ListAttachedUserPoliciesInput{UserName: aws.String("robert")})
+	require.NoError(t, err)
+	require.Len(t, attached.AttachedPolicies, 1)
+	assert.Equal(t, *policy.Arn, *attached.AttachedPolicies[0].PolicyArn)
+
+	inline, err := svc.ListUserPolicies(testAccountID, &iam.ListUserPoliciesInput{UserName: aws.String("robert")})
+	require.NoError(t, err)
+	assert.Equal(t, []*string{aws.String("inline")}, inline.PolicyNames)
+
+	groups, err := svc.ListGroupsForUser(testAccountID, &iam.ListGroupsForUserInput{UserName: aws.String("robert")})
+	require.NoError(t, err)
+	require.Len(t, groups.Groups, 1)
+	assert.Equal(t, "devs", *groups.Groups[0].GroupName)
+
+	members, err := svc.GetGroup(testAccountID, &iam.GetGroupInput{GroupName: aws.String("devs")})
+	require.NoError(t, err)
+	require.Len(t, members.Users, 1)
+	assert.Equal(t, "robert", *members.Users[0].UserName)
+}
+
+func TestUpdateUser_PathOnly(t *testing.T) {
+	t.Parallel()
+	svc := setupTestIAMService(t)
+	createTestUser(t, svc, "carol")
+
+	_, err := svc.UpdateUser(testAccountID, &iam.UpdateUserInput{UserName: aws.String("carol"), NewPath: aws.String("/ops/")})
+	require.NoError(t, err)
+
+	got, err := svc.GetUser(testAccountID, &iam.GetUserInput{UserName: aws.String("carol")})
+	require.NoError(t, err)
+	assert.Equal(t, "/ops/", *got.User.Path)
+	assert.Equal(t, "arn:aws:iam::"+testAccountID+":user/ops/carol", *got.User.Arn)
+}
+
+func TestUpdateUser_Errors(t *testing.T) {
+	t.Parallel()
+	svc := setupTestIAMService(t)
+	createTestUser(t, svc, "dave")
+	createTestUser(t, svc, "erin")
+
+	tests := []struct {
+		name  string
+		input *iam.UpdateUserInput
+		want  string
+	}{
+		{"missing user", &iam.UpdateUserInput{UserName: aws.String("ghost"), NewPath: aws.String("/x/")}, awserrors.ErrorIAMNoSuchEntity},
+		{"missing user rename", &iam.UpdateUserInput{UserName: aws.String("ghost"), NewUserName: aws.String("ghost2")}, awserrors.ErrorIAMNoSuchEntity},
+		{"taken name", &iam.UpdateUserInput{UserName: aws.String("dave"), NewUserName: aws.String("erin")}, awserrors.ErrorIAMEntityAlreadyExists},
+		{"bad name", &iam.UpdateUserInput{UserName: aws.String("dave"), NewUserName: aws.String("no spaces")}, awserrors.ErrorValidationError},
+		{"bad path", &iam.UpdateUserInput{UserName: aws.String("dave"), NewPath: aws.String("noslash")}, awserrors.ErrorValidationError},
+	}
+	for _, tc := range tests {
+		_, err := svc.UpdateUser(testAccountID, tc.input)
+		require.Error(t, err, tc.name)
+		assert.True(t, awserrors.IsErrorCode(err, tc.want), "%s: got %v want %s", tc.name, err, tc.want)
+	}
+
+	got, err := svc.GetUser(testAccountID, &iam.GetUserInput{UserName: aws.String("dave")})
+	require.NoError(t, err)
+	assert.Equal(t, "/", *got.User.Path, "a refused update leaves the user unchanged")
+}
+
+// racingUsersKV writes the old user record once, right after the renamed
+// record is created, as a concurrent AttachUserPolicy on another node would.
+type racingUsersKV struct {
+	jetstream.KeyValue
+
+	oldKey string
+	raced  bool
+}
+
+func (k *racingUsersKV) Create(ctx context.Context, key string, value []byte, opts ...jetstream.KVCreateOpt) (uint64, error) {
+	rev, err := k.KeyValue.Create(ctx, key, value, opts...)
+	if err != nil || k.raced {
+		return rev, err
+	}
+	k.raced = true
+	entry, gerr := k.Get(ctx, k.oldKey)
+	if gerr != nil {
+		return rev, gerr
+	}
+	var u User
+	if uerr := json.Unmarshal(entry.Value(), &u); uerr != nil {
+		return rev, uerr
+	}
+	u.AttachedPolicies = append(u.AttachedPolicies, "arn:aws:iam::"+testAccountID+":policy/raced")
+	data, _ := json.Marshal(u)
+	if _, perr := k.Update(ctx, k.oldKey, data, entry.Revision()); perr != nil {
+		return rev, perr
+	}
+	return rev, nil
+}
+
+func TestUpdateUser_RenameRetriesAfterRacingWrite(t *testing.T) {
+	t.Parallel()
+	svc := setupTestIAMService(t)
+	createTestUser(t, svc, "frank")
+	key, err := svc.CreateAccessKey(testAccountID, &iam.CreateAccessKeyInput{UserName: aws.String("frank")})
+	require.NoError(t, err)
+	svc.usersBucket = &racingUsersKV{KeyValue: svc.usersBucket, oldKey: testAccountID + ".frank"}
+
+	_, err = svc.UpdateUser(testAccountID, &iam.UpdateUserInput{UserName: aws.String("frank"), NewUserName: aws.String("franklin")})
+	require.NoError(t, err)
+
+	user, err := svc.getUser(t.Context(), testAccountID, "franklin")
+	require.NoError(t, err)
+	assert.Equal(t, []string{"arn:aws:iam::" + testAccountID + ":policy/raced"}, user.AttachedPolicies,
+		"the racing write survives the rename")
+	_, err = svc.getUser(t.Context(), testAccountID, "frank")
+	assert.Equal(t, awserrors.ErrorIAMNoSuchEntity, err.Error())
+	ak, err := svc.LookupAccessKey(*key.AccessKey.AccessKeyId)
+	require.NoError(t, err)
+	assert.Equal(t, "franklin", ak.UserName)
+}

spinifex/domains/lambda/codestore/credentials.go

@@ -0,0 +1,86 @@
+package codestore
+
+import (
+	"errors"
+	"fmt"
+	"log/slog"
+	"sync"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/service/sts"
+	telemetry "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+)
+
+const (
+	// refreshMargin re-assumes this long before a session expires.
+	refreshMargin = 5 * time.Minute
+	// retryInterval spaces re-assume attempts while an old session is still valid.
+	retryInterval = 30 * time.Second
+)
+
+// AssumeFunc mints a code-store session. The composition layer binds it to the
+// in-process STS entry point for one Region.
+type AssumeFunc func() (*sts.AssumeRoleOutput, error)
+
+// sessionProvider re-assumes the role ahead of expiry. A failed re-assume keeps
+// the current session in use until it expires, then fails closed.
+type sessionProvider struct {
+	assume AssumeFunc
+	now    func() time.Time
+
+	mu        sync.Mutex
+	value     credentials.Value
+	expires   time.Time
+	lastError time.Time
+}
+
+var _ credentials.Provider = (*sessionProvider)(nil)
+
+func newSessionProvider(assume AssumeFunc, now func() time.Time) *sessionProvider {
+	return &sessionProvider{assume: assume, now: now}
+}
+
+func (p *sessionProvider) Retrieve() (credentials.Value, error) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	now := p.now()
+	valid := p.value.AccessKeyID != "" && now.Before(p.expires)
+	if valid && (now.Before(p.expires.Add(-refreshMargin)) || now.Before(p.lastError.Add(retryInterval))) {
+		return p.value, nil
+	}
+
+	out, err := p.assume()
+	if err == nil && (out == nil || out.Credentials == nil) {
+		err = errors.New("empty session")
+	}
+	if err != nil {
+		p.lastError = now
+		if valid {
+			slog.Warn("Lambda code store: re-assume failed; keeping current session",
+				"remaining_ms", telemetry.Millis(p.expires.Sub(now)), "err", err)
+			return p.value, nil
+		}
+		return credentials.Value{}, fmt.Errorf("assume code-store role: %w", err)
+	}
+
+	c := out.Credentials
+	p.value = credentials.Value{
+		AccessKeyID:     aws.StringValue(c.AccessKeyId),
+		SecretAccessKey: aws.StringValue(c.SecretAccessKey),
+		SessionToken:    aws.StringValue(c.SessionToken),
+		ProviderName:    "LambdaCodeStoreSession",
+	}
+	p.expires = aws.TimeValue(c.Expiration)
+	p.lastError = time.Time{}
+	return p.value, nil
+}
+
+// IsExpired reports true inside the refresh margin so the SDK calls Retrieve.
+func (p *sessionProvider) IsExpired() bool {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	return p.value.AccessKeyID == "" || !p.now().Before(p.expires.Add(-refreshMargin))
+}

spinifex/domains/lambda/codestore/credentials_test.go

@@ -0,0 +1,94 @@
+package codestore
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/sts"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type fakeAssume struct {
+	calls   int
+	err     error
+	expires time.Time
+}
+
+func (f *fakeAssume) assume() (*sts.AssumeRoleOutput, error) {
+	f.calls++
+	if f.err != nil {
+		return nil, f.err
+	}
+	return &sts.AssumeRoleOutput{Credentials: &sts.Credentials{
+		AccessKeyId:     aws.String("ASIA" + string(rune('A'+f.calls))),
+		SecretAccessKey: aws.String("secret"),
+		SessionToken:    aws.String("token"),
+		Expiration:      aws.Time(f.expires),
+	}}, nil
+}
+
+type clock struct{ t time.Time }
+
+func (c *clock) now() time.Time { return c.t }
+
+func TestSessionProvider_ReusesUntilRefreshMargin(t *testing.T) {
+	c := &clock{t: time.Unix(1_000_000, 0)}
+	a := &fakeAssume{expires: c.t.Add(time.Hour)}
+	p := newSessionProvider(a.assume, c.now)
+
+	first, err := p.Retrieve()
+	require.NoError(t, err)
+	c.t = c.t.Add(54 * time.Minute)
+	assert.False(t, p.IsExpired())
+	again, err := p.Retrieve()
+	require.NoError(t, err)
+	assert.Equal(t, first.AccessKeyID, again.AccessKeyID)
+	assert.Equal(t, 1, a.calls)
+
+	c.t = c.t.Add(2 * time.Minute)
+	a.expires = c.t.Add(time.Hour)
+	assert.True(t, p.IsExpired())
+	fresh, err := p.Retrieve()
+	require.NoError(t, err)
+	assert.NotEqual(t, first.AccessKeyID, fresh.AccessKeyID)
+	assert.Equal(t, 2, a.calls)
+}
+
+// A failed re-assume keeps the still-valid session, spaces retries, and fails
+// closed once the session has expired.
+func TestSessionProvider_KeepsSessionOnFailureUntilExpiry(t *testing.T) {
+	c := &clock{t: time.Unix(1_000_000, 0)}
+	a := &fakeAssume{expires: c.t.Add(time.Hour)}
+	p := newSessionProvider(a.assume, c.now)
+	first, err := p.Retrieve()
+	require.NoError(t, err)
+
+	a.err = errors.New("sts unavailable")
+	c.t = c.t.Add(58 * time.Minute)
+	kept, err := p.Retrieve()
+	require.NoError(t, err)
+	assert.Equal(t, first.AccessKeyID, kept.AccessKeyID)
+	assert.Equal(t, 2, a.calls)
+
+	c.t = c.t.Add(10 * time.Second)
+	_, err = p.Retrieve()
+	require.NoError(t, err)
+	assert.Equal(t, 2, a.calls, "no re-assume inside the retry interval")
+
+	c.t = c.t.Add(2 * time.Minute)
+	_, err = p.Retrieve()
+	require.Error(t, err)
+	assert.Equal(t, 3, a.calls)
+}
+
+func TestSessionProvider_FirstFailureFails(t *testing.T) {
+	a := &fakeAssume{err: errors.New("AccessDenied")}
+	p := newSessionProvider(a.assume, time.Now)
+
+	_, err := p.Retrieve()
+	require.Error(t, err)
+	assert.True(t, p.IsExpired())
+}

spinifex/domains/lambda/codestore/manager.go

@@ -0,0 +1,74 @@
+package codestore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"sync"
+	"time"
+
+	telemetry "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+)
+
+// ErrNotReady means the code store is not provisioned or its role is not
+// assumable yet; callers report ServiceException.
+var ErrNotReady = errors.New("lambda code store not ready")
+
+const (
+	initialBackoff = time.Second
+	maxBackoff     = 5 * time.Minute
+)
+
+// Manager provisions and opens the store on first use, so a process that never
+// uses Lambda creates nothing. After a failure, calls inside the capped backoff
+// window fail fast with the last reason instead of retrying.
+type Manager struct {
+	provision func(context.Context) error
+	open      func() (*Store, error)
+	now       func() time.Time
+
+	mu      sync.Mutex
+	store   *Store
+	reason  error
+	backoff time.Duration
+	retryAt time.Time
+}
+
+// NewManager binds the bounded provisioning step and the scoped opener. It does
+// no I/O.
+func NewManager(provision func(context.Context) error, open func() (*Store, error)) *Manager {
+	return &Manager{provision: provision, open: open, now: time.Now}
+}
+
+// Store returns the opened store, initialising it on first use. An ErrNotOwned
+// refusal is retried after backoff, so an operator's fix takes effect without a
+// restart.
+func (m *Manager) Store(ctx context.Context) (*Store, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.store != nil {
+		return m.store, nil
+	}
+	if now := m.now(); now.Before(m.retryAt) {
+		return nil, fmt.Errorf("%w: %w", ErrNotReady, m.reason)
+	}
+
+	store, err := m.attempt(ctx)
+	if err == nil {
+		m.store, m.reason, m.backoff = store, nil, 0
+		slog.InfoContext(ctx, "Lambda code store ready")
+		return store, nil
+	}
+	m.backoff = min(max(m.backoff*2, initialBackoff), maxBackoff)
+	m.reason, m.retryAt = err, m.now().Add(m.backoff)
+	slog.WarnContext(ctx, "Lambda code store not ready", "err", err, "retry_ms", telemetry.Millis(m.backoff))
+	return nil, fmt.Errorf("%w: %w", ErrNotReady, err)
+}
+
+func (m *Manager) attempt(ctx context.Context) (*Store, error) {
+	if err := m.provision(ctx); err != nil {
+		return nil, err
+	}
+	return m.open()
+}

spinifex/domains/lambda/codestore/provision.go

@@ -0,0 +1,178 @@
+// Package codestore keeps Lambda code copies in the Region's system-account
+// bucket, reached only through a short-lived session of the Region's scoped
+// code-store role. The composition layer owns every privileged client.
+package codestore
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"reflect"
+)
+
+const (
+	// OwnerTagKey marks the role and bucket Lambda intends to own. It names the
+	// intended owner; any principal able to tag could also set it.
+	OwnerTagKey = "spinifex:owner"
+	// OwnerTagValue is the owner tag's value.
+	OwnerTagValue = "lambda-code-store"
+	// PolicyName is the role's only inline policy.
+	PolicyName = "lambda-code-store"
+)
+
+// TrustPolicy lets only the in-process lambda.amazonaws.com path assume the role.
+const TrustPolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"lambda.amazonaws.com"},"Action":"sts:AssumeRole"}]}`
+
+var (
+	// ErrExists is returned by an admin Create when the resource already exists
+	// under the system account.
+	ErrExists = errors.New("already exists")
+	// ErrNotFound is returned by RoleAdmin.Describe for an absent role.
+	ErrNotFound = errors.New("not found")
+	// ErrOwnedElsewhere is returned by BucketAdmin.Create when another account
+	// owns the bucket name.
+	ErrOwnedElsewhere = errors.New("owned by another account")
+	// ErrNotOwned is a refusal that persists until an operator resolves it.
+	ErrNotOwned = errors.New("lambda code store: existing resource is not Lambda's")
+)
+
+// BucketName is the Region's code bucket in the system account.
+func BucketName(region string) string {
+	return "lambda-code-" + region
+}
+
+type policyStatement struct {
+	Effect   string   `json:"Effect"`
+	Action   []string `json:"Action"`
+	Resource string   `json:"Resource"`
+}
+
+type policyDocument struct {
+	Version   string            `json:"Version"`
+	Statement []policyStatement `json:"Statement"`
+}
+
+// RolePolicy grants object access to the Region's code bucket and nothing else.
+func RolePolicy(region string) (string, error) {
+	b := "arn:aws:s3:::" + BucketName(region)
+	doc, err := json.Marshal(policyDocument{
+		Version: "2012-10-17",
+		Statement: []policyStatement{
+			{Effect: "Allow", Action: []string{"s3:PutObject", "s3:GetObject", "s3:DeleteObject"}, Resource: b + "/*"},
+			{Effect: "Allow", Action: []string{"s3:ListBucket"}, Resource: b},
+		},
+	})
+	if err != nil {
+		return "", fmt.Errorf("lambda code store policy: %w", err)
+	}
+	return string(doc), nil
+}
+
+// RoleState is what provisioning reads back about an existing role.
+type RoleState struct {
+	TrustPolicy      string
+	Tags             map[string]string
+	InlinePolicies   []string
+	AttachedPolicies int
+}
+
+// RoleAdmin is bound by the composition layer to the one code-store role.
+type RoleAdmin interface {
+	Create(trustPolicy string, tags map[string]string) error
+	Describe() (*RoleState, error)
+	PutPolicy(name, document string) error
+}
+
+// BucketAdmin is bound by the composition layer to the one code bucket.
+// Create tags the bucket only when it created it, never an existing one.
+type BucketAdmin interface {
+	Create(ctx context.Context, tags map[string]string) error
+	Tags(ctx context.Context) (map[string]string, error)
+}
+
+func ownerTags() map[string]string {
+	return map[string]string{OwnerTagKey: OwnerTagValue}
+}
+
+// Provision creates or verifies the Region's role, policy and bucket. A retry
+// completes Lambda's own partial run; any state it cannot attribute to Lambda
+// is refused with ErrNotOwned and left untouched.
+func Provision(ctx context.Context, region string, role RoleAdmin, bucket BucketAdmin) error {
+	if err := ensureRole(role); err != nil {
+		return err
+	}
+	policy, err := RolePolicy(region)
+	if err != nil {
+		return err
+	}
+	if err := role.PutPolicy(PolicyName, policy); err != nil {
+		return fmt.Errorf("put code-store role policy: %w", err)
+	}
+	return ensureBucket(ctx, region, bucket)
+}
+
+func ensureRole(role RoleAdmin) error {
+	err := role.Create(TrustPolicy, ownerTags())
+	if err == nil {
+		return nil
+	}
+	if !errors.Is(err, ErrExists) {
+		return fmt.Errorf("create code-store role: %w", err)
+	}
+	st, err := role.Describe()
+	if err != nil {
+		return fmt.Errorf("describe code-store role: %w", err)
+	}
+	return checkRole(st)
+}
+
+// checkRole accepts only Lambda's own shape: owner tag, exact trust, no attached
+// policies, and no inline policy but Lambda's. No inline policy at all is the
+// state a crash between CreateRole and PutRolePolicy leaves, so it is retried.
+func checkRole(st *RoleState) error {
+	switch {
+	case st.Tags[OwnerTagKey] != OwnerTagValue:
+		return fmt.Errorf("%w: role has no %s=%s tag", ErrNotOwned, OwnerTagKey, OwnerTagValue)
+	case !sameJSON(st.TrustPolicy, TrustPolicy):
+		return fmt.Errorf("%w: role trust policy differs from the code-store trust policy", ErrNotOwned)
+	case st.AttachedPolicies > 0:
+		return fmt.Errorf("%w: role has %d attached managed policies", ErrNotOwned, st.AttachedPolicies)
+	}
+	for _, name := range st.InlinePolicies {
+		if name != PolicyName {
+			return fmt.Errorf("%w: role has inline policy %q", ErrNotOwned, name)
+		}
+	}
+	return nil
+}
+
+func ensureBucket(ctx context.Context, region string, bucket BucketAdmin) error {
+	err := bucket.Create(ctx, ownerTags())
+	if err == nil {
+		return nil
+	}
+	if errors.Is(err, ErrOwnedElsewhere) {
+		return fmt.Errorf("%w: bucket %s is owned by another account", ErrNotOwned, BucketName(region))
+	}
+	if !errors.Is(err, ErrExists) {
+		return fmt.Errorf("create code bucket: %w", err)
+	}
+	tags, err := bucket.Tags(ctx)
+	if err != nil {
+		return fmt.Errorf("read code bucket tags: %w", err)
+	}
+	if tags[OwnerTagKey] != OwnerTagValue {
+		return fmt.Errorf("%w: bucket %s exists without the %s=%s tag; an operator must tag or delete it",
+			ErrNotOwned, BucketName(region), OwnerTagKey, OwnerTagValue)
+	}
+	return nil
+}
+
+func sameJSON(a, b string) bool {
+	var x, y any
+	if json.Unmarshal([]byte(a), &x) != nil || json.Unmarshal([]byte(b), &y) != nil {
+		return false
+	}
+	return reflect.DeepEqual(x, y)
+}

spinifex/domains/lambda/codestore/provision_test.go

@@ -0,0 +1,165 @@
+package codestore
+
+import (
+	"context"
+	"errors"
+	"slices"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const testRegion = "ap-southeast-2"
+
+type fakeRole struct {
+	state    *RoleState
+	created  int
+	policies map[string]string
+}
+
+func (f *fakeRole) Create(trust string, tags map[string]string) error {
+	if f.state != nil {
+		return ErrExists
+	}
+	f.created++
+	f.state = &RoleState{TrustPolicy: trust, Tags: tags}
+	return nil
+}
+
+func (f *fakeRole) Describe() (*RoleState, error) {
+	if f.state == nil {
+		return nil, ErrNotFound
+	}
+	return f.state, nil
+}
+
+func (f *fakeRole) PutPolicy(name, doc string) error {
+	if f.policies == nil {
+		f.policies = map[string]string{}
+	}
+	f.policies[name] = doc
+	if !slices.Contains(f.state.InlinePolicies, name) {
+		f.state.InlinePolicies = append(f.state.InlinePolicies, name)
+	}
+	return nil
+}
+
+type fakeBucket struct {
+	exists, foreign bool
+	tags            map[string]string
+	tagErr          error
+	tagWrites       int
+}
+
+func (f *fakeBucket) Create(_ context.Context, tags map[string]string) error {
+	switch {
+	case f.foreign:
+		return ErrOwnedElsewhere
+	case f.exists:
+		return ErrExists
+	}
+	f.exists = true
+	if f.tagErr != nil {
+		return f.tagErr
+	}
+	f.tagWrites++
+	f.tags = tags
+	return nil
+}
+
+func (f *fakeBucket) Tags(context.Context) (map[string]string, error) { return f.tags, nil }
+
+func mustPolicy(t *testing.T) string {
+	t.Helper()
+	p, err := RolePolicy(testRegion)
+	require.NoError(t, err)
+	return p
+}
+
+func ownRole(inline ...string) *fakeRole {
+	return &fakeRole{state: &RoleState{TrustPolicy: TrustPolicy, Tags: ownerTags(), InlinePolicies: inline}}
+}
+
+func TestProvision_FreshCreatesTaggedRolePolicyAndBucket(t *testing.T) {
+	role, bucket := &fakeRole{}, &fakeBucket{}
+
+	require.NoError(t, Provision(t.Context(), testRegion, role, bucket))
+
+	assert.Equal(t, 1, role.created)
+	assert.Equal(t, OwnerTagValue, role.state.Tags[OwnerTagKey])
+	assert.JSONEq(t, mustPolicy(t), role.policies[PolicyName])
+	assert.Equal(t, OwnerTagValue, bucket.tags[OwnerTagKey])
+}
+
+// A crash between CreateRole and PutRolePolicy leaves Lambda's tagged role with
+// no inline policy; the next run adds it.
+func TestProvision_RetriesOwnRoleWithoutPolicy(t *testing.T) {
+	role, bucket := ownRole(), &fakeBucket{}
+
+	require.NoError(t, Provision(t.Context(), testRegion, role, bucket))
+
+	assert.Zero(t, role.created)
+	assert.JSONEq(t, mustPolicy(t), role.policies[PolicyName])
+}
+
+func TestProvision_RestoresDriftedOwnPolicy(t *testing.T) {
+	role, bucket := ownRole(PolicyName), &fakeBucket{}
+	role.policies = map[string]string{PolicyName: `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:*","Resource":"*"}]}`}
+
+	require.NoError(t, Provision(t.Context(), testRegion, role, bucket))
+	assert.JSONEq(t, mustPolicy(t), role.policies[PolicyName])
+}
+
+func TestProvision_RefusesRoleNotLambdas(t *testing.T) {
+	cases := map[string]*RoleState{
+		"no owner tag":          {TrustPolicy: TrustPolicy},
+		"other trust":           {TrustPolicy: `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":"arn:aws:iam::000000000000:root"},"Action":"sts:AssumeRole"}]}`, Tags: ownerTags()},
+		"attached policy":       {TrustPolicy: TrustPolicy, Tags: ownerTags(), AttachedPolicies: 1},
+		"foreign inline policy": {TrustPolicy: TrustPolicy, Tags: ownerTags(), InlinePolicies: []string{PolicyName, "admin"}},
+	}
+	for name, st := range cases {
+		t.Run(name, func(t *testing.T) {
+			role, bucket := &fakeRole{state: st}, &fakeBucket{}
+
+			err := Provision(t.Context(), testRegion, role, bucket)
+
+			require.ErrorIs(t, err, ErrNotOwned)
+			assert.Nil(t, role.policies, "no policy written to a role that is not Lambda's")
+			assert.False(t, bucket.exists, "provisioning stops before the bucket")
+		})
+	}
+}
+
+func TestProvision_AcceptsOwnTaggedBucket(t *testing.T) {
+	bucket := &fakeBucket{exists: true, tags: ownerTags()}
+
+	require.NoError(t, Provision(t.Context(), testRegion, &fakeRole{}, bucket))
+	assert.Zero(t, bucket.tagWrites)
+}
+
+// A crash after CreateBucket and before its tags leaves an untagged bucket that
+// cannot be told apart from another system component's, so it is refused.
+func TestProvision_RefusesUntaggedBucketAfterPartialCreate(t *testing.T) {
+	bucket := &fakeBucket{tagErr: errors.New("connection reset")}
+	require.Error(t, Provision(t.Context(), testRegion, &fakeRole{}, bucket))
+
+	bucket.tagErr = nil
+	err := Provision(t.Context(), testRegion, &fakeRole{}, bucket)
+
+	require.ErrorIs(t, err, ErrNotOwned)
+	assert.Contains(t, err.Error(), "lambda-code-ap-southeast-2 exists without the spinifex:owner=lambda-code-store tag")
+	assert.Zero(t, bucket.tagWrites, "an existing bucket is never tagged into use")
+}
+
+func TestProvision_RefusesBucketOwnedElsewhere(t *testing.T) {
+	err := Provision(t.Context(), testRegion, &fakeRole{}, &fakeBucket{foreign: true})
+	require.ErrorIs(t, err, ErrNotOwned)
+}
+
+func TestRolePolicy_ScopesToRegionalBucket(t *testing.T) {
+	assert.JSONEq(t, `{"Version":"2012-10-17","Statement":[
+		{"Effect":"Allow","Action":["s3:PutObject","s3:GetObject","s3:DeleteObject"],"Resource":"arn:aws:s3:::lambda-code-ap-southeast-2/*"},
+		{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":"arn:aws:s3:::lambda-code-ap-southeast-2"}]}`,
+		mustPolicy(t))
+}

spinifex/domains/lambda/codestore/store.go

@@ -0,0 +1,210 @@
+package codestore
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"regexp"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+)
+
+// MaxCodeBytes bounds one stored code copy, AWS's direct-upload ZIP limit.
+const MaxCodeBytes = 50 << 20
+
+var (
+	// ErrCodeNotFound means no copy exists for the reference.
+	ErrCodeNotFound = errors.New("lambda code copy not found")
+	// ErrDigestMismatch means the stored bytes do not match the recorded SHA-256.
+	ErrDigestMismatch = errors.New("lambda code copy digest mismatch")
+	// ErrCodeTooLarge means the copy exceeds MaxCodeBytes.
+	ErrCodeTooLarge = errors.New("lambda code copy too large")
+	// ErrInvalidRef means a reference segment is empty or unsafe as a key segment.
+	ErrInvalidRef = errors.New("invalid lambda code reference")
+)
+
+var (
+	refSegment = regexp.MustCompile(`^[A-Za-z0-9_-][A-Za-z0-9._-]{0,127}$`)
+	digestHex  = regexp.MustCompile(`^[0-9a-f]{64}$`)
+)
+
+// Ref names one code revision; keys group by account for S11 inventory.
+type Ref struct {
+	AccountID  string
+	FunctionID string
+	RevisionID string
+}
+
+func (r Ref) prefix() (string, error) {
+	for _, seg := range []string{r.AccountID, r.FunctionID, r.RevisionID} {
+		if !refSegment.MatchString(seg) {
+			return "", fmt.Errorf("%w: %q", ErrInvalidRef, seg)
+		}
+	}
+	return r.AccountID + "/" + r.FunctionID + "/" + r.RevisionID + "/", nil
+}
+
+// key names a copy by its digest, so different bytes for the same revision land
+// on a different object and can never replace the recorded copy.
+func (r Ref) key(sha256Hex string) (string, error) {
+	p, err := r.prefix()
+	if err != nil {
+		return "", err
+	}
+	if !digestHex.MatchString(sha256Hex) {
+		return "", fmt.Errorf("%w: digest %q", ErrInvalidRef, sha256Hex)
+	}
+	return p + sha256Hex + ".zip", nil
+}
+
+// Store reads and writes code copies in one Region's code bucket.
+type Store struct {
+	bucket  string
+	objects objectstore.ObjectStore
+}
+
+// OpenConfig is what the composition layer supplies to open the scoped store.
+type OpenConfig struct {
+	Region        string // Spinifex Region naming the bucket
+	Endpoint      string // Predastore URL
+	SigningRegion string // Predastore SigV4 Region
+	Assume        AssumeFunc
+	HTTPClient    *http.Client
+	now           func() time.Time
+}
+
+// Open builds the store on a refreshing code-store session and draws the first
+// session, so an unassumable role fails here rather than on first use.
+func Open(cfg OpenConfig) (*Store, error) {
+	if cfg.Region == "" || cfg.Endpoint == "" || cfg.Assume == nil {
+		return nil, errors.New("lambda code store: region, endpoint and assume are required")
+	}
+	now := cfg.now
+	if now == nil {
+		now = time.Now
+	}
+	client := cfg.HTTPClient
+	if client == nil {
+		client = &http.Client{Timeout: 120 * time.Second}
+	}
+	creds := credentials.NewCredentials(newSessionProvider(cfg.Assume, now))
+	if _, err := creds.Get(); err != nil {
+		return nil, err
+	}
+	sess, err := session.NewSession(&aws.Config{
+		Endpoint:         aws.String(cfg.Endpoint),
+		Region:           aws.String(cfg.SigningRegion),
+		Credentials:      creds,
+		S3ForcePathStyle: aws.Bool(true),
+		HTTPClient:       client,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("lambda code store session: %w", err)
+	}
+	return newStore(cfg.Region, objectstore.NewS3ObjectStore(s3.New(sess))), nil
+}
+
+func newStore(region string, objects objectstore.ObjectStore) *Store {
+	return &Store{bucket: BucketName(region), objects: objects}
+}
+
+// Put stores code under its digest and returns the hex SHA-256 the revision
+// record keeps. A retry with the same bytes rewrites identical content.
+func (s *Store) Put(ctx context.Context, ref Ref, code []byte) (string, error) {
+	if len(code) > MaxCodeBytes {
+		return "", ErrCodeTooLarge
+	}
+	sum := sha256.Sum256(code)
+	digest := hex.EncodeToString(sum[:])
+	key, err := ref.key(digest)
+	if err != nil {
+		return "", err
+	}
+	if _, err := s.objects.PutObject(ctx, &s3.PutObjectInput{
+		Bucket:        aws.String(s.bucket),
+		Key:           aws.String(key),
+		Body:          bytes.NewReader(code),
+		ContentLength: aws.Int64(int64(len(code))),
+	}); err != nil {
+		return "", fmt.Errorf("put lambda code copy: %w", err)
+	}
+	return digest, nil
+}
+
+// Get returns the copy only if it matches wantSHA256, the recorded digest.
+func (s *Store) Get(ctx context.Context, ref Ref, wantSHA256 string) ([]byte, error) {
+	key, err := ref.key(wantSHA256)
+	if err != nil {
+		return nil, err
+	}
+	out, err := s.objects.GetObject(ctx, &s3.GetObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)})
+	if err != nil {
+		if objectstore.IsNoSuchKeyError(err) {
+			return nil, ErrCodeNotFound
+		}
+		return nil, fmt.Errorf("get lambda code copy: %w", err)
+	}
+	defer out.Body.Close()
+	code, err := io.ReadAll(io.LimitReader(out.Body, MaxCodeBytes+1))
+	if err != nil {
+		return nil, fmt.Errorf("read lambda code copy: %w", err)
+	}
+	if len(code) > MaxCodeBytes {
+		return nil, ErrCodeTooLarge
+	}
+	sum := sha256.Sum256(code)
+	if hex.EncodeToString(sum[:]) != wantSHA256 {
+		return nil, ErrDigestMismatch
+	}
+	return code, nil
+}
+
+// Delete removes the copy with the recorded digest; an absent copy is already
+// deleted.
+func (s *Store) Delete(ctx context.Context, ref Ref, sha256Hex string) error {
+	key, err := ref.key(sha256Hex)
+	if err != nil {
+		return err
+	}
+	return s.deleteKey(ctx, key)
+}
+
+// DeleteRevision removes every copy under the revision, including ones a failed
+// or superseded attempt left. Callers own the guarantee that no create or update
+// for the revision is still in progress.
+func (s *Store) DeleteRevision(ctx context.Context, ref Ref) error {
+	prefix, err := ref.prefix()
+	if err != nil {
+		return err
+	}
+	objects, _, err := objectstore.ListAll(ctx, s.objects, &s3.ListObjectsV2Input{
+		Bucket: aws.String(s.bucket), Prefix: aws.String(prefix),
+	})
+	if err != nil {
+		return fmt.Errorf("list lambda code copies: %w", err)
+	}
+	for _, o := range objects {
+		if err := s.deleteKey(ctx, aws.StringValue(o.Key)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *Store) deleteKey(ctx context.Context, key string) error {
+	_, err := s.objects.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(s.bucket), Key: aws.String(key)})
+	if err != nil && !objectstore.IsNoSuchKeyError(err) {
+		return fmt.Errorf("delete lambda code copy: %w", err)
+	}
+	return nil
+}

spinifex/domains/lambda/codestore/store_test.go

@@ -0,0 +1,169 @@
+package codestore
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/sts"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var testRef = Ref{AccountID: "000000000001", FunctionID: "fn-0123", RevisionID: "rev-1"}
+
+func copyKey(sum string) string { return "000000000001/fn-0123/rev-1/" + sum + ".zip" }
+
+func TestStore_PutGetDelete(t *testing.T) {
+	mem := objectstore.NewMemoryObjectStore()
+	s := newStore(testRegion, mem)
+	code := []byte("PK\x03\x04 zip bytes")
+
+	sum, err := s.Put(t.Context(), testRef, code)
+	require.NoError(t, err)
+	want := sha256.Sum256(code)
+	assert.Equal(t, hex.EncodeToString(want[:]), sum)
+
+	_, err = mem.HeadObject(t.Context(), &s3.HeadObjectInput{
+		Bucket: aws.String("lambda-code-ap-southeast-2"), Key: aws.String(copyKey(sum))})
+	require.NoError(t, err, "copy is keyed by account, function, revision and digest")
+
+	got, err := s.Get(t.Context(), testRef, sum)
+	require.NoError(t, err)
+	assert.Equal(t, code, got)
+
+	require.NoError(t, s.Delete(t.Context(), testRef, sum))
+	_, err = s.Get(t.Context(), testRef, sum)
+	require.ErrorIs(t, err, ErrCodeNotFound)
+	require.NoError(t, s.Delete(t.Context(), testRef, sum), "deleting an absent copy succeeds")
+}
+
+// A retry or a racing write with different bytes for the same revision lands on
+// its own digest key, so the recorded copy is never replaced.
+func TestStore_DifferentBytesNeverReplaceRecordedCopy(t *testing.T) {
+	s := newStore(testRegion, objectstore.NewMemoryObjectStore())
+	recorded, err := s.Put(t.Context(), testRef, []byte("first"))
+	require.NoError(t, err)
+
+	var wg sync.WaitGroup
+	others := make([]string, 8)
+	for i := range others {
+		wg.Go(func() {
+			sum, err := s.Put(t.Context(), testRef, []byte(fmt.Sprintf("other-%d", i)))
+			assert.NoError(t, err)
+			others[i] = sum
+		})
+	}
+	wg.Wait()
+
+	got, err := s.Get(t.Context(), testRef, recorded)
+	require.NoError(t, err)
+	assert.Equal(t, []byte("first"), got)
+	for i, sum := range others {
+		got, err := s.Get(t.Context(), testRef, sum)
+		require.NoError(t, err)
+		assert.Equal(t, []byte(fmt.Sprintf("other-%d", i)), got)
+	}
+}
+
+// DeleteRevision clears every copy of one revision, including a failed
+// attempt's, and leaves other revisions alone.
+func TestStore_DeleteRevisionRemovesAllCopiesOfThatRevisionOnly(t *testing.T) {
+	s := newStore(testRegion, objectstore.NewMemoryObjectStore())
+	a, err := s.Put(t.Context(), testRef, []byte("kept attempt"))
+	require.NoError(t, err)
+	b, err := s.Put(t.Context(), testRef, []byte("failed attempt"))
+	require.NoError(t, err)
+	sibling := Ref{AccountID: testRef.AccountID, FunctionID: testRef.FunctionID, RevisionID: "rev-10"}
+	c, err := s.Put(t.Context(), sibling, []byte("next revision"))
+	require.NoError(t, err)
+
+	require.NoError(t, s.DeleteRevision(t.Context(), testRef))
+
+	for _, sum := range []string{a, b} {
+		_, err := s.Get(t.Context(), testRef, sum)
+		require.ErrorIs(t, err, ErrCodeNotFound)
+	}
+	_, err = s.Get(t.Context(), sibling, c)
+	require.NoError(t, err, "rev-10 shares the rev-1 string prefix but not its key prefix")
+}
+
+func TestStore_GetRefusesDigestMismatch(t *testing.T) {
+	mem := objectstore.NewMemoryObjectStore()
+	s := newStore(testRegion, mem)
+	sum, err := s.Put(t.Context(), testRef, []byte("original"))
+	require.NoError(t, err)
+	_, err = mem.PutObject(t.Context(), &s3.PutObjectInput{
+		Bucket: aws.String(BucketName(testRegion)), Key: aws.String(copyKey(sum)),
+		Body: bytes.NewReader([]byte("replaced"))})
+	require.NoError(t, err)
+
+	_, err = s.Get(t.Context(), testRef, sum)
+	require.ErrorIs(t, err, ErrDigestMismatch)
+}
+
+func TestStore_RejectsUnsafeRefAndOversize(t *testing.T) {
+	s := newStore(testRegion, objectstore.NewMemoryObjectStore())
+	for _, ref := range []Ref{
+		{AccountID: "", FunctionID: "f", RevisionID: "r"},
+		{AccountID: "a", FunctionID: "..", RevisionID: "r"},
+		{AccountID: "a", FunctionID: "f/../../other", RevisionID: "r"},
+	} {
+		_, err := s.Put(t.Context(), ref, []byte("x"))
+		require.ErrorIs(t, err, ErrInvalidRef, "%+v", ref)
+	}
+	_, err := s.Put(t.Context(), testRef, make([]byte, MaxCodeBytes+1))
+	require.ErrorIs(t, err, ErrCodeTooLarge)
+	_, err = s.Get(t.Context(), testRef, "../../other")
+	require.ErrorIs(t, err, ErrInvalidRef, "a digest is never used as a raw key segment")
+}
+
+func TestOpen_FailsWhenRoleNotAssumable(t *testing.T) {
+	_, err := Open(OpenConfig{Region: testRegion, Endpoint: "https://127.0.0.1:1", SigningRegion: testRegion,
+		Assume: func() (*sts.AssumeRoleOutput, error) { return nil, errors.New("AccessDenied") }})
+	require.Error(t, err)
+}
+
+func TestManager_InitialisesOnFirstUseWithBackoff(t *testing.T) {
+	attempts := 0
+	m := NewManager(func(context.Context) error {
+		attempts++
+		if attempts < 3 {
+			return ErrNotOwned
+		}
+		return nil
+	}, func() (*Store, error) { return newStore(testRegion, objectstore.NewMemoryObjectStore()), nil })
+	now := time.Unix(1_000_000, 0)
+	m.now = func() time.Time { return now }
+	assert.Zero(t, attempts, "construction does no provisioning")
+
+	_, err := m.Store(t.Context())
+	require.ErrorIs(t, err, ErrNotReady)
+	require.ErrorIs(t, err, ErrNotOwned)
+	_, err = m.Store(t.Context())
+	require.ErrorIs(t, err, ErrNotReady)
+	assert.Equal(t, 1, attempts, "a call inside the backoff window does not retry")
+
+	now = now.Add(time.Second)
+	_, err = m.Store(t.Context())
+	require.ErrorIs(t, err, ErrNotReady)
+	now = now.Add(time.Second)
+	_, err = m.Store(t.Context())
+	require.ErrorIs(t, err, ErrNotReady, "backoff doubled to 2s")
+	now = now.Add(time.Second)
+	first, err := m.Store(t.Context())
+	require.NoError(t, err)
+	again, err := m.Store(t.Context())
+	require.NoError(t, err)
+	assert.Same(t, first, again)
+	assert.Equal(t, 3, attempts)
+}

spinifex/domains/sts/code_store_role.go

@@ -0,0 +1,50 @@
+package sts
+
+import (
+	"context"
+	"errors"
+	"log/slog"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/sts"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+)
+
+// lambdaServicePrincipal is the principal the Lambda code-store role trusts.
+const lambdaServicePrincipal = "lambda.amazonaws.com"
+
+// CodeStoreRoleName names the Region's Lambda code-store role in the system account.
+func CodeStoreRoleName(region string) string {
+	return "lambda-code-store-" + region
+}
+
+// CodeStoreRoleARN is the only role AssumeCodeStoreRole will issue.
+func CodeStoreRoleARN(region string) string {
+	return "arn:aws:iam::" + identifiers.GlobalAccountID + ":role/" + CodeStoreRoleName(region)
+}
+
+// AssumeCodeStoreRole mints a session for the Region's Lambda code-store role.
+// In-process only: it is not on STSService, NATS or HTTPS, and takes no role
+// ARN, so a caller can never use it to issue any other role.
+func (s *STSServiceImpl) AssumeCodeStoreRole(region, sessionName string) (*sts.AssumeRoleOutput, error) {
+	if region == "" || sessionName == "" {
+		return nil, errors.New(awserrors.ErrorMissingParameter)
+	}
+	if !roleSessionNameRegex.MatchString(sessionName) || !roleSessionNameRegex.MatchString(region) {
+		return nil, errors.New(awserrors.ErrorValidationError)
+	}
+
+	roleARN := CodeStoreRoleARN(region)
+	out, err := s.assumeRoleForCaller(context.Background(), "", lambdaServicePrincipal, roleARN, sessionName, "", 0)
+	if err != nil {
+		return nil, err
+	}
+
+	slog.Info("AssumeCodeStoreRole success",
+		"role_arn", roleARN,
+		"session_name", sessionName,
+		"expires_at", aws.TimeValue(out.Credentials.Expiration),
+	)
+	return out, nil
+}

spinifex/domains/sts/code_store_role_test.go

@@ -0,0 +1,77 @@
+package sts
+
+import (
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	testCodeStoreRegion  = "ap-southeast-2"
+	lambdaTrustPolicyDoc = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"lambda.amazonaws.com"},"Action":"sts:AssumeRole"}]}`
+)
+
+func newCodeStoreSetup(t *testing.T, trust string) *STSServiceImpl {
+	t.Helper()
+	svc, nc := newTestSetup(t)
+	putTestAccount(t, nc, identifiers.GlobalAccountID, iamdomain.AccountStatusActive)
+	createRoleInAccount(t, svc, identifiers.GlobalAccountID, CodeStoreRoleName(testCodeStoreRegion), trust)
+	return svc
+}
+
+func TestAssumeCodeStoreRole_IssuesRegionalSystemRole(t *testing.T) {
+	svc := newCodeStoreSetup(t, lambdaTrustPolicyDoc)
+
+	out, err := svc.AssumeCodeStoreRole(testCodeStoreRegion, "node-1")
+	require.NoError(t, err)
+
+	assert.Equal(t, "arn:aws:sts::000000000000:assumed-role/lambda-code-store-ap-southeast-2/node-1",
+		aws.StringValue(out.AssumedRoleUser.Arn))
+	stored, err := svc.LookupSessionCredential(aws.StringValue(out.Credentials.AccessKeyId))
+	require.NoError(t, err)
+	assert.Equal(t, identifiers.GlobalAccountID, stored.AccountID)
+}
+
+// The entry point asserts only lambda.amazonaws.com, so a role whose trust names
+// another principal is refused even though its name is the code-store name.
+func TestAssumeCodeStoreRole_RefusesOtherTrust(t *testing.T) {
+	svc := newCodeStoreSetup(t, trustPolicyAllowingEC2Service())
+
+	_, err := svc.AssumeCodeStoreRole(testCodeStoreRegion, "node-1")
+	assertErrorCode(t, err, awserrors.ErrorAccessDenied)
+}
+
+// No Region has a role yet, so another Region's name resolves to nothing.
+func TestAssumeCodeStoreRole_OtherRegionHasNoRole(t *testing.T) {
+	svc := newCodeStoreSetup(t, lambdaTrustPolicyDoc)
+
+	_, err := svc.AssumeCodeStoreRole("us-east-1", "node-1")
+	assertErrorCode(t, err, awserrors.ErrorAccessDenied)
+}
+
+// An HTTPS caller is never attributed lambda.amazonaws.com, so even a system
+// account principal cannot assume the code-store role over AssumeRole.
+func TestAssumeCodeStoreRole_HTTPSAssumeRoleCannotMatch(t *testing.T) {
+	svc := newCodeStoreSetup(t, lambdaTrustPolicyDoc)
+
+	caller := "arn:aws:iam::000000000000:user/operator"
+	_, err := svc.AssumeRole(identifiers.GlobalAccountID, caller, "AIDAOPERATOR",
+		basicAssumeRoleInput(CodeStoreRoleARN(testCodeStoreRegion), "s1"))
+	assertErrorCode(t, err, awserrors.ErrorAccessDenied)
+}
+
+func TestAssumeCodeStoreRole_ValidatesInputs(t *testing.T) {
+	svc := newCodeStoreSetup(t, lambdaTrustPolicyDoc)
+
+	_, err := svc.AssumeCodeStoreRole("", "node-1")
+	assertErrorCode(t, err, awserrors.ErrorMissingParameter)
+	_, err = svc.AssumeCodeStoreRole("ap/southeast", "node-1")
+	assertErrorCode(t, err, awserrors.ErrorValidationError)
+	_, err = svc.AssumeCodeStoreRole(testCodeStoreRegion, "bad name")
+	assertErrorCode(t, err, awserrors.ErrorValidationError)
+}

spinifex/ingress/aws/pipeline/gwhelpers_test.go

@@ -781,6 +781,10 @@ func (svc *echoIAMService) UpdateAccessKey(accountID string, input *iam.UpdateAc
 	return echoIAMCall[iam.UpdateAccessKeyInput, iam.UpdateAccessKeyOutput](svc, accountID, input)
 }
 
+func (svc *echoIAMService) UpdateUser(accountID string, input *iam.UpdateUserInput) (*iam.UpdateUserOutput, error) {
+	return echoIAMCall[iam.UpdateUserInput, iam.UpdateUserOutput](svc, accountID, input)
+}
+
 func (svc *echoIAMService) UpdateAssumeRolePolicy(accountID string, input *iam.UpdateAssumeRolePolicyInput) (*iam.UpdateAssumeRolePolicyOutput, error) {
 	return echoIAMCall[iam.UpdateAssumeRolePolicyInput, iam.UpdateAssumeRolePolicyOutput](svc, accountID, input)
 }

spinifex/operator/admin/account_delete.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	operatorv1 "github.com/mulgadc/spinifex/contracts/operator/v1"
 	"log/slog"
+	"os"
 	"regexp"
 	"strings"
 	"sync"
@@ -74,8 +75,31 @@ type accountDeletionJob struct {
 	// Error is the failure reason, including the stuck-resource summary. The
 	// per-stage Stuck entries carry which resources and why.
 	Error string `json:"error,omitempty"`
+
+	// OperationID names the whole deletion and survives every takeover.
+	// DeletionID is kept readable and equals it for records written now.
+	OperationID string `json:"operationId,omitempty"`
+	// Fence is 1 on acceptance and grows by one per takeover. A writer whose
+	// fence is no longer the stored one has been superseded and must stop.
+	Fence      uint64             `json:"fence,omitempty"`
+	Holder     string             `json:"holder,omitempty"`
+	LeaseUntil string             `json:"leaseUntil,omitempty"`
+	Takeovers  []deletionTakeover `json:"takeovers,omitempty"`
 }
 
+// deletionTakeover records one change of holder.
+type deletionTakeover struct {
+	Fence  uint64 `json:"fence"`
+	Holder string `json:"holder"`
+	At     string `json:"at"`
+	Reason string `json:"reason"`
+}
+
+const (
+	takeoverReasonLeaseExpired = "lease_expired"
+	takeoverReasonFailed       = "failed"
+)
+
 // deleteAccount starts a teardown, or answers a dry run inline.
 func (d Deps) deleteAccount(ctx context.Context, body []byte) (any, error) {
 	var req operatorv1.DeleteAccountRequest
@@ -133,7 +157,7 @@ func (d Deps) deleteAccount(ctx context.Context, body []byte) (any, error) {
 		return nil, teardownError("DeleteAccount", req.AccountID, err)
 	}
 
-	job, replay, err := claimDeletionJob(ctx, jobs, &req)
+	job, revision, replay, err := claimDeletionJob(ctx, jobs, &req, d.holderID())
 	if err != nil {
 		return nil, err
 	}
@@ -141,7 +165,7 @@ func (d Deps) deleteAccount(ctx context.Context, body []byte) (any, error) {
 		return deletionJobResponse(replay), nil
 	}
 
-	d.runAccountDeletion(engine, jobs, job)
+	d.runAccountDeletion(engine, jobs, job, revision)
 
 	return &DeleteAccountResponse{
 		DeletionID: job.DeletionID, AccountID: job.AccountID, State: job.State,
@@ -209,6 +233,19 @@ func (d Deps) listAccounts(_ context.Context, _ []byte) (any, error) {
 	return response, nil
 }
 
+// holderID names this gateway in the job record. The node ID is used when the
+// composition supplies one, otherwise the host name.
+func (d Deps) holderID() string {
+	if d.NodeID != "" {
+		return d.NodeID
+	}
+	host, err := os.Hostname()
+	if err != nil {
+		return "unknown"
+	}
+	return host
+}
+
 // runAccountDeletion runs the teardown in the background on its own context.
 // The request's context is cancelled the moment the response is written, and a
 // teardown cancelled halfway leaves the account half-emptied.
@@ -216,12 +253,13 @@ func (d Deps) runAccountDeletion(
 	engine *accountteardown.Engine,
 	jobs jetstream.KeyValue,
 	job *accountDeletionJob,
+	revision uint64,
 ) {
-	tracker := &deletionTracker{jobs: jobs, job: job}
+	ctx, cancel := context.WithTimeout(context.Background(), accountDeletionTimeout)
+	tracker := &deletionTracker{jobs: jobs, job: job, revision: revision, cancel: cancel}
 	engine.OnStage = tracker.stageFinished
 
 	go func() {
-		ctx, cancel := context.WithTimeout(context.Background(), accountDeletionTimeout)
 		defer cancel()
 
 		stop := tracker.startHeartbeat(ctx)
@@ -237,12 +275,17 @@ func (d Deps) runAccountDeletion(
 }
 
 // deletionTracker keeps the stored job in step with a teardown as it runs. Its
-// mutex covers the heartbeat and the stage callback, which write concurrently.
+// mutex covers the heartbeat and the stage callback, which write concurrently,
+// and the revision each write is conditional on.
 type deletionTracker struct {
 	jobs jetstream.KeyValue
+	// cancel stops the teardown once the tracker finds it has been superseded.
+	cancel context.CancelFunc
 
-	mu  sync.Mutex
-	job *accountDeletionJob
+	mu       sync.Mutex
+	job      *accountDeletionJob
+	revision uint64
+	lost     bool
 }
 
 func (t *deletionTracker) stageFinished(ctx context.Context, stage accountteardown.StageResult) {
@@ -268,6 +311,13 @@ func (t *deletionTracker) finished(ctx context.Context, result *accountteardown.
 	t.mu.Unlock()
 
 	t.persist(ctx)
+
+	t.mu.Lock()
+	lost := t.lost
+	t.mu.Unlock()
+	if lost {
+		return
+	}
 	if err != nil {
 		slog.Error("DeleteAccount: teardown failed", "accountID", accountID, "err", err)
 		return
@@ -275,9 +325,9 @@ func (t *deletionTracker) finished(ctx context.Context, result *accountteardown.
 	slog.Info("DeleteAccount: teardown complete", "accountID", accountID, "state", state)
 }
 
-// startHeartbeat refreshes UpdatedAt while the teardown runs. Without it a
-// gateway restarted mid-teardown leaves a job that looks live forever, and the
-// account can never be retried through the API.
+// startHeartbeat refreshes UpdatedAt and the lease while the teardown runs.
+// Without it a gateway restarted mid-teardown leaves a job that looks live
+// forever, and the account can never be retried through the API.
 func (t *deletionTracker) startHeartbeat(ctx context.Context) func() {
 	ticker := time.NewTicker(accountDeletionHeartbeat)
 	done := make(chan struct{})
@@ -299,35 +349,81 @@ func (t *deletionTracker) startHeartbeat(ctx context.Context) func() {
 	return func() { close(done) }
 }
 
+// persistAttempts bounds the retries after a revision conflict that turns out
+// to be this holder's own earlier write.
+const persistAttempts = 3
+
+// persist writes the job at the revision this holder last saw. A conflict is
+// resolved by re-reading: a different fence means another holder took over, so
+// this one stops for good; the same fence only means the revision moved.
 func (t *deletionTracker) persist(ctx context.Context) {
 	t.mu.Lock()
-	t.job.UpdatedAt = time.Now().UTC().Format(time.RFC3339)
-	record, err := json.Marshal(t.job)
-	accountID := t.job.AccountID
-	t.mu.Unlock()
-
-	if err != nil {
-		slog.Error("DeleteAccount: failed to marshal job record", "accountID", accountID, "err", err)
+	defer t.mu.Unlock()
+	if t.lost {
 		return
 	}
-	if _, err := t.jobs.Put(ctx, accountID, record); err != nil {
-		// The teardown itself is unaffected; only its progress report is. Losing
-		// the run over a KV write would be the worse outcome.
-		slog.Error("DeleteAccount: failed to store job progress", "accountID", accountID, "err", err)
+
+	for range persistAttempts {
+		now := time.Now().UTC()
+		t.job.UpdatedAt = now.Format(time.RFC3339)
+		t.job.LeaseUntil = now.Add(accountDeletionStale).Format(time.RFC3339)
+		record, err := json.Marshal(t.job)
+		if err != nil {
+			slog.Error("DeleteAccount: failed to marshal job record", "accountID", t.job.AccountID, "err", err)
+			return
+		}
+
+		revision, err := t.jobs.Update(ctx, t.job.AccountID, record, t.revision)
+		if err == nil {
+			t.revision = revision
+			return
+		}
+		if !isRevisionConflict(err) {
+			// The teardown itself is unaffected; only its progress report is. Losing
+			// the run over a KV write would be the worse outcome.
+			slog.Error("DeleteAccount: failed to store job progress", "accountID", t.job.AccountID, "err", err)
+			return
+		}
+
+		current, currentRevision, readErr := readDeletionEntry(ctx, t.jobs, t.job.AccountID)
+		if readErr != nil || current == nil {
+			slog.Error("DeleteAccount: failed to re-read job after a conflict", "accountID", t.job.AccountID, "err", readErr)
+			return
+		}
+		if current.Fence != t.job.Fence {
+			t.lost = true
+			slog.Warn("DeleteAccount: superseded by another holder, stopping",
+				"account_id", t.job.AccountID, "operation_id", t.job.OperationID,
+				"own_fence", t.job.Fence, "current_fence", current.Fence)
+			if t.cancel != nil {
+				t.cancel()
+			}
+			return
+		}
+		t.revision = currentRevision
 	}
+	slog.Error("DeleteAccount: job progress kept conflicting", "accountID", t.job.AccountID)
+}
+
+func isRevisionConflict(err error) bool {
+	return errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch)
 }
 
 // claimDeletionJob CAS-creates the job record, which is what stops two
 // gateways tearing the same account down at once. It returns a non-nil replay
-// when an existing job answers the request instead.
+// when an existing job answers the request instead, and the revision the new
+// holder's later writes are conditional on.
 func claimDeletionJob(
 	ctx context.Context,
 	jobs jetstream.KeyValue,
 	req *operatorv1.DeleteAccountRequest,
-) (*accountDeletionJob, *accountDeletionJob, error) {
+	holder string,
+) (*accountDeletionJob, uint64, *accountDeletionJob, error) {
 	now := time.Now().UTC()
+	operationID := uuid.NewV4().String()
 	job := &accountDeletionJob{
-		DeletionID:  uuid.NewV4().String(),
+		DeletionID:  operationID,
+		OperationID: operationID,
 		AccountID:   req.AccountID,
 		AccountName: req.AccountName,
 		ClientToken: req.ClientToken,
@@ -335,48 +431,77 @@ func claimDeletionJob(
 		Force:       req.Force,
 		StartedAt:   now.Format(time.RFC3339),
 		UpdatedAt:   now.Format(time.RFC3339),
+		Fence:       1,
+		Holder:      holder,
+		LeaseUntil:  now.Add(accountDeletionStale).Format(time.RFC3339),
 	}
 	record, err := json.Marshal(job)
 	if err != nil {
 		slog.Error("DeleteAccount: failed to marshal job record", "err", err)
-		return nil, nil, errors.New(awserrors.ErrorInternalError)
+		return nil, 0, nil, errors.New(awserrors.ErrorInternalError)
 	}
 
-	if _, err := jobs.Create(ctx, req.AccountID, record); err == nil {
-		return job, nil, nil
+	revision, err := jobs.Create(ctx, req.AccountID, record)
+	if err == nil {
+		return job, revision, nil, nil
 	} else if !errors.Is(err, jetstream.ErrKeyExists) {
 		slog.Error("DeleteAccount: failed to claim deletion job", "accountID", req.AccountID, "err", err)
-		return nil, nil, errors.New(awserrors.ErrorInternalError)
+		return nil, 0, nil, errors.New(awserrors.ErrorInternalError)
 	}
 
 	// Another gateway claimed it between the replay check and here, so the same
 	// token has to be recognised a second time.
 	if replay, err := replayDeletionJob(ctx, jobs, req); err != nil || replay != nil {
-		return nil, replay, err
+		return nil, 0, replay, err
 	}
 
-	existing, err := readDeletionJob(ctx, jobs, req.AccountID)
+	existing, existingRevision, err := readDeletionEntry(ctx, jobs, req.AccountID)
 	if err != nil {
-		return nil, nil, err
+		return nil, 0, nil, err
 	}
 	if existing == nil {
 		// Deleted between Create and Get. Retryable rather than a fault.
-		return nil, nil, errors.New(awserrors.ErrorOperationInProgress)
+		return nil, 0, nil, errors.New(awserrors.ErrorOperationInProgress)
 	}
 
-	if existing.State == operatorv1.DeletionStateRunning && !deletionJobStale(existing, now) {
-		return nil, nil, errors.New(awserrors.ErrorOperationInProgress)
+	reason := takeoverReasonFailed
+	if existing.State == operatorv1.DeletionStateRunning {
+		if !deletionLeaseExpired(existing, now) {
+			return nil, 0, nil, errors.New(awserrors.ErrorOperationInProgress)
+		}
+		reason = takeoverReasonLeaseExpired
 	}
 
-	// A failed job, or one whose gateway stopped heartbeating, is retried under
-	// a new deletion id. Teardown is idempotent: it re-lists what is left.
-	if _, err := jobs.Put(ctx, req.AccountID, record); err != nil {
+	// A failed job, or one whose holder stopped renewing its lease, is taken
+	// over at the next fence. Teardown is idempotent: it re-lists what is left.
+	// The operation keeps its id; a record from before fences gets one now.
+	if existing.OperationID != "" {
+		job.OperationID = existing.OperationID
+		job.DeletionID = existing.OperationID
+	}
+	job.Fence = existing.Fence + 1
+	job.Takeovers = append(append([]deletionTakeover(nil), existing.Takeovers...), deletionTakeover{
+		Fence: job.Fence, Holder: holder, At: now.Format(time.RFC3339), Reason: reason,
+	})
+	record, err = json.Marshal(job)
+	if err != nil {
+		slog.Error("DeleteAccount: failed to marshal job record", "err", err)
+		return nil, 0, nil, errors.New(awserrors.ErrorInternalError)
+	}
+
+	revision, err = jobs.Update(ctx, req.AccountID, record, existingRevision)
+	if err != nil {
+		if isRevisionConflict(err) {
+			// Another taker won the same expired lease.
+			return nil, 0, nil, errors.New(awserrors.ErrorOperationInProgress)
+		}
 		slog.Error("DeleteAccount: failed to take over deletion job", "accountID", req.AccountID, "err", err)
-		return nil, nil, errors.New(awserrors.ErrorInternalError)
+		return nil, 0, nil, errors.New(awserrors.ErrorInternalError)
 	}
-	slog.Warn("DeleteAccount: retrying a job that did not finish",
-		"accountID", req.AccountID, "previousDeletionID", existing.DeletionID, "previousState", existing.State)
-	return job, nil, nil
+	slog.Warn("DeleteAccount: taking over a job that did not finish",
+		"account_id", req.AccountID, "operation_id", job.OperationID,
+		"previous_fence", existing.Fence, "fence", job.Fence, "previous_state", existing.State, "reason", reason)
+	return job, revision, nil, nil
 }
 
 // replayDeletionJob answers a retry from the stored record. A token bound to a
@@ -395,6 +520,11 @@ func replayDeletionJob(
 		if existing.AccountName != req.AccountName {
 			return nil, errors.New(awserrors.ErrorIdempotentParameterMismatch)
 		}
+		// A RUNNING job whose lease has lapsed has no holder to follow, so the
+		// retry falls through to takeover and resumes it.
+		if existing.State == operatorv1.DeletionStateRunning && deletionLeaseExpired(existing, time.Now().UTC()) {
+			return nil, nil
+		}
 		return existing, nil
 	}
 
@@ -418,8 +548,22 @@ func deletionJobResponse(job *accountDeletionJob) *DeleteAccountResponse {
 	}
 }
 
-// deletionJobStale reports whether a RUNNING job has stopped heartbeating for
-// long enough that the gateway running it is presumed gone.
+// deletionLeaseExpired reports whether a RUNNING job has no live holder. The
+// lease is authoritative; a record from before leases falls back to its
+// heartbeat age.
+func deletionLeaseExpired(job *accountDeletionJob, now time.Time) bool {
+	if job.LeaseUntil != "" {
+		until, err := time.Parse(time.RFC3339, job.LeaseUntil)
+		if err != nil {
+			return true
+		}
+		return now.After(until)
+	}
+	return deletionJobStale(job, now)
+}
+
+// deletionJobStale reports whether a job's heartbeat has stopped for long
+// enough that the gateway running it is presumed gone.
 func deletionJobStale(job *accountDeletionJob, now time.Time) bool {
 	updated, err := time.Parse(time.RFC3339, job.UpdatedAt)
 	if err != nil {
@@ -431,21 +575,27 @@ func deletionJobStale(job *accountDeletionJob, now time.Time) bool {
 }
 
 func readDeletionJob(ctx context.Context, jobs jetstream.KeyValue, accountID string) (*accountDeletionJob, error) {
+	job, _, err := readDeletionEntry(ctx, jobs, accountID)
+	return job, err
+}
+
+// readDeletionEntry also returns the revision a conditional write is based on.
+func readDeletionEntry(ctx context.Context, jobs jetstream.KeyValue, accountID string) (*accountDeletionJob, uint64, error) {
 	entry, err := jobs.Get(ctx, accountID)
 	if err != nil {
 		if errors.Is(err, jetstream.ErrKeyNotFound) {
-			return nil, nil
+			return nil, 0, nil
 		}
 		slog.Error("DeleteAccount: failed to read deletion job", "accountID", accountID, "err", err)
-		return nil, errors.New(awserrors.ErrorInternalError)
+		return nil, 0, errors.New(awserrors.ErrorInternalError)
 	}
 
 	var job accountDeletionJob
 	if err := json.Unmarshal(entry.Value(), &job); err != nil {
 		slog.Error("DeleteAccount: failed to unmarshal deletion job", "accountID", accountID, "err", err)
-		return nil, errors.New(awserrors.ErrorInternalError)
+		return nil, 0, errors.New(awserrors.ErrorInternalError)
 	}
-	return &job, nil
+	return &job, entry.Revision(), nil
 }
 
 // accountDeletionStore opens the job bucket. Opened per request, like the

spinifex/operator/admin/account_delete_fence_test.go

@@ -0,0 +1,236 @@
+package admin
+
+//test:in-package — the deletion job record, its claim rules and the progress
+// tracker are unexported, and the fence is only reachable through them.
+
+import (
+	"encoding/json"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	operatorv1 "github.com/mulgadc/spinifex/contracts/operator/v1"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	accountteardown "github.com/mulgadc/spinifex/spinifex/operator/accountlifecycle"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// expireLease rewrites the stored record so its holder looks dead.
+func expireLease(t *testing.T, jobs jetstream.KeyValue, accountID string) {
+	t.Helper()
+	job, err := readDeletionJob(t.Context(), jobs, accountID)
+	require.NoError(t, err)
+	require.NotNil(t, job)
+	past := time.Now().UTC().Add(-2 * accountDeletionStale).Format(time.RFC3339)
+	job.UpdatedAt, job.LeaseUntil = past, past
+	record, err := json.Marshal(job)
+	require.NoError(t, err)
+	_, err = jobs.Put(t.Context(), accountID, record)
+	require.NoError(t, err)
+}
+
+func claimAs(t *testing.T, jobs jetstream.KeyValue, accountID, token, holder string) (*accountDeletionJob, uint64, error) {
+	t.Helper()
+	job, revision, replay, err := claimDeletionJob(t.Context(), jobs, &operatorv1.DeleteAccountRequest{
+		AccountID: accountID, AccountName: "tenant@example.com", ClientToken: token,
+	}, holder)
+	if err == nil {
+		require.Nil(t, replay)
+	}
+	return job, revision, err
+}
+
+// A holder that has been taken over must not write again: its next write finds
+// the new holder's fence, and it stops and cancels its teardown.
+func TestSupersededHolderStopsAndLeavesTheNewHoldersRecord(t *testing.T) {
+	gw := newDeleteAccountGateway(t)
+	jobs, err := gw.accountDeletionStore(t.Context())
+	require.NoError(t, err)
+	const accountID = "000000000042"
+
+	tokenA, tokenB := strings.Repeat("a", 32), strings.Repeat("b", 32)
+	holderA, revisionA, err := claimAs(t, jobs, accountID, tokenA, "node-a")
+	require.NoError(t, err)
+	var cancelled atomic.Bool
+	trackerA := &deletionTracker{jobs: jobs, job: holderA, revision: revisionA, cancel: func() { cancelled.Store(true) }}
+
+	expireLease(t, jobs, accountID)
+	holderB, _, err := claimAs(t, jobs, accountID, tokenB, "node-b")
+	require.NoError(t, err)
+	require.Equal(t, holderA.OperationID, holderB.OperationID, "the operation survives a takeover")
+	require.Equal(t, uint64(2), holderB.Fence)
+
+	trackerA.stageFinished(t.Context(), accountteardown.StageResult{Stage: accountteardown.StageCompute})
+	assert.True(t, cancelled.Load(), "a superseded holder must cancel its teardown")
+	afterStage := describeDeletion(t, gw, accountID)
+	assert.Equal(t, uint64(2), afterStage.Fence)
+	assert.Equal(t, "node-b", afterStage.Holder)
+	assert.Equal(t, tokenB, afterStage.ClientToken)
+	assert.Empty(t, afterStage.Stages, "the old holder's stage must not land in the new record")
+
+	trackerA.finished(t.Context(), nil, nil)
+	afterFinish := describeDeletion(t, gw, accountID)
+	assert.Equal(t, operatorv1.DeletionStateRunning, afterFinish.State, "the old holder cannot declare the operation finished")
+	assert.Equal(t, uint64(2), afterFinish.Fence)
+}
+
+// A retry carrying the stored token against a RUNNING job whose lease has
+// lapsed has no holder to follow, so it takes the job over and resumes it.
+func TestSameTokenRetryResumesAStaleRunningJob(t *testing.T) {
+	gw := newDeleteAccountGateway(t)
+	account := newTenantAccount(t, gw, "tenant@example.com")
+	past := time.Now().UTC().Add(-2 * accountDeletionStale).Format(time.RFC3339)
+	storeDeletionJob(t, gw, &accountDeletionJob{
+		DeletionID:  "op-abandoned",
+		OperationID: "op-abandoned",
+		AccountID:   account.AccountID,
+		AccountName: "tenant@example.com",
+		ClientToken: deleteClientToken,
+		State:       operatorv1.DeletionStateRunning,
+		UpdatedAt:   past,
+		LeaseUntil:  past,
+		Fence:       1,
+		Holder:      "node-dead",
+	})
+
+	output, err := gw.deleteAccount(t.Context(),
+		deleteAccountBody(t, account.AccountID, "tenant@example.com", deleteClientToken))
+	require.NoError(t, err)
+	response, ok := output.(*DeleteAccountResponse)
+	require.True(t, ok)
+	assert.Equal(t, "op-abandoned", response.DeletionID, "the operation id is unchanged by a takeover")
+
+	stored := describeDeletion(t, gw, account.AccountID)
+	assert.Equal(t, uint64(2), stored.Fence)
+	assert.Equal(t, "op-abandoned", stored.OperationID)
+	require.Len(t, stored.Takeovers, 1)
+	assert.Equal(t, uint64(2), stored.Takeovers[0].Fence)
+	assert.Equal(t, takeoverReasonLeaseExpired, stored.Takeovers[0].Reason)
+	assert.NotEmpty(t, stored.Takeovers[0].Holder)
+}
+
+// A same-token retry against a job whose lease is live still follows it.
+func TestSameTokenRetryFollowsALiveJob(t *testing.T) {
+	gw := newDeleteAccountGateway(t)
+	account := newTenantAccount(t, gw, "tenant@example.com")
+	now := time.Now().UTC()
+	storeDeletionJob(t, gw, &accountDeletionJob{
+		DeletionID:  "op-live",
+		OperationID: "op-live",
+		AccountID:   account.AccountID,
+		AccountName: "tenant@example.com",
+		ClientToken: deleteClientToken,
+		State:       operatorv1.DeletionStateRunning,
+		UpdatedAt:   now.Format(time.RFC3339),
+		LeaseUntil:  now.Add(accountDeletionStale).Format(time.RFC3339),
+		Fence:       1,
+		Holder:      "node-live",
+	})
+
+	output, err := gw.deleteAccount(t.Context(),
+		deleteAccountBody(t, account.AccountID, "tenant@example.com", deleteClientToken))
+	require.NoError(t, err)
+	response, ok := output.(*DeleteAccountResponse)
+	require.True(t, ok)
+	assert.Equal(t, "op-live", response.DeletionID)
+	assert.Equal(t, uint64(1), describeDeletion(t, gw, account.AccountID).Fence, "no takeover of a live lease")
+}
+
+// Two gateways racing for one expired lease: the conditional write lets exactly
+// one through, and the other is told the operation is in progress.
+func TestConcurrentTakeoverHasExactlyOneWinner(t *testing.T) {
+	gw := newDeleteAccountGateway(t)
+	jobs, err := gw.accountDeletionStore(t.Context())
+	require.NoError(t, err)
+	const accountID = "000000000042"
+
+	_, _, err = claimAs(t, jobs, accountID, strings.Repeat("a", 32), "node-a")
+	require.NoError(t, err)
+	expireLease(t, jobs, accountID)
+
+	const takers = 8
+	var (
+		wg      sync.WaitGroup
+		winners atomic.Int32
+		losers  atomic.Int32
+	)
+	for i := range takers {
+		wg.Go(func() {
+			token := strings.Repeat(string(rune('c'+i)), 32)
+			_, _, replay, err := claimDeletionJob(t.Context(), jobs, &operatorv1.DeleteAccountRequest{
+				AccountID: accountID, AccountName: "tenant@example.com", ClientToken: token,
+			}, "node-"+token[:1])
+			switch {
+			case err == nil && replay == nil:
+				winners.Add(1)
+			case err != nil && err.Error() == awserrors.ErrorOperationInProgress:
+				losers.Add(1)
+			default:
+				t.Errorf("unexpected outcome: replay=%v err=%v", replay, err)
+			}
+		})
+	}
+	wg.Wait()
+
+	assert.Equal(t, int32(1), winners.Load())
+	assert.Equal(t, int32(takers-1), losers.Load())
+	stored := describeDeletion(t, gw, accountID)
+	assert.Equal(t, uint64(2), stored.Fence)
+	assert.Len(t, stored.Takeovers, 1)
+}
+
+// A record from a binary that wrote no fence reads as fence 0 and is taken over
+// to fence 1.
+func TestLegacyRecordWithoutAFenceIsTakenOverToFenceOne(t *testing.T) {
+	gw := newDeleteAccountGateway(t)
+	jobs := storeDeletionJob(t, gw, &accountDeletionJob{
+		DeletionID:  "d-legacy",
+		AccountID:   "000000000042",
+		AccountName: "tenant@example.com",
+		ClientToken: strings.Repeat("a", 32),
+		State:       operatorv1.DeletionStateRunning,
+		UpdatedAt:   time.Now().UTC().Add(-2 * accountDeletionStale).Format(time.RFC3339),
+	})
+
+	job, _, err := claimAs(t, jobs, "000000000042", strings.Repeat("b", 32), "node-b")
+	require.NoError(t, err)
+	assert.Equal(t, uint64(1), job.Fence)
+	assert.NotEmpty(t, job.OperationID)
+	assert.Equal(t, job.OperationID, job.DeletionID)
+	assert.NotEqual(t, "d-legacy", job.DeletionID)
+	require.Len(t, job.Takeovers, 1)
+	assert.Equal(t, "node-b", job.Takeovers[0].Holder)
+}
+
+// A revision conflict caused by this holder's own fence is not a takeover: the
+// write is retried at the new revision and the holder carries on.
+func TestHeartbeatConflictWithOwnFenceRetries(t *testing.T) {
+	gw := newDeleteAccountGateway(t)
+	jobs, err := gw.accountDeletionStore(t.Context())
+	require.NoError(t, err)
+	const accountID = "000000000042"
+
+	job, revision, err := claimAs(t, jobs, accountID, strings.Repeat("a", 32), "node-a")
+	require.NoError(t, err)
+	var cancelled atomic.Bool
+	tracker := &deletionTracker{jobs: jobs, job: job, revision: revision, cancel: func() { cancelled.Store(true) }}
+
+	// The revision moves under the tracker without the fence changing.
+	record, err := json.Marshal(job)
+	require.NoError(t, err)
+	moved, err := jobs.Put(t.Context(), accountID, record)
+	require.NoError(t, err)
+	require.Greater(t, moved, revision)
+
+	tracker.stageFinished(t.Context(), accountteardown.StageResult{Stage: accountteardown.StageCompute})
+
+	assert.False(t, cancelled.Load())
+	stored := describeDeletion(t, gw, accountID)
+	require.Len(t, stored.Stages, 1)
+	assert.Equal(t, uint64(1), stored.Fence)
+	assert.NotEmpty(t, stored.LeaseUntil, "the heartbeat renews the lease")
+}

spinifex/operator/admin/account_delete_job_test.go

@@ -199,6 +199,15 @@ func storeDeletionJob(t *testing.T, gw Deps, job *accountDeletionJob) jetstream.
 	return jobs
 }
 
+// newTestTracker builds a tracker for a record seeded with storeDeletionJob,
+// at the revision a real holder would have been handed.
+func newTestTracker(t *testing.T, jobs jetstream.KeyValue, job *accountDeletionJob) *deletionTracker {
+	t.Helper()
+	entry, err := jobs.Get(t.Context(), job.AccountID)
+	require.NoError(t, err)
+	return &deletionTracker{jobs: jobs, job: job, revision: entry.Revision()}
+}
+
 func describeDeletion(t *testing.T, gw Deps, accountID string) *accountDeletionJob {
 	t.Helper()
 	body, err := json.Marshal(operatorv1.DescribeAccountDeletionRequest{AccountID: accountID})
@@ -512,7 +521,7 @@ func TestDeletionTrackerRecordsEachStageAsItFinishes(t *testing.T) {
 		State:      operatorv1.DeletionStateRunning,
 	}
 	jobs := storeDeletionJob(t, gw, job)
-	tracker := &deletionTracker{jobs: jobs, job: job}
+	tracker := newTestTracker(t, jobs, job)
 
 	tracker.stageFinished(t.Context(), accountteardown.StageResult{
 		Stage:   accountteardown.StageCompute,
@@ -538,7 +547,7 @@ func TestDeletionTrackerRecordsFailure(t *testing.T) {
 		State:      operatorv1.DeletionStateRunning,
 	}
 	jobs := storeDeletionJob(t, gw, job)
-	tracker := &deletionTracker{jobs: jobs, job: job}
+	tracker := newTestTracker(t, jobs, job)
 
 	tracker.finished(t.Context(), nil, assert.AnError)
 
@@ -558,7 +567,7 @@ func TestDeletionTrackerHeartbeatStops(t *testing.T) {
 		State:      operatorv1.DeletionStateRunning,
 	}
 	jobs := storeDeletionJob(t, gw, job)
-	tracker := &deletionTracker{jobs: jobs, job: job}
+	tracker := newTestTracker(t, jobs, job)
 
 	stop := tracker.startHeartbeat(t.Context())
 	stop()

spinifex/operator/admin/registration.go

@@ -46,6 +46,9 @@ type Deps struct {
 	NATS        *nats.Conn
 	RootCAs     *x509.CertPool
 	ActiveNodes func(ctx context.Context) int
+	// NodeID names this gateway in a deletion job's holder field; the host
+	// name stands in when it is empty.
+	NodeID      string
 	ModelAccess ModelAccess
 	Credentials ochre.CredentialResolver
 	Access      ochre.AccessResolver

spinifex/runtime/daemon/daemon_sqs_worker_test.go

@@ -258,7 +258,7 @@ type fakeRoleIAM struct {
 
 func (f *fakeRoleIAM) GetRole(_ string, in *awsiam.GetRoleInput) (*awsiam.GetRoleOutput, error) {
 	if f.trust == "" {
-		return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "The role with name %s cannot be found.", aws.StringValue(in.RoleName))
 	}
 	return &awsiam.GetRoleOutput{Role: &awsiam.Role{RoleName: in.RoleName, AssumeRolePolicyDocument: aws.String(f.trust)}}, nil
 }
@@ -267,7 +267,7 @@ func (f *fakeRoleIAM) CreateRole(_ string, in *awsiam.CreateRoleInput) (*awsiam.
 	f.writes = append(f.writes, "CreateRole")
 	if f.createdTrust != "" {
 		f.trust = f.createdTrust
-		return nil, errors.New(awserrors.ErrorIAMEntityAlreadyExists)
+		return nil, awserrors.Errorf(awserrors.ErrorIAMEntityAlreadyExists, "Role with name %s already exists.", aws.StringValue(in.RoleName))
 	}
 	f.trust = aws.StringValue(in.AssumeRolePolicyDocument)
 	return &awsiam.CreateRoleOutput{}, nil
@@ -279,9 +279,9 @@ func (f *fakeRoleIAM) PutRolePolicy(_ string, in *awsiam.PutRolePolicyInput) (*a
 	return &awsiam.PutRolePolicyOutput{}, nil
 }
 
-func (f *fakeRoleIAM) GetInstanceProfile(string, *awsiam.GetInstanceProfileInput) (*awsiam.GetInstanceProfileOutput, error) {
+func (f *fakeRoleIAM) GetInstanceProfile(_ string, in *awsiam.GetInstanceProfileInput) (*awsiam.GetInstanceProfileOutput, error) {
 	if f.profile == nil {
-		return nil, errors.New(awserrors.ErrorIAMNoSuchEntity)
+		return nil, awserrors.Errorf(awserrors.ErrorIAMNoSuchEntity, "Instance Profile %s cannot be found.", aws.StringValue(in.InstanceProfileName))
 	}
 	p := *f.profile
 	p.Roles = slices.Clone(f.profile.Roles)

spinifex/runtime/daemon/sqs_worker_deps.go

@@ -96,13 +96,13 @@ func ensureSQSRoleTrust(svc iamdomain.SystemInstanceRoleEnsurer, r sqsRole) erro
 	acct, name := awsidentifiers.GlobalAccountID, aws.String(r.name)
 	role, err := svc.GetRole(acct, &awsiam.GetRoleInput{RoleName: name})
 	if err != nil {
-		if err.Error() != awserrors.ErrorIAMNoSuchEntity {
+		if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMNoSuchEntity {
 			return fmt.Errorf("%s: read role: %w", r.label, err)
 		}
 		_, cerr := svc.CreateRole(acct, &awsiam.CreateRoleInput{RoleName: name,
 			AssumeRolePolicyDocument: aws.String(iamdomain.EC2InstanceTrustPolicy),
 			Description:              aws.String(r.description)})
-		if cerr != nil && cerr.Error() != awserrors.ErrorIAMEntityAlreadyExists {
+		if code, _ := awserrors.ResolveErrorCode(cerr); cerr != nil && code != awserrors.ErrorIAMEntityAlreadyExists {
 			return fmt.Errorf("%s: create role: %w", r.label, cerr)
 		}
 		if role, err = svc.GetRole(acct, &awsiam.GetRoleInput{RoleName: name}); err != nil {
@@ -132,11 +132,11 @@ func ensureSQSRoleInstanceProfile(svc iamdomain.SystemInstanceRoleEnsurer, r sqs
 	}
 	prof, err := get()
 	if err != nil {
-		if err.Error() != awserrors.ErrorIAMNoSuchEntity {
+		if code, _ := awserrors.ResolveErrorCode(err); code != awserrors.ErrorIAMNoSuchEntity {
 			return "", fmt.Errorf("%s: read instance profile: %w", r.label, err)
 		}
-		if _, cerr := svc.CreateInstanceProfile(acct, &awsiam.CreateInstanceProfileInput{InstanceProfileName: name}); cerr != nil &&
-			cerr.Error() != awserrors.ErrorIAMEntityAlreadyExists {
+		_, cerr := svc.CreateInstanceProfile(acct, &awsiam.CreateInstanceProfileInput{InstanceProfileName: name})
+		if code, _ := awserrors.ResolveErrorCode(cerr); cerr != nil && code != awserrors.ErrorIAMEntityAlreadyExists {
 			return "", fmt.Errorf("%s: create instance profile: %w", r.label, cerr)
 		}
 		if prof, err = get(); err != nil {
@@ -147,8 +147,8 @@ func ensureSQSRoleInstanceProfile(svc iamdomain.SystemInstanceRoleEnsurer, r sqs
 		return "", fmt.Errorf("%s: instance profile %s carries a role other than %s", r.label, r.name, r.name)
 	}
 	if len(prof.Roles) == 0 {
-		if _, aerr := svc.AddRoleToInstanceProfile(acct, &awsiam.AddRoleToInstanceProfileInput{
-			InstanceProfileName: name, RoleName: name}); aerr != nil && aerr.Error() != awserrors.ErrorIAMLimitExceeded {
+		_, aerr := svc.AddRoleToInstanceProfile(acct, &awsiam.AddRoleToInstanceProfileInput{InstanceProfileName: name, RoleName: name})
+		if code, _ := awserrors.ResolveErrorCode(aerr); aerr != nil && code != awserrors.ErrorIAMLimitExceeded {
 			return "", fmt.Errorf("%s: add role to instance profile: %w", r.label, aerr)
 		}
 		if prof, err = get(); err != nil {

spinifex/runtime/roles/awsgw/awsgw.go

@@ -365,6 +365,21 @@ func launchService(config *config.ClusterConfig) error {
 		ecrRegistry, activeAccountIDs(iamService), ecrregistry.DefaultLifecycleSweepInterval)
 	go lifecycleSweeper.Run(janitorCtx)
 
+	// Lambda code copies live in a Regional system bucket reached through a
+	// scoped role session. Nothing is provisioned until the store is first used.
+	lambdaCodeStore, err := newLambdaCodeStore(codeStoreWiring{
+		Region:        nodeConfig.Region,
+		Node:          config.Node,
+		Endpoint:      netaddr.DialTarget(nodeConfig.Predastore.Host),
+		SigningRegion: nodeConfig.Predastore.Region,
+		RootAccessKey: nodeConfig.Predastore.AccessKey,
+		RootSecretKey: nodeConfig.Predastore.SecretKey,
+	}, iamService, stsService)
+	if err != nil {
+		return err
+	}
+	_ = lambdaCodeStore // no Lambda handler is registered to use it yet
+
 	// ECR auth bridge: load (or first-run create) the ES256 signing key from the
 	// cluster-replicated awsgw-keys KV bucket, then build the token issuer
 	// (GetAuthorizationToken) and verifier (/v2 Authorization). The bridge reads
@@ -892,8 +907,10 @@ func parseBedrockEndpoints(raw string) map[string]string {
 
 // withClusterNodes sets the configured node sets teardown reaches: the
 // daemon on every node, and the gateway only on nodes that run awsgw. An
-// empty gateway set would fail every account termination closed.
+// empty gateway set would fail every account termination closed. NodeID names
+// this gateway as the holder of a deletion job.
 func withClusterNodes(d operatoradmin.Deps, config *config.ClusterConfig) operatoradmin.Deps {
+	d.NodeID = config.Node
 	d.ExpectedNodes = len(config.Nodes)
 	d.NodeIDs = slices.Sorted(maps.Keys(config.Nodes))
 	d.GatewayNodeIDs = config.NodesRunning("awsgw")

spinifex/runtime/roles/awsgw/compose_test.go

@@ -40,7 +40,7 @@ func TestComposeGateway_SetsEveryProductionHook(t *testing.T) {
 // Account termination probes the gateway node set before sealing, so awsgw
 // must hand the operator surface both node sets from the cluster config.
 func TestWithClusterNodes_SetsDaemonAndGatewaySets(t *testing.T) {
-	cfg := &config.ClusterConfig{Nodes: map[string]config.Config{
+	cfg := &config.ClusterConfig{Node: "n2", Nodes: map[string]config.Config{
 		"n1": {Services: []string{"nats", "awsgw", "daemon"}},
 		"n2": {Services: []string{"daemon"}},
 		"n3": {},
@@ -48,6 +48,7 @@ func TestWithClusterNodes_SetsDaemonAndGatewaySets(t *testing.T) {
 
 	got := withClusterNodes(operatoradmin.Deps{}, cfg)
 
+	assert.Equal(t, "n2", got.NodeID, "the deletion job holder is this node")
 	assert.Equal(t, 3, got.ExpectedNodes)
 	assert.Equal(t, []string{"n1", "n2", "n3"}, got.NodeIDs)
 	// n3 has no service list, which means every service, awsgw included.

spinifex/runtime/roles/awsgw/lambda_code_store.go

@@ -0,0 +1,196 @@
+package awsgw
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/sts"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/codestore"
+	stsdomain "github.com/mulgadc/spinifex/spinifex/domains/sts"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+)
+
+// codeStoreRoleIAM is the IAM surface the code-store role adapter uses.
+type codeStoreRoleIAM interface {
+	CreateRole(accountID string, input *iam.CreateRoleInput) (*iam.CreateRoleOutput, error)
+	GetRole(accountID string, input *iam.GetRoleInput) (*iam.GetRoleOutput, error)
+	ListRoleTags(accountID string, input *iam.ListRoleTagsInput) (*iam.ListRoleTagsOutput, error)
+	ListRolePolicies(accountID string, input *iam.ListRolePoliciesInput) (*iam.ListRolePoliciesOutput, error)
+	ListAttachedRolePolicies(accountID string, input *iam.ListAttachedRolePoliciesInput) (*iam.ListAttachedRolePoliciesOutput, error)
+	PutRolePolicy(accountID string, input *iam.PutRolePolicyInput) (*iam.PutRolePolicyOutput, error)
+}
+
+// codeStoreRole binds IAM to the one system-account code-store role, so the
+// Lambda domain never holds a general IAM client.
+type codeStoreRole struct {
+	iam  codeStoreRoleIAM
+	name string
+}
+
+var _ codestore.RoleAdmin = (*codeStoreRole)(nil)
+
+func (r *codeStoreRole) Create(trust string, tags map[string]string) error {
+	in := &iam.CreateRoleInput{
+		RoleName:                 aws.String(r.name),
+		AssumeRolePolicyDocument: aws.String(trust),
+		Description:              aws.String("Spinifex-managed Lambda code-store role"),
+	}
+	for k, v := range tags {
+		in.Tags = append(in.Tags, &iam.Tag{Key: aws.String(k), Value: aws.String(v)})
+	}
+	_, err := r.iam.CreateRole(identifiers.GlobalAccountID, in)
+	if err != nil && err.Error() == awserrors.ErrorIAMEntityAlreadyExists {
+		return codestore.ErrExists
+	}
+	return err
+}
+
+func (r *codeStoreRole) Describe() (*codestore.RoleState, error) {
+	acct, name := identifiers.GlobalAccountID, aws.String(r.name)
+	got, err := r.iam.GetRole(acct, &iam.GetRoleInput{RoleName: name})
+	if err != nil {
+		if err.Error() == awserrors.ErrorIAMNoSuchEntity {
+			return nil, codestore.ErrNotFound
+		}
+		return nil, err
+	}
+	tags, err := r.iam.ListRoleTags(acct, &iam.ListRoleTagsInput{RoleName: name})
+	if err != nil {
+		return nil, err
+	}
+	inline, err := r.iam.ListRolePolicies(acct, &iam.ListRolePoliciesInput{RoleName: name})
+	if err != nil {
+		return nil, err
+	}
+	attached, err := r.iam.ListAttachedRolePolicies(acct, &iam.ListAttachedRolePoliciesInput{RoleName: name})
+	if err != nil {
+		return nil, err
+	}
+	if aws.BoolValue(tags.IsTruncated) || aws.BoolValue(inline.IsTruncated) || aws.BoolValue(attached.IsTruncated) {
+		return nil, errors.New("code-store role listing truncated")
+	}
+	st := &codestore.RoleState{
+		TrustPolicy:      aws.StringValue(got.Role.AssumeRolePolicyDocument),
+		Tags:             map[string]string{},
+		InlinePolicies:   aws.StringValueSlice(inline.PolicyNames),
+		AttachedPolicies: len(attached.AttachedPolicies),
+	}
+	for _, t := range tags.Tags {
+		st.Tags[aws.StringValue(t.Key)] = aws.StringValue(t.Value)
+	}
+	return st, nil
+}
+
+func (r *codeStoreRole) PutPolicy(name, doc string) error {
+	_, err := r.iam.PutRolePolicy(identifiers.GlobalAccountID, &iam.PutRolePolicyInput{
+		RoleName:       aws.String(r.name),
+		PolicyName:     aws.String(name),
+		PolicyDocument: aws.String(doc),
+	})
+	return err
+}
+
+// codeStoreBucket binds a system-root S3 client to the one code bucket. The
+// client stays in this adapter; the Lambda domain sees only Create and Tags.
+type codeStoreBucket struct {
+	s3   *s3.S3
+	name string
+}
+
+var _ codestore.BucketAdmin = (*codeStoreBucket)(nil)
+
+func (b *codeStoreBucket) Create(ctx context.Context, tags map[string]string) error {
+	_, err := b.s3.CreateBucketWithContext(ctx, &s3.CreateBucketInput{Bucket: aws.String(b.name)})
+	if err != nil {
+		if ae, ok := errors.AsType[awserr.Error](err); ok {
+			switch ae.Code() {
+			case s3.ErrCodeBucketAlreadyOwnedByYou:
+				return codestore.ErrExists
+			case s3.ErrCodeBucketAlreadyExists:
+				return codestore.ErrOwnedElsewhere
+			}
+		}
+		return err
+	}
+	set := &s3.Tagging{}
+	for k, v := range tags {
+		set.TagSet = append(set.TagSet, &s3.Tag{Key: aws.String(k), Value: aws.String(v)})
+	}
+	if _, err := b.s3.PutBucketTaggingWithContext(ctx, &s3.PutBucketTaggingInput{
+		Bucket: aws.String(b.name), Tagging: set,
+	}); err != nil {
+		return fmt.Errorf("tag new code bucket: %w", err)
+	}
+	return nil
+}
+
+func (b *codeStoreBucket) Tags(ctx context.Context) (map[string]string, error) {
+	out, err := b.s3.GetBucketTaggingWithContext(ctx, &s3.GetBucketTaggingInput{Bucket: aws.String(b.name)})
+	if err != nil {
+		if ae, ok := errors.AsType[awserr.Error](err); ok && ae.Code() == "NoSuchTagSet" {
+			return map[string]string{}, nil
+		}
+		return nil, err
+	}
+	tags := map[string]string{}
+	for _, t := range out.TagSet {
+		tags[aws.StringValue(t.Key)] = aws.StringValue(t.Value)
+	}
+	return tags, nil
+}
+
+// codeStoreAssumer is the in-process STS entry point the store's session uses.
+type codeStoreAssumer interface {
+	AssumeCodeStoreRole(region, sessionName string) (*sts.AssumeRoleOutput, error)
+}
+
+// codeStoreWiring is what the gateway composition root supplies.
+type codeStoreWiring struct {
+	Region        string
+	Node          string
+	Endpoint      string
+	SigningRegion string
+	RootAccessKey string
+	RootSecretKey string
+}
+
+// newLambdaCodeStore composes the code store. The root client and root keys stay
+// here; Lambda gets the bound admins for provisioning and a scoped session.
+func newLambdaCodeStore(w codeStoreWiring, roles codeStoreRoleIAM, assumer codeStoreAssumer) (*codestore.Manager, error) {
+	sess, err := session.NewSession(&aws.Config{
+		Endpoint:         aws.String(w.Endpoint),
+		Region:           aws.String(w.SigningRegion),
+		Credentials:      credentials.NewStaticCredentials(w.RootAccessKey, w.RootSecretKey, ""),
+		S3ForcePathStyle: aws.Bool(true),
+		HTTPClient:       &http.Client{Timeout: 30 * time.Second},
+	})
+	if err != nil {
+		return nil, fmt.Errorf("lambda code store admin session: %w", err)
+	}
+	role := &codeStoreRole{iam: roles, name: stsdomain.CodeStoreRoleName(w.Region)}
+	bucket := &codeStoreBucket{s3: s3.New(sess), name: codestore.BucketName(w.Region)}
+	sessionName := "awsgw-" + w.Node
+
+	provision := func(ctx context.Context) error {
+		return codestore.Provision(ctx, w.Region, role, bucket)
+	}
+	open := func() (*codestore.Store, error) {
+		return codestore.Open(codestore.OpenConfig{
+			Region:        w.Region,
+			Endpoint:      w.Endpoint,
+			SigningRegion: w.SigningRegion,
+			Assume:        func() (*sts.AssumeRoleOutput, error) { return assumer.AssumeCodeStoreRole(w.Region, sessionName) },
+		})
+	}
+	return codestore.NewManager(provision, open), nil
+}

spinifex/runtime/roles/awsgw/lambda_code_store_test.go

@@ -0,0 +1,195 @@
+package awsgw
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/aws/aws-sdk-go/service/s3"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
+	"github.com/mulgadc/spinifex/spinifex/domains/lambda/codestore"
+	stsdomain "github.com/mulgadc/spinifex/spinifex/domains/sts"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const codeStoreTestRegion = "ap-southeast-2"
+
+type nopJWKS struct{}
+
+func (nopJWKS) LookupClusterJWKS(context.Context, string, string) ([]byte, bool, error) {
+	return nil, false, nil
+}
+
+// fakeCodeBucket stands in for Predastore so the role path runs on real IAM.
+type fakeCodeBucket struct{ tags map[string]string }
+
+func (f *fakeCodeBucket) Create(_ context.Context, tags map[string]string) error {
+	if f.tags != nil {
+		return codestore.ErrExists
+	}
+	f.tags = tags
+	return nil
+}
+
+func (f *fakeCodeBucket) Tags(context.Context) (map[string]string, error) { return f.tags, nil }
+
+func newCodeStoreIAM(t *testing.T) (*iamdomain.IAMServiceImpl, *stsdomain.STSServiceImpl) {
+	t.Helper()
+	_, nc, js := testutil.StartTestJetStream(t)
+	key, err := iamdomain.GenerateMasterKey()
+	require.NoError(t, err)
+	iamSvc, err := iamdomain.NewIAMServiceImpl(t.Context(), nc, key)
+	require.NoError(t, err)
+	stsSvc, err := stsdomain.NewSTSServiceImpl(t.Context(), nc, iamSvc, key, nopJWKS{})
+	require.NoError(t, err)
+
+	kv, err := js.KeyValue(t.Context(), iamdomain.KVBucketAccounts)
+	require.NoError(t, err)
+	acct, err := json.Marshal(iamdomain.Account{AccountID: identifiers.GlobalAccountID, AccountName: "system", Status: iamdomain.AccountStatusActive})
+	require.NoError(t, err)
+	_, err = kv.Put(t.Context(), identifiers.GlobalAccountID, acct)
+	require.NoError(t, err)
+	return iamSvc, stsSvc
+}
+
+// Provisioning through the real IAM adapter yields a role the in-process STS
+// entry point can assume, so the role name, trust and tags agree end to end.
+func TestLambdaCodeStoreRole_ProvisionThenAssume(t *testing.T) {
+	iamSvc, stsSvc := newCodeStoreIAM(t)
+	role := &codeStoreRole{iam: iamSvc, name: stsdomain.CodeStoreRoleName(codeStoreTestRegion)}
+
+	require.NoError(t, codestore.Provision(t.Context(), codeStoreTestRegion, role, &fakeCodeBucket{}))
+
+	st, err := role.Describe()
+	require.NoError(t, err)
+	assert.Equal(t, codestore.OwnerTagValue, st.Tags[codestore.OwnerTagKey])
+	assert.Equal(t, []string{codestore.PolicyName}, st.InlinePolicies)
+	_, err = stsSvc.AssumeCodeStoreRole(codeStoreTestRegion, "awsgw-node1")
+	require.NoError(t, err)
+}
+
+func TestLambdaCodeStoreRole_RetriesOwnCrashAndRefusesUntagged(t *testing.T) {
+	iamSvc, _ := newCodeStoreIAM(t)
+	name := stsdomain.CodeStoreRoleName(codeStoreTestRegion)
+	role := &codeStoreRole{iam: iamSvc, name: name}
+
+	// Lambda's own crash after CreateRole: tagged role, no inline policy.
+	require.NoError(t, role.Create(codestore.TrustPolicy, map[string]string{codestore.OwnerTagKey: codestore.OwnerTagValue}))
+	require.NoError(t, codestore.Provision(t.Context(), codeStoreTestRegion, role, &fakeCodeBucket{}))
+
+	other := &codeStoreRole{iam: iamSvc, name: stsdomain.CodeStoreRoleName("us-east-1")}
+	_, err := iamSvc.CreateRole(identifiers.GlobalAccountID, &iam.CreateRoleInput{
+		RoleName: aws.String(other.name), AssumeRolePolicyDocument: aws.String(codestore.TrustPolicy)})
+	require.NoError(t, err)
+	err = codestore.Provision(t.Context(), "us-east-1", other, &fakeCodeBucket{})
+	require.ErrorIs(t, err, codestore.ErrNotOwned)
+	policies, err := iamSvc.ListRolePolicies(identifiers.GlobalAccountID, &iam.ListRolePoliciesInput{RoleName: aws.String(other.name)})
+	require.NoError(t, err)
+	assert.Empty(t, policies.PolicyNames, "no policy written to an untagged role")
+}
+
+// s3Responder answers the bucket admin's requests with fixed S3 replies.
+func s3Responder(t *testing.T, create, tagging int, createCode string) (*codeStoreBucket, *[]string) {
+	t.Helper()
+	var calls []string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = io.Copy(io.Discard, r.Body)
+		_, isTagging := r.URL.Query()["tagging"]
+		calls = append(calls, r.Method+" tagging="+strconv.FormatBool(isTagging))
+		switch {
+		case r.Method == http.MethodPut && !isTagging:
+			w.WriteHeader(create)
+			if createCode != "" {
+				fmt.Fprintf(w, "<Error><Code>%s</Code></Error>", createCode)
+			}
+		case r.Method == http.MethodGet && isTagging:
+			w.WriteHeader(tagging)
+			if tagging == http.StatusNotFound {
+				_, _ = w.Write([]byte("<Error><Code>NoSuchTagSet</Code></Error>"))
+				return
+			}
+			_, _ = w.Write([]byte(`<Tagging><TagSet><Tag><Key>spinifex:owner</Key><Value>lambda-code-store</Value></Tag></TagSet></Tagging>`))
+		default:
+			w.WriteHeader(tagging)
+		}
+	}))
+	t.Cleanup(srv.Close)
+	sess := session.Must(session.NewSession(&aws.Config{
+		Endpoint: aws.String(srv.URL), Region: aws.String(codeStoreTestRegion), S3ForcePathStyle: aws.Bool(true),
+		Credentials: credentials.NewStaticCredentials("AKIA", "secret", ""), MaxRetries: aws.Int(0),
+	}))
+	return &codeStoreBucket{s3: s3.New(sess), name: codestore.BucketName(codeStoreTestRegion)}, &calls
+}
+
+func TestLambdaCodeStoreBucket_MapsCreateOutcomes(t *testing.T) {
+	ctx := t.Context()
+
+	b, calls := s3Responder(t, http.StatusOK, http.StatusOK, "")
+	require.NoError(t, b.Create(ctx, map[string]string{"k": "v"}))
+	assert.Equal(t, []string{"PUT tagging=false", "PUT tagging=true"}, *calls, "a new bucket is tagged")
+
+	b, calls = s3Responder(t, http.StatusConflict, http.StatusOK, "BucketAlreadyOwnedByYou")
+	require.ErrorIs(t, b.Create(ctx, nil), codestore.ErrExists)
+	assert.Len(t, *calls, 1, "an existing bucket is never tagged")
+
+	b, _ = s3Responder(t, http.StatusConflict, http.StatusOK, "BucketAlreadyExists")
+	require.ErrorIs(t, b.Create(ctx, nil), codestore.ErrOwnedElsewhere)
+}
+
+func TestLambdaCodeStoreBucket_ReadsTags(t *testing.T) {
+	b, _ := s3Responder(t, http.StatusOK, http.StatusOK, "")
+	tags, err := b.Tags(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, codestore.OwnerTagValue, tags[codestore.OwnerTagKey])
+
+	b, _ = s3Responder(t, http.StatusOK, http.StatusNotFound, "")
+	tags, err = b.Tags(t.Context())
+	require.NoError(t, err)
+	assert.Empty(t, tags)
+}
+
+// Composing the store at gateway start makes no IAM or S3 call; the first use
+// provisions the role and bucket and opens a session through the real STS path.
+func TestNewLambdaCodeStore_ProvisionsOnlyOnFirstUse(t *testing.T) {
+	iamSvc, stsSvc := newCodeStoreIAM(t)
+	var s3Calls []string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, isTagging := r.URL.Query()["tagging"]
+		s3Calls = append(s3Calls, r.Method+" "+r.URL.Path+" tagging="+strconv.FormatBool(isTagging))
+		w.WriteHeader(http.StatusOK)
+	}))
+	t.Cleanup(srv.Close)
+
+	mgr, err := newLambdaCodeStore(codeStoreWiring{
+		Region: codeStoreTestRegion, Node: "node1", Endpoint: srv.URL, SigningRegion: codeStoreTestRegion,
+		RootAccessKey: "AKIAROOT", RootSecretKey: "root-secret",
+	}, iamSvc, stsSvc)
+	require.NoError(t, err)
+
+	roleName := aws.String(stsdomain.CodeStoreRoleName(codeStoreTestRegion))
+	_, err = iamSvc.GetRole(identifiers.GlobalAccountID, &iam.GetRoleInput{RoleName: roleName})
+	require.EqualError(t, err, "NoSuchEntity", "no role before first use")
+	assert.Empty(t, s3Calls, "no bucket request before first use")
+
+	_, err = mgr.Store(t.Context())
+	require.NoError(t, err)
+
+	_, err = iamSvc.GetRole(identifiers.GlobalAccountID, &iam.GetRoleInput{RoleName: roleName})
+	require.NoError(t, err)
+	assert.Equal(t, []string{
+		"PUT /lambda-code-ap-southeast-2 tagging=false",
+		"PUT /lambda-code-ap-southeast-2 tagging=true",
+	}, s3Calls)
+}

tests/integration/iam_groups_test.go

@@ -238,11 +238,16 @@ func TestIAMGroupsLifecycle(t *testing.T) {
 	_, err = iamCli.CreateUser(&iam.CreateUserInput{UserName: aws.String(grpLifecycleUser)})
 	require.NoError(t, err, "create-user")
 
-	// remove-user-from-group on a non-member surfaces NoSuchEntity (must not no-op).
+	// remove-user-from-group on a non-member of an existing group succeeds, as on AWS.
 	_, err = iamCli.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
 		GroupName: aws.String(grpLifecycleGroup),
 		UserName:  aws.String(grpLifecycleUser),
 	})
+	require.NoError(t, err, "remove-user-from-group non-member")
+	_, err = iamCli.RemoveUserFromGroup(&iam.RemoveUserFromGroupInput{
+		GroupName: aws.String("ghost-group"),
+		UserName:  aws.String(grpLifecycleUser),
+	})
 	requireAWSErrorCode(t, err, "NoSuchEntity")
 
 	// Ghost-entity binds → NoSuchEntity. Group is validated before the user.

tests/integration/iam_roles_test.go

@@ -115,7 +115,7 @@ func TestIAMRolesAndProfiles(t *testing.T) {
 	})
 	require.NoError(t, err, "update-role-description")
 	require.Equal(t, "described", aws.StringValue(descOut.Role.Description))
-	require.Equal(t, int64(7200), aws.Int64Value(descOut.Role.MaxSessionDuration))
+	require.Nil(t, descOut.Role.MaxSessionDuration, "AWS omits MaxSessionDuration from update-role-description")
 	got, err = iamCli.GetRole(&iam.GetRoleInput{RoleName: aws.String(iamRoleAppName)})
 	require.NoError(t, err, "get-role after update-role-description")
 	require.Equal(t, "described", aws.StringValue(got.Role.Description))