fix: send ALB health checks as HTTP/1.1 with Host and ELB-HealthChecker User-Agent (#1296)

HAProxy's short 'option httpchk GET <path>' form sends HTTP/1.0 with no Host header, which Envoy rejects with 426 (and 400 for HTTP/1.1 without Host), so Envoy-fronted targets never became healthy. Match the AWS request shape: GET <path> HTTP/1.1, Host set to the target IP (with the port only when it is not the scheme default), and User-Agent ELB-HealthChecker/2.0.

go.mod

@@ -4,12 +4,15 @@ go 1.27.3
 
 require (
 	github.com/aws/aws-sdk-go v1.55.8
-	github.com/aws/aws-sdk-go-v2 v1.47.3
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.23
+	github.com/aws/aws-sdk-go-v2 v1.47.4
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.24
 	github.com/aws/aws-sdk-go-v2/config v1.33.9
 	github.com/aws/aws-sdk-go-v2/credentials v1.20.9
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3
+	github.com/aws/aws-sdk-go-v2/service/lambda v1.113.3
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.114.4
+	github.com/aws/aws-sdk-go-v2/service/sqs v1.52.6
+	github.com/aws/smithy-go v1.28.6
 	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/charmbracelet/bubbles v1.0.0
 	github.com/charmbracelet/bubbletea v1.3.10
@@ -64,8 +67,8 @@ require (
 	github.com/Microsoft/hcsshim v0.15.0-rc.4 // indirect
 	github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect
 	github.com/atotto/clipboard v0.1.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.7 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.7 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 // indirect
@@ -75,7 +78,6 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 // indirect
-	github.com/aws/smithy-go v1.28.6 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect

go.sum

@@ -22,20 +22,20 @@ github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z
 github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk=
-github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.23 h1:/fGPpKrXTjzkhsDXExBgNT2cGwxiF9rLrQXh8RlXMhI=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.23/go.mod h1:QWUDuRi9PFjBHIU2iH1Pkldbe7zqLfXJC4wWLbF8nuQ=
+github.com/aws/aws-sdk-go-v2 v1.47.4 h1:bTvk6UdSSSvaQsfLVj1JRDEEqj/PDRwGf1KanaRzau0=
+github.com/aws/aws-sdk-go-v2 v1.47.4/go.mod h1:XkrjAZvr705Nytv6PkPUhoSB2W9+vVHBp5oAWpkh8Ak=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.24 h1:t7FxqJ5KHDeKceyD4HtGaG6z7hMSeVGjs5/4c+L5ArM=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.24/go.mod h1:5XK6NRVlK8s8yj4zKcM43HLluGA/rAz+azLLvWDMRr0=
 github.com/aws/aws-sdk-go-v2/config v1.33.9 h1:Hv6bcQUJVnmuWaKjqD6Ot8sE75NrWsWV20oGT5aWle0=
 github.com/aws/aws-sdk-go-v2/config v1.33.9/go.mod h1:ivowmrZNfBQLcg4QkaItgi5wae4Bzot9RzIdbY1hfTg=
 github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0=
 github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 h1:CqL8NN3p4I0JTwwV5MEoatIl4Y2amZ2X9dRD+0hbRL0=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6/go.mod h1:exdJ32hMqXl0Wyo6NwqEC5yIhtua3+PDaKE8P29m4qE=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 h1:FpBq24TG6WVSObTNhQOv7Q0088rounUUlwn3zlNItkM=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6/go.mod h1:ONzdPVYC95oBOu0VWNEsKEntcoemP3xVKjT+qWml1s4=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.7 h1:fSNAt6hs4qLjb9tMZc6vj3PHlOCYGXK2JjJZ7AW2tf4=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.7/go.mod h1:TMav+EuXQu5kNpZZz/6rIC8xx4mCsW+6AKHrjvOSAsE=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.7 h1:7NS6kqBjkWIN8P/AEdmtyq6eQGAhN1oHTZdzXclpH6Y=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.7/go.mod h1:1PFqtAUsIgF1nEVojzRJSf/v88zjzqrGW+w+UAtkXyQ=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3VvGMOdbGemtUhX4=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 h1:e6YW5uME30TkrElR+4/xcY7NB6CCj2dcv7vtmGXhfhQ=
@@ -46,10 +46,14 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 h1:LHHdobjo5
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6/go.mod h1:CBGB0evAiaMD3Lk4wPO7LQCcCZLuhzL5Qf/wfEUHQ5w=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 h1:mYViDr2azcgxOpRrObZOl631bvavqILoE/L+Kxgbwys=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6/go.mod h1:OXVdAo9jNhggB+hJRGOyy7WolGTKB6Rg7IIulDeSzPM=
+github.com/aws/aws-sdk-go-v2/service/lambda v1.113.3 h1:oSHldse7GDHpdq81XeAJnhIbsdDKP9bMIt00G8W6uXI=
+github.com/aws/aws-sdk-go-v2/service/lambda v1.113.3/go.mod h1:Y//K046w4uhiR3o77beo/y52Jv2uEY13iJ+3+jicLO8=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.114.4 h1:MrkWRB58Qw5TEkb51QVooVLjlM3IevO2iVXEz74B8pg=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.114.4/go.mod h1:yfIxaKRHasvfy6tNEm3/LpqKsupMzQjMaWj9Y0jMVbo=
 github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A=
 github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M=
+github.com/aws/aws-sdk-go-v2/service/sqs v1.52.6 h1:Suq1LNhMBpsmShEv3vO+J3H8HEeK+GwYMJDYLE4IJSs=
+github.com/aws/aws-sdk-go-v2/service/sqs v1.52.6/go.mod h1:rHGuzFg4AfVnWZa3/533tn5GQd1LzVkTXI9WMT8LqAc=
 github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk=
 github.com/aws/aws-sdk-go-v2/service/sso v1.38.4/go.mod h1:+FEO9jrVMpc9oN3CdAxJbtlQEu3z8UddDPSedpeM3PY=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 h1:j6c+HA4AoS0EBEjArPjavFEBrucJEryfrQU0TffmACk=

spinifex/domains/ec2/image/deregister_legacy_config_test.go

@@ -12,21 +12,6 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-// strictDeleteStore answers a delete of an absent key with NoSuchKey, which is
-// what Predastore does. MemoryObjectStore silently succeeds instead, so a
-// handler that treats a missing key as a failure passes every test and still
-// breaks in production.
-type strictDeleteStore struct {
-	*objectstore.MemoryObjectStore
-}
-
-func (s strictDeleteStore) DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput) (*awss3.DeleteObjectOutput, error) {
-	if _, err := s.HeadObject(ctx, &awss3.HeadObjectInput{Bucket: input.Bucket, Key: input.Key}); err != nil {
-		return nil, err
-	}
-	return s.MemoryObjectStore.DeleteObject(ctx, input)
-}
-
 // TestDeregisterImage_NoLegacyConfig covers every AMI created since the
 // metadata store landed: they have no {imageID}/config.json, so deleting one
 // answers NoSuchKey. Treating that as a failure returned InternalError after
@@ -34,7 +19,7 @@ func (s strictDeleteStore) DeleteObject(ctx context.Context, input *awss3.Delete
 // InvalidAMIID.NotFound for an image it had just been told it could not delete.
 func TestDeregisterImage_NoLegacyConfig(t *testing.T) {
 	mem := objectstore.NewMemoryObjectStore()
-	svc := NewImageServiceImplWithStore(strictDeleteStore{MemoryObjectStore: mem}, testBucket)
+	svc := NewImageServiceImplWithStore(mem, testBucket)
 
 	amiID := "ami-nolegacy001"
 	createTestAMIConfigWithOwner(t, mem, amiID, "no-legacy-config", testAccountID)
@@ -50,3 +35,27 @@ func TestDeregisterImage_NoLegacyConfig(t *testing.T) {
 	require.Error(t, getErr)
 	assert.True(t, objectstore.IsNoSuchKeyError(getErr), "the AMI document must still be deleted")
 }
+
+// racingDeleteStore lets a concurrent deleter win every DeleteObject, so the
+// caller's own delete answers NoSuchKey as Predastore would.
+type racingDeleteStore struct {
+	*objectstore.MemoryObjectStore
+}
+
+func (s racingDeleteStore) DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput) (*awss3.DeleteObjectOutput, error) {
+	_, _ = s.MemoryObjectStore.DeleteObject(ctx, input)
+	return s.MemoryObjectStore.DeleteObject(ctx, input)
+}
+
+// Two deregisters that both pass the ownership read race to delete the
+// document; the loser's NoSuchKey means the image is already gone.
+func TestDeregisterImage_ConcurrentlyDeletedDocumentIsSuccess(t *testing.T) {
+	mem := objectstore.NewMemoryObjectStore()
+	svc := NewImageServiceImplWithStore(racingDeleteStore{MemoryObjectStore: mem}, testBucket)
+
+	amiID := "ami-racing00001"
+	createTestAMIConfigWithOwner(t, mem, amiID, "racing-deregister", testAccountID)
+
+	_, err := svc.DeregisterImage(context.Background(), &ec2.DeregisterImageInput{ImageId: aws.String(amiID)}, testAccountID)
+	require.NoError(t, err)
+}

spinifex/domains/ec2/image/service_impl.go

@@ -1161,7 +1161,7 @@ func (s *ImageServiceImpl) DeregisterImage(ctx context.Context, input *ec2.Dereg
 		return nil, err
 	}
 
-	if err := s.metadata.DeleteAMI(ctx, imageID); err != nil {
+	if err := s.metadata.DeleteAMI(ctx, imageID); err != nil && !objectstore.IsNoSuchKeyError(err) {
 		slog.ErrorContext(ctx, "DeregisterImage: failed to delete AMI document", "imageId", imageID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}

spinifex/domains/ec2/snapshot/service_impl.go

@@ -793,7 +793,7 @@ func (s *SnapshotServiceImpl) DeleteSnapshot(ctx context.Context, input *ec2.Del
 			Bucket: aws.String(s.config.Predastore.Bucket),
 			Key:    obj.Key,
 		})
-		if err != nil {
+		if err != nil && !objectstore.IsNoSuchKeyError(err) {
 			slog.ErrorContext(ctx, "DeleteSnapshot failed to delete object", "key", *obj.Key, "err", err)
 			sweepErr = errors.Join(sweepErr, err)
 		}
@@ -808,7 +808,7 @@ func (s *SnapshotServiceImpl) DeleteSnapshot(ctx context.Context, input *ec2.Del
 	// explicitly, and last: a crash with it still present leaves a snapshot that
 	// DescribeSnapshots shows and DeleteSnapshot can retry, rather than chunks
 	// nothing enumerates.
-	if err := s.metadata.DeleteSnapshot(ctx, accountID, snapshotID); err != nil {
+	if err := s.metadata.DeleteSnapshot(ctx, accountID, snapshotID); err != nil && !objectstore.IsNoSuchKeyError(err) {
 		slog.ErrorContext(ctx, "DeleteSnapshot failed to delete metadata document", "snapshotId", snapshotID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}

spinifex/domains/ec2/snapshot/snapshot_partition_test.go

@@ -155,6 +155,48 @@ func (s *deleteFailingStore) DeleteObject(ctx context.Context, input *s3.DeleteO
 	return s.MemoryObjectStore.DeleteObject(ctx, input)
 }
 
+// racingDeleteStore lets a concurrent deleter win every DeleteObject once race
+// is set, so the caller's own delete answers NoSuchKey as Predastore would.
+type racingDeleteStore struct {
+	*objectstore.MemoryObjectStore
+
+	race bool
+}
+
+func (s *racingDeleteStore) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+	if s.race {
+		_, _ = s.MemoryObjectStore.DeleteObject(ctx, input)
+	}
+	return s.MemoryObjectStore.DeleteObject(ctx, input)
+}
+
+// A chunk or document another deleter already removed is deleted, not a
+// failure: the sweep and the document delete must both finish.
+func TestDeleteSnapshot_ConcurrentlyDeletedKeysAreDone(t *testing.T) {
+	ctx := context.Background()
+	objects := &racingDeleteStore{MemoryObjectStore: objectstore.NewMemoryObjectStore()}
+	cfg := &config.Config{Predastore: config.PredastoreConfig{Bucket: "test-bucket"}}
+	svc := NewSnapshotServiceImplWithStore(cfg, objects, nil)
+	svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}))
+	createTestVolume(t, svc, objects.MemoryObjectStore, "vol-race", 8)
+
+	snap, err := svc.CreateSnapshot(ctx, &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-race")}, testAccountID)
+	require.NoError(t, err)
+	snapshotID := aws.StringValue(snap.SnapshotId)
+	_, err = objects.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String("test-bucket"), Key: aws.String(snapshotID + "/chunk-0"),
+		Body: strings.NewReader("blocks"),
+	})
+	require.NoError(t, err)
+	objects.race = true
+
+	_, err = svc.DeleteSnapshot(ctx, &ec2.DeleteSnapshotInput{SnapshotId: snap.SnapshotId}, testAccountID)
+	require.NoError(t, err)
+
+	_, err = svc.metadata.GetSnapshot(ctx, testAccountID, snapshotID)
+	assert.True(t, objectstore.IsNoSuchKeyError(err), "the document must be gone, got %v", err)
+}
+
 // Isolation must be enforced *by* the read, not by a filter after it. A
 // whole-tree listing plus a correct in-memory filter would return the same
 // snapshots, so this asserts the access pattern: the listing prefix is the

spinifex/domains/ec2/tags/service_impl.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"log/slog"
 	"maps"
+	"regexp"
 	"slices"
 	"strings"
 
@@ -89,6 +90,29 @@ func describeTagsResourceType(resourceID string) string {
 	return string(kind)
 }
 
+// Taggable resources whose ids arn.EC2TypeForID does not type, because their
+// ARN type is named by a parameter rather than derived from the id.
+var untypedTaggablePrefixes = []string{"lt-", "cr-", "sir-"}
+
+// A recognised prefix followed by characters that cannot break the tag store's
+// KV key, which a wildcard, space or dot in the id would.
+var taggableIDShape = regexp.MustCompile(`^[a-z]+-[A-Za-z0-9_-]+$`)
+
+// isTaggableID reports whether CreateTags accepts resourceID. Only new writes
+// are checked: entries already stored under an untypable id are left in place
+// and stay visible to DescribeTags and removable by DeleteTags.
+func isTaggableID(resourceID string) bool {
+	if !taggableIDShape.MatchString(resourceID) {
+		return false
+	}
+	if _, ok := arn.EC2TypeForID(resourceID); ok {
+		return true
+	}
+	return slices.ContainsFunc(untypedTaggablePrefixes, func(p string) bool {
+		return strings.HasPrefix(resourceID, p)
+	})
+}
+
 // getTagsKey returns the S3 key for storing tags for a resource, scoped by account.
 func getTagsKey(accountID, resourceID string) string {
 	return "tags/" + accountID + "/" + resourceID + ".json"
@@ -207,6 +231,13 @@ func (s *TagsServiceImpl) CreateTags(ctx context.Context, input *ec2.CreateTagsI
 
 	slog.InfoContext(ctx, "CreateTags request", "resources", len(input.Resources), "tags", len(input.Tags))
 
+	for _, resourceID := range input.Resources {
+		if resourceID != nil && !isTaggableID(*resourceID) {
+			slog.InfoContext(ctx, "CreateTags rejected unrecognised resource id", "resourceId", *resourceID)
+			return nil, errors.New(awserrors.ErrorInvalidID)
+		}
+	}
+
 	for _, resourceID := range input.Resources {
 		if resourceID == nil {
 			continue

spinifex/domains/ec2/tags/service_impl_test.go

@@ -140,6 +140,63 @@ func TestCreateTags_MissingResources(t *testing.T) {
 	assert.Contains(t, err.Error(), awserrors.ErrorMissingParameter)
 }
 
+// An id with no recognised prefix is rejected before any resource in the
+// request is written, so a valid id alongside it is not tagged either.
+func TestCreateTags_UnrecognisedPrefixIsInvalidID(t *testing.T) {
+	svc, _ := setupTestTagsService(t)
+
+	for _, id := range []string{"not-a-real-id", "", "vpc", "vpc-", "vpc-*", "vpc-a b", "vpc->", "vpc-a.b"} {
+		_, err := svc.CreateTags(context.Background(), &ec2.CreateTagsInput{
+			Resources: []*string{aws.String("vpc-0abc"), aws.String(id)},
+			Tags:      []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("x")}},
+		}, testAccountID)
+		require.Error(t, err, id)
+		assert.Equal(t, awserrors.ErrorInvalidID, err.Error(), id)
+	}
+
+	out, err := svc.DescribeTags(context.Background(), &ec2.DescribeTagsInput{}, testAccountID)
+	require.NoError(t, err)
+	assert.Empty(t, out.Tags)
+}
+
+// Launch templates, capacity reservations and spot requests have no ARN-type
+// prefix mapping but are still taggable resources. A nil entry is skipped.
+func TestCreateTags_AcceptsTaggablePrefixesWithoutARNType(t *testing.T) {
+	svc, _ := setupTestTagsService(t)
+
+	ids := []string{"lt-0abc", "cr-0abc", "sir-0abc", "eipalloc-0abc", "pg-0abc"}
+	resources := []*string{nil}
+	for _, id := range ids {
+		resources = append(resources, aws.String(id))
+	}
+	_, err := svc.CreateTags(context.Background(), &ec2.CreateTagsInput{
+		Resources: resources,
+		Tags:      []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("x")}},
+	}, testAccountID)
+	require.NoError(t, err)
+
+	out, err := svc.DescribeTags(context.Background(), &ec2.DescribeTagsInput{}, testAccountID)
+	require.NoError(t, err)
+	assert.Len(t, out.Tags, len(ids))
+}
+
+// Tags stored for an untypable id before validation existed stay removable.
+func TestDeleteTags_UntypableIDStillDeletable(t *testing.T) {
+	svc, _ := setupTestTagsService(t)
+	require.NoError(t, svc.mutateTags(context.Background(), testAccountID, "not-a-real-id", func(tags map[string]string) {
+		tags["Name"] = "residue"
+	}))
+
+	_, err := svc.DeleteTags(context.Background(), &ec2.DeleteTagsInput{
+		Resources: []*string{aws.String("not-a-real-id")},
+	}, testAccountID)
+	require.NoError(t, err)
+
+	out, err := svc.DescribeTags(context.Background(), &ec2.DescribeTagsInput{}, testAccountID)
+	require.NoError(t, err)
+	assert.Empty(t, out.Tags)
+}
+
 // TestCreateTags_MissingTags tests creating tags without tags.
 func TestCreateTags_MissingTags(t *testing.T) {
 	svc, _ := setupTestTagsService(t)

spinifex/domains/ec2/volume/delete_volume_unreadable_test.go

@@ -92,6 +92,42 @@ func TestDeleteVolume_UnreadableDocumentIsInternalError(t *testing.T) {
 	require.NotNil(t, provVol, "the provider's data must survive a failed delete")
 }
 
+// racingDeleteStore lets a concurrent deleter win every DeleteObject, so the
+// caller's own delete answers NoSuchKey as Predastore would.
+type racingDeleteStore struct {
+	*objectstore.MemoryObjectStore
+}
+
+func (s *racingDeleteStore) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+	_, _ = s.MemoryObjectStore.DeleteObject(ctx, input)
+	return s.MemoryObjectStore.DeleteObject(ctx, input)
+}
+
+// A document another delete removed after this one read it is deleted, so the
+// call must succeed rather than answer InternalError for a finished delete.
+func TestDeleteVolume_ConcurrentlyDeletedDocumentIsSuccess(t *testing.T) {
+	memory := objectstore.NewMemoryObjectStore()
+	svc := newTestVolumeServiceWithStore("ap-southeast-2a", memory)
+	svc.snapshotKV = setupTestVolumeKV(t)
+
+	volumeID := "vol-racingdelete00"
+	createVolumeInStoreWithMeta(t, svc, memory, volumeID, ebsmetadata.Volume{
+		VolumeID: volumeID, CapacityGiB: 10, State: "available",
+	})
+
+	racing := &racingDeleteStore{MemoryObjectStore: memory}
+	svc.store = racing
+	svc.metadata = ebsmetadata.NewStore(racing, "test-bucket")
+
+	_, err := svc.DeleteVolume(context.Background(), &ec2.DeleteVolumeInput{
+		VolumeId: aws.String(volumeID),
+	}, testVolAccountID)
+	require.NoError(t, err)
+
+	_, getErr := ebsmetadata.NewStore(memory, "test-bucket").GetVolume(context.Background(), testVolAccountID, volumeID)
+	assert.True(t, objectstore.IsNoSuchKeyError(getErr), "the document must be gone, got %v", getErr)
+}
+
 // TestDeleteVolume_AbsentDocumentIsStillNotFound guards the other side of the
 // fix: absence (NoSuchKey) must keep returning InvalidVolume.NotFound, not be
 // swept up into the new InternalError branch.

spinifex/domains/ec2/volume/service_impl.go

@@ -1135,7 +1135,7 @@ func (s *VolumeServiceImpl) DeleteVolume(ctx context.Context, input *ec2.DeleteV
 		// it gets back, so a bare code makes a persistent fault unreadable.
 		return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "delete volume %s: %v", volumeID, err)
 	}
-	if err := s.metadata.DeleteVolume(ctx, accountID, volumeID); err != nil {
+	if err := s.metadata.DeleteVolume(ctx, accountID, volumeID); err != nil && !objectstore.IsNoSuchKeyError(err) {
 		return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "delete volume metadata %s: %v", volumeID, err)
 	}
 

spinifex/domains/ecr/registry/registry.go

@@ -853,7 +853,10 @@ func (reg *Registry) deleteUploadBytesAt(ctx context.Context, key string) error
 		Bucket: aws.String(reg.bucket()),
 		Key:    aws.String(key),
 	})
-	return err
+	if err != nil && !objectstore.IsNoSuchKeyError(err) {
+		return err
+	}
+	return nil
 }
 
 func (reg *Registry) writeJSON(ctx context.Context, w http.ResponseWriter, status int, v any) {

spinifex/domains/ecr/registry/registry_image.go

@@ -273,7 +273,7 @@ func (reg *Registry) reclaimManifest(ctx context.Context, account, repo, digest
 	if _, err := reg.Store.DeleteObject(ctx, &s3.DeleteObjectInput{
 		Bucket: aws.String(ecr.AccountBucket(account)),
 		Key:    aws.String(ecr.ManifestKey(repo, digest)),
-	}); err != nil {
+	}); err != nil && !objectstore.IsNoSuchKeyError(err) {
 		slog.WarnContext(ctx, "ECR/GC: manifest object delete failed", "repo", repo, "digest", digest, "err", err)
 	}
 
@@ -292,7 +292,7 @@ func (reg *Registry) reclaimManifest(ctx context.Context, account, repo, digest
 		if _, err := reg.Store.DeleteObject(ctx, &s3.DeleteObjectInput{
 			Bucket: aws.String(ecr.AccountBucket(account)),
 			Key:    aws.String(ecr.BlobKey(child)),
-		}); err != nil {
+		}); err != nil && !objectstore.IsNoSuchKeyError(err) {
 			slog.WarnContext(ctx, "ECR/GC: blob delete failed", "digest", child, "err", err)
 		}
 	}

spinifex/domains/ecs/conv.go

@@ -91,13 +91,14 @@ func containerDefsFromAWS(in []*ecs.ContainerDefinition) []taskdefinition.Contai
 		if c == nil {
 			continue
 		}
+		// AWS assumes a container is essential when the parameter is omitted.
 		def := taskdefinition.Container{
 			Name:      aws.StringValue(c.Name),
 			Image:     aws.StringValue(c.Image),
 			CPU:       int(aws.Int64Value(c.Cpu)),
 			MemoryMiB: int(aws.Int64Value(c.Memory)),
 			GPU:       gpuCountFromResourceRequirements(c.ResourceRequirements),
-			Essential: aws.BoolValue(c.Essential),
+			Essential: c.Essential == nil || *c.Essential,
 			Command:   awsStringSlice(c.Command),
 		}
 		if c.Environment != nil {

spinifex/domains/ecs/conv_test.go

@@ -163,3 +163,20 @@ func TestTaskDefReservedGPU(t *testing.T) {
 	empty := &taskdefinition.Record{Containers: []taskdefinition.Container{{CPU: 128}}}
 	assert.Zero(t, empty.ReservedGPU())
 }
+
+// AWS assumes a container is essential when the essential parameter is omitted,
+// so an omitted value must describe back as true while explicit values are kept.
+func TestContainerDefsFromAWS_EssentialDefaultsTrue(t *testing.T) {
+	defs := containerDefsFromAWS([]*ecs.ContainerDefinition{
+		{Name: aws.String("omitted"), Image: aws.String("registry/app:1")},
+		{Name: aws.String("sidecar"), Image: aws.String("registry/side:1"), Essential: aws.Bool(false)},
+		{Name: aws.String("explicit"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true)},
+	})
+	require.Len(t, defs, 3)
+	assert.True(t, defs[0].Essential)
+	assert.True(t, aws.BoolValue(containerToAWS(defs[0]).Essential))
+	assert.True(t, containerToAssign(defs[0]).Essential)
+	assert.False(t, defs[1].Essential)
+	assert.False(t, aws.BoolValue(containerToAWS(defs[1]).Essential))
+	assert.True(t, defs[2].Essential)
+}

spinifex/domains/ecs/records.go

@@ -153,6 +153,13 @@ type ClusterRecord struct {
 	// an ASG primitive).
 	CapacityProviders               []string                       `json:"capacityProviders,omitempty"`
 	DefaultCapacityProviderStrategy []CapacityProviderStrategyItem `json:"defaultCapacityProviderStrategy,omitempty"`
+	Settings                        []ClusterSetting               `json:"settings,omitempty"`
+}
+
+// ClusterSetting is one CreateCluster setting, stored and described back.
+type ClusterSetting struct {
+	Name  string `json:"name"`
+	Value string `json:"value"`
 }
 
 // CapacityProviderStrategyItem is one entry of a capacity-provider strategy:

spinifex/domains/ecs/service.go

@@ -258,6 +258,11 @@ func (s *Service) CreateCluster(ctx context.Context, input *ecs.CreateClusterInp
 			Status:    ClusterStatusActive,
 			CreatedAt: time.Now().UTC(),
 		}
+		settings, err := clusterSettingsFromAWS(input.Settings)
+		if err != nil {
+			return nil, err
+		}
+		rec.Settings = settings
 		if len(input.Tags) > 0 {
 			rec.Tags = map[string]string{}
 			for _, t := range input.Tags {
@@ -271,8 +276,8 @@ func (s *Service) CreateCluster(ctx context.Context, input *ecs.CreateClusterInp
 	return &ecs.CreateClusterOutput{Cluster: rec.toAWS()}, nil
 }
 
-// DescribeClusters returns meta for the named clusters; unknown names are
-// silently skipped (AWS returns them under "failures", omitted here in v1).
+// DescribeClusters returns meta for the named clusters; unknown names surface
+// as MISSING failures. Settings are only returned under Include=SETTINGS.
 func (s *Service) DescribeClusters(ctx context.Context, input *ecs.DescribeClustersInput, accountID string) (*ecs.DescribeClustersOutput, error) {
 	kv, err := s.bucket(ctx, accountID)
 	if err != nil {
@@ -282,6 +287,7 @@ func (s *Service) DescribeClusters(ctx context.Context, input *ecs.DescribeClust
 	if len(names) == 0 {
 		names = []string{defaultCluster}
 	}
+	includeSettings := slices.Contains(awsStringSlice(input.Include), ecs.ClusterFieldSettings)
 	out := &ecs.DescribeClustersOutput{}
 	for _, name := range names {
 		name = ClusterShortName(name)
@@ -291,9 +297,13 @@ func (s *Service) DescribeClusters(ctx context.Context, input *ecs.DescribeClust
 			return nil, err
 		}
 		if !found {
+			out.Failures = append(out.Failures, &ecs.Failure{Arn: aws.String(ClusterARN(s.region, accountID, name)), Reason: aws.String("MISSING")})
 			continue
 		}
 		c := rec.toAWS()
+		if !includeSettings {
+			c.Settings = nil
+		}
 		if err := s.applyClusterCounts(ctx, kv, name, c); err != nil {
 			return nil, err
 		}
@@ -382,9 +392,31 @@ func (r *ClusterRecord) toAWS() *ecs.Cluster {
 	for _, item := range r.DefaultCapacityProviderStrategy {
 		c.DefaultCapacityProviderStrategy = append(c.DefaultCapacityProviderStrategy, item.toAWS())
 	}
+	for _, st := range r.Settings {
+		c.Settings = append(c.Settings, &ecs.ClusterSetting{Name: aws.String(st.Name), Value: aws.String(st.Value)})
+	}
 	return c
 }
 
+// clusterSettingsFromAWS validates CreateCluster settings against the names
+// and values AWS accepts, so an unsupported setting is refused, not dropped.
+func clusterSettingsFromAWS(in []*ecs.ClusterSetting) ([]ClusterSetting, error) {
+	var out []ClusterSetting
+	for _, st := range in {
+		if st == nil {
+			continue
+		}
+		name, value := aws.StringValue(st.Name), aws.StringValue(st.Value)
+		if name != ecs.ClusterSettingNameContainerInsights || !slices.Contains(containerInsightsValues, value) {
+			return nil, errors.New(awserrors.ErrorECSInvalidParameter)
+		}
+		out = append(out, ClusterSetting{Name: name, Value: value})
+	}
+	return out, nil
+}
+
+var containerInsightsValues = []string{"enabled", "disabled", "enhanced"}
+
 // tagsToAWS converts a stored tag map into the AWS list form with a stable
 // key order so Describe output is deterministic.
 func tagsToAWS(tags map[string]string) []*ecs.Tag {
@@ -548,7 +580,16 @@ func (s *Service) DescribeTaskDefinition(ctx context.Context, input *ecs.Describ
 	if err != nil {
 		return nil, err
 	}
-	rec, err := s.taskDefs.Resolve(ctx, kv, aws.StringValue(input.TaskDefinition))
+	ref := aws.StringValue(input.TaskDefinition)
+	if family, _ := taskdefinition.ParseRef(ref); family == "" {
+		return nil, errors.New(awserrors.ErrorECSInvalidParameter)
+	}
+	// AWS answers an unknown revision with ClientException, not the
+	// InvalidParameterException that launches get for the same reference.
+	rec, err := s.taskDefs.Resolve(ctx, kv, ref)
+	if err != nil && err.Error() == awserrors.ErrorECSInvalidParameter {
+		return nil, errors.New(awserrors.ErrorECSClientException)
+	}
 	if err != nil {
 		return nil, err
 	}

spinifex/domains/ecs/service_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ecs"
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
 	"github.com/mulgadc/spinifex/internal/testkit"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -61,6 +62,48 @@ func TestService_DescribeClusters_DefaultAndMissing(t *testing.T) {
 	miss, err := svc.DescribeClusters(context.Background(), &ecs.DescribeClustersInput{Clusters: []*string{aws.String("nope")}}, testAccountID)
 	require.NoError(t, err)
 	assert.Empty(t, miss.Clusters)
+	require.Len(t, miss.Failures, 1)
+	assert.Equal(t, ClusterARN(testRegion, testAccountID, "nope"), aws.StringValue(miss.Failures[0].Arn))
+	assert.Equal(t, "MISSING", aws.StringValue(miss.Failures[0].Reason))
+}
+
+func TestService_CreateCluster_StoresSettings(t *testing.T) {
+	svc, _ := newTestService(t)
+	setting := &ecs.ClusterSetting{Name: aws.String(ecs.ClusterSettingNameContainerInsights), Value: aws.String("enabled")}
+	created, err := svc.CreateCluster(context.Background(), &ecs.CreateClusterInput{
+		ClusterName: aws.String("web"), Settings: []*ecs.ClusterSetting{setting},
+	}, testAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, []*ecs.ClusterSetting{setting}, created.Cluster.Settings)
+
+	withSettings, err := svc.DescribeClusters(context.Background(), &ecs.DescribeClustersInput{
+		Clusters: []*string{aws.String("web")}, Include: []*string{aws.String(ecs.ClusterFieldSettings)},
+	}, testAccountID)
+	require.NoError(t, err)
+	require.Len(t, withSettings.Clusters, 1)
+	assert.Equal(t, []*ecs.ClusterSetting{setting}, withSettings.Clusters[0].Settings)
+
+	plain, err := svc.DescribeClusters(context.Background(), &ecs.DescribeClustersInput{Clusters: []*string{aws.String("web")}}, testAccountID)
+	require.NoError(t, err)
+	require.Len(t, plain.Clusters, 1)
+	assert.Empty(t, plain.Clusters[0].Settings)
+}
+
+func TestService_CreateCluster_RejectsUnsupportedSetting(t *testing.T) {
+	svc, _ := newTestService(t)
+	for _, st := range []*ecs.ClusterSetting{
+		{Name: aws.String("unknownSetting"), Value: aws.String("enabled")},
+		{Name: aws.String(ecs.ClusterSettingNameContainerInsights), Value: aws.String("sometimes")},
+	} {
+		_, err := svc.CreateCluster(context.Background(), &ecs.CreateClusterInput{
+			ClusterName: aws.String("web"), Settings: []*ecs.ClusterSetting{st},
+		}, testAccountID)
+		require.Error(t, err)
+		assert.Equal(t, awserrors.ErrorECSInvalidParameter, err.Error())
+	}
+	list, err := svc.ListClusters(context.Background(), &ecs.ListClustersInput{}, testAccountID)
+	require.NoError(t, err)
+	assert.Empty(t, list.ClusterArns)
 }
 
 func TestService_DescribeClusters_ReturnsTags(t *testing.T) {

spinifex/domains/ecs/taskdef_lifecycle_test.go

@@ -71,11 +71,15 @@ func TestDescribeTaskDefinition_ResolvesEveryReferenceForm(t *testing.T) {
 		assert.Equal(t, want, aws.Int64Value(out.TaskDefinition.Revision), ref)
 	}
 
-	for _, ref := range []string{"app:3", "ghost", ""} {
+	for ref, want := range map[string]string{
+		"app:3": awserrors.ErrorECSClientException,
+		"ghost": awserrors.ErrorECSClientException,
+		"":      awserrors.ErrorECSInvalidParameter,
+	} {
 		_, err := svc.DescribeTaskDefinition(context.Background(),
 			&ecs.DescribeTaskDefinitionInput{TaskDefinition: aws.String(ref)}, testAccountID)
 		require.Error(t, err, ref)
-		assert.Equal(t, awserrors.ErrorECSInvalidParameter, err.Error(), ref)
+		assert.Equal(t, want, err.Error(), ref)
 	}
 }
 

spinifex/domains/elbv2/haproxy.go

@@ -46,7 +46,8 @@ backend {{.Name}}
     http-request return status {{.FixedResponse.StatusCode}}{{if .FixedResponse.ContentType}} content-type "{{.FixedResponse.ContentType}}"{{end}}{{if .FixedResponse.Body}} string "{{.FixedResponse.Body}}"{{end}}
 {{- else}}
     balance roundrobin{{if .HealthCheckPath}}
-    option httpchk GET {{.HealthCheckPath}}
+    option httpchk
+    http-check send meth GET ver HTTP/1.1 hdr Host %[bc_dst]%[bc_dst_port,regsub(^,:),regsub(^:{{.HealthCheckDefaultPort}}$,)] hdr User-Agent ELB-HealthChecker/2.0 uri {{.HealthCheckPath}}
     http-check expect status {{.HealthCheckMatcher}}{{end}}
 {{- range .Servers}}
     server {{.Name}} {{.Addr}}:{{.Port}} check{{if .Secure}} check-ssl ssl verify none{{end}} inter {{.CheckInterval}}s fall {{.Fall}} rise {{.Rise}}
@@ -102,9 +103,12 @@ type HAProxyBackend struct {
 	Name               string
 	HealthCheckPath    string
 	HealthCheckMatcher string
-	Servers            []HAProxyServer
-	FixedResponse      *HAProxyFixedResponse
-	Redirect           *HAProxyRedirect
+	// HealthCheckDefaultPort is left out of the check's Host header, as AWS
+	// sends "Host: <ip>" on the scheme's default port and "<ip>:<port>" otherwise.
+	HealthCheckDefaultPort int64
+	Servers                []HAProxyServer
+	FixedResponse          *HAProxyFixedResponse
+	Redirect               *HAProxyRedirect
 }
 
 // HAProxyFixedResponse renders an `http-request return` directive. All fields are validated before use.
@@ -185,10 +189,15 @@ func buildHAProxyConfig(lb *LoadBalancerRecord, listeners []*ListenerRecord, tgB
 			return
 		}
 
+		secure := strings.EqualFold(tg.Protocol, ProtocolHTTPS)
 		backend := HAProxyBackend{
-			Name:               backendName,
-			HealthCheckPath:    tg.HealthCheck.Path,
-			HealthCheckMatcher: tg.HealthCheck.Matcher,
+			Name:                   backendName,
+			HealthCheckPath:        tg.HealthCheck.Path,
+			HealthCheckMatcher:     tg.HealthCheck.Matcher,
+			HealthCheckDefaultPort: 80,
+		}
+		if secure {
+			backend.HealthCheckDefaultPort = 443
 		}
 
 		for _, target := range tg.Targets {
@@ -202,7 +211,7 @@ func buildHAProxyConfig(lb *LoadBalancerRecord, listeners []*ListenerRecord, tgB
 				CheckInterval: tg.HealthCheck.IntervalSeconds,
 				Fall:          tg.HealthCheck.UnhealthyThreshold,
 				Rise:          tg.HealthCheck.HealthyThreshold,
-				Secure:        strings.EqualFold(tg.Protocol, ProtocolHTTPS),
+				Secure:        secure,
 			})
 		}
 		cfg.Backends = append(cfg.Backends, backend)

spinifex/domains/elbv2/haproxy_test.go

@@ -57,8 +57,13 @@ func TestGenerateHAProxyConfig_SingleListenerAndBackend(t *testing.T) {
 	assert.Contains(t, config, "default_backend")
 
 	// Verify backend
-	assert.Contains(t, config, "option httpchk GET /health")
-	assert.Contains(t, config, "http-check expect status 200")
+	// AWS shape: HTTP/1.1, Host is the target IP (plus port when not 80), ELB User-Agent.
+	assert.Contains(t, config, "    option httpchk\n"+
+		"    http-check send meth GET ver HTTP/1.1"+
+		" hdr Host %[bc_dst]%[bc_dst_port,regsub(^,:),regsub(^:80$,)]"+
+		" hdr User-Agent ELB-HealthChecker/2.0 uri /health\n"+
+		"    http-check expect status 200\n")
+	assert.NotContains(t, config, "option httpchk GET")
 	assert.Contains(t, config, "10.0.1.10:8080")
 	assert.Contains(t, config, "10.0.1.11:8080")
 	assert.Contains(t, config, "check inter 30s fall 2 rise 5")
@@ -97,6 +102,28 @@ func TestGenerateHAProxyConfig_HTTPSBackendReencrypts(t *testing.T) {
 	config, _, err := GenerateHAProxyConfigWithCerts(lb, listeners, tgByArn, nil, "10.0.2.5", nil)
 	require.NoError(t, err)
 	assert.Contains(t, config, "10.0.2.10:30081 check check-ssl ssl verify none inter 10s")
+	// 443 is the HTTPS default port, so AWS omits it from the Host header.
+	assert.Contains(t, config, "hdr Host %[bc_dst]%[bc_dst_port,regsub(^,:),regsub(^:443$,)]")
+}
+
+// A '#' starts a comment in haproxy config, so uri must come last on the send
+// line: a fragment in the path then only trims the uri, not the headers.
+func TestGenerateHAProxyConfig_HealthCheckURIIsLastOnSendLine(t *testing.T) {
+	t.Parallel()
+	lb := &LoadBalancerRecord{LoadBalancerID: "lb-frag"}
+	tgArn := "arn:tg-frag"
+	listeners := []*ListenerRecord{
+		{ListenerArn: "arn:lst-frag", Port: 80, DefaultActions: []ListenerAction{{Type: ActionTypeForward, TargetGroupArn: tgArn}}},
+	}
+	hc := DefaultHealthCheck()
+	hc.Path = "/healthz?probe=lb#frag"
+	tgByArn := map[string]*TargetGroupRecord{
+		tgArn: {HealthCheck: hc, Targets: []Target{{Id: "i-f", Port: 80, PrivateIP: "10.0.0.9", HealthState: TargetHealthHealthy}}},
+	}
+
+	config, _, err := GenerateHAProxyConfigWithCerts(lb, listeners, tgByArn, nil, "0.0.0.0", nil)
+	require.NoError(t, err)
+	assert.Contains(t, config, "hdr User-Agent ELB-HealthChecker/2.0 uri /healthz?probe=lb#frag\n")
 }
 
 func TestGenerateHAProxyConfig_MultipleListeners(t *testing.T) {
@@ -519,8 +546,8 @@ func TestGenerateALBConfig_StillWorks(t *testing.T) {
 
 	assert.Contains(t, config, "mode http")
 	assert.Contains(t, config, "option httplog")
-	assert.Contains(t, config, "option httpchk GET /health")
-	assert.Contains(t, config, "http-check expect status 200")
+	assert.Contains(t, config, "ver HTTP/1.1 hdr Host %[bc_dst]")
+	assert.Contains(t, config, "uri /health\n    http-check expect status 200\n")
 	assert.NotContains(t, config, "mode tcp")
 	assert.NotContains(t, config, "option tcplog")
 }

spinifex/foundation/aws/errors/awserrors.go

@@ -424,6 +424,7 @@ var (
 	ErrorRoute53TooManyTagsForZone = "TooManyTags"
 	// ECS JSON-1.1 exception codes. ECS clients key on the "Exception"-suffixed
 	// __type; do not cross-wire with the EC2/ACM not-found codes above.
+	ErrorECSClientException                             = "ClientException"
 	ErrorECSClusterNotFound                             = "ClusterNotFoundException"
 	ErrorECSInvalidParameter                            = "InvalidParameterException"
 	ErrorECSServiceNotFound                             = "ServiceNotFoundException"
@@ -724,6 +725,33 @@ func ResolveRetryAfter(err error) (time.Duration, bool) {
 	return 0, false
 }
 
+// fieldsError attaches extra response-body fields to an error without changing
+// its text, for protocols that carry them (e.g. a REST-JSON throttle Reason).
+type fieldsError struct {
+	err    error
+	fields map[string]string
+}
+
+func (e *fieldsError) Error() string { return e.err.Error() }
+func (e *fieldsError) Unwrap() error { return e.err }
+
+// WithFields attaches extra top-level response-body fields to err. Protocols
+// without extra fields ignore them.
+func WithFields(err error, fields map[string]string) error {
+	if err == nil || len(fields) == 0 {
+		return err
+	}
+	return &fieldsError{err: err, fields: fields}
+}
+
+// ResolveFields returns the fields attached to err via WithFields, if any.
+func ResolveFields(err error) map[string]string {
+	if fe, ok := errors.AsType[*fieldsError](err); ok {
+		return fe.fields
+	}
+	return nil
+}
+
 // errorLookupByService overrides ErrorLookup's message and HTTP status for a
 // (service, code) pair whose wire code is shared by services with different
 // canonical wording — e.g. ACM and EKS both use "ResourceInUseException".
@@ -1214,6 +1242,7 @@ var ErrorLookup = map[string]ErrorMessage{
 	ErrorResourceNotFound:                                      {HTTPCode: 404, Message: "The specified resource was not found."},
 	ErrorEKSResourceInUse:                                      {HTTPCode: 409, Message: "A cluster with this name already exists in this account and Region. Delete the existing cluster, or choose a different name."},
 	ErrorEKSResourceNotFound:                                   {HTTPCode: 404, Message: "The specified cluster could not be found. You can view your available clusters with ListClusters. Amazon EKS clusters are Region specific."},
+	ErrorECSClientException:                                    {HTTPCode: 400, Message: "Unable to describe task definition."},
 	ErrorECSClusterNotFound:                                    {HTTPCode: 400, Message: "The specified cluster was not found. You can view your available clusters with ListClusters. Amazon ECS clusters are Region specific."},
 	ErrorECSInvalidParameter:                                   {HTTPCode: 400, Message: "The specified parameter is not valid. Review the available parameters for the API request."},
 	ErrorECSServiceNotFound:                                    {HTTPCode: 400, Message: "The specified service was not found. You can view your available services with ListServices. Amazon ECS services are cluster specific and Region specific."},

spinifex/foundation/aws/errors/awserrors_test.go

@@ -335,3 +335,21 @@ func TestIsNotFound(t *testing.T) {
 		})
 	}
 }
+
+func TestWithFields_SurvivesWrappingAndKeepsTheCode(t *testing.T) {
+	err := fmt.Errorf("handler: %w", WithFields(Errorf(ErrorInvalidParameterValue, "bad"), map[string]string{"Reason": "R"}))
+
+	code, message, ok := ResolveErrorDetail(err)
+	if !ok || code != ErrorInvalidParameterValue || message != "bad" {
+		t.Errorf("ResolveErrorDetail = %q, %q, %v", code, message, ok)
+	}
+	if got := ResolveFields(err); len(got) != 1 || got["Reason"] != "R" {
+		t.Errorf("ResolveFields = %v", got)
+	}
+	if got := ResolveFields(errors.New(ErrorInvalidParameterValue)); got != nil {
+		t.Errorf("unattached ResolveFields = %v", got)
+	}
+	if WithFields(nil, map[string]string{"a": "b"}) != nil {
+		t.Error("WithFields(nil) must stay nil")
+	}
+}

spinifex/gateway/auth.go

@@ -532,16 +532,31 @@ func (gw *GatewayConfig) writeSigV4Error(w http.ResponseWriter, r *http.Request,
 	}
 
 	svc := signedService(r)
+	if svc == "" {
+		// A request rejected while its signature was still being parsed has no
+		// service in context; its claimed scope still picks a registration's
+		// envelope, and nothing else.
+		if hint := credentialScopeService(r); hint != "" {
+			if _, ok := gw.registered(hint); ok {
+				svc = hint
+			}
+		}
+	}
 
-	// AWS JSON 1.1 services (bedrock family, EKS, …) need a JSON error body, or
-	// the SDK chokes deserializing our XML into its shape. An unserved scope is
-	// absent from jsonErrorService by definition, so its protocol is read off
-	// the request itself instead.
-	if gw.jsonErrorService(svc) || envelope.RequestSignalsJSONProtocol(r) {
-		w.Header().Set("Content-Type", envelope.JSONContentType)
-		w.Header().Set("X-Amzn-Errortype", envelope.JSONErrorType(errorCode))
-		w.WriteHeader(errorMsg.HTTPCode)
-		_, _ = w.Write(envelope.JSONBody(errorCode, errorMsg.Message))
+	// JSON services (bedrock family, EKS, registered JSON services, …) need a
+	// JSON error body in their own protocol, or the SDK chokes deserializing
+	// our XML into its shape. An unserved scope is absent from the tables by
+	// definition, so its protocol is read off the request itself instead.
+	p, isJSON := gw.errorProtocol(svc)
+	if !isJSON && envelope.RequestSignalsJSONProtocol(r) {
+		p, isJSON = envelope.ProtocolJSON11, true
+	}
+	if isJSON {
+		if p == envelope.ProtocolJSON10Query && errorCode == awserrors.ErrorRequestEntityTooLarge {
+			envelope.WriteTooLargeText(w, int64(sigv4.MaxPayloadLen))
+			return
+		}
+		gw.writeJSONError(w, p, svc, envelope.JSONError{Code: errorCode, Message: errorMsg.Message, Status: errorMsg.HTTPCode})
 		return
 	}
 
@@ -552,6 +567,21 @@ func (gw *GatewayConfig) writeSigV4Error(w http.ResponseWriter, r *http.Request,
 	_, _ = w.Write(xmlError)
 }
 
+// credentialScopeService returns the service named in the request's
+// Authorization credential scope, unverified, or "" when it has none.
+func credentialScopeService(r *http.Request) string {
+	_, rest, ok := strings.Cut(r.Header.Get("Authorization"), "Credential=")
+	if !ok {
+		return ""
+	}
+	scope, _, _ := strings.Cut(rest, ",")
+	parts := strings.Split(strings.TrimSpace(scope), "/")
+	if len(parts) != 5 {
+		return ""
+	}
+	return parts[3]
+}
+
 // signedService returns the credential-scope service the client signed with,
 // including a scope the gateway does not serve. A rejection has to render in
 // the client's format, and GetService reports those scopes as empty.

spinifex/gateway/error_envelope_sdk_test.go

@@ -0,0 +1,153 @@
+// Drives the real AWS SDK v2 clients against the gateway's error envelopes and
+// asserts they decode them into their modelled errors.
+
+package gateway
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"maps"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/lambda"
+	lambdatypes "github.com/aws/aws-sdk-go-v2/service/lambda/types"
+	"github.com/aws/aws-sdk-go-v2/service/sqs"
+	sqstypes "github.com/aws/aws-sdk-go-v2/service/sqs/types"
+	"github.com/aws/smithy-go"
+	"github.com/mulgadc/bluebottle/pkg/sigv4"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// registerTestCodes adds codes only the SDK fixtures raise, as the Lambda and
+// SQS domains would, and removes them afterwards.
+func registerTestCodes(t *testing.T, codes map[string]awserrors.ErrorMessage) {
+	t.Helper()
+	maps.Copy(awserrors.ErrorLookup, codes)
+	t.Cleanup(func() {
+		for code := range codes {
+			delete(awserrors.ErrorLookup, code)
+		}
+	})
+}
+
+func sdkConfig(url string) aws.Config {
+	return aws.Config{
+		Region:           testRegion,
+		Credentials:      credentials.NewStaticCredentialsProvider(testAccessKey, testSecretKey, ""),
+		RetryMaxAttempts: 1,
+		BaseEndpoint:     aws.String(url),
+	}
+}
+
+// sdkApp serves fixture sqs (JSON 1.0) and lambda (REST-JSON) registrations
+// through the full gateway pipeline; their dispatchers fail with dispatchErr.
+func sdkApp(t *testing.T, dispatchErr error) aws.Config {
+	t.Helper()
+	c := &dispatchCapture{dispatchErr: dispatchErr}
+	_, app := dispatchSeamApp(t, allowAllECRDocs(),
+		envelopeRegistration("sqs", dispatch.ErrorEnvelopeJSON10Query, c, func(r *dispatch.Registration) {
+			r.QueryErrorCodes = map[string]string{"QueueDoesNotExist": "AWS.SimpleQueueService.NonExistentQueue"}
+			r.ErrorNamespace = "com.amazonaws.sqs"
+		}),
+		envelopeRegistration("lambda", dispatch.ErrorEnvelopeRESTJSON, c, nil))
+	srv := httptest.NewServer(app)
+	t.Cleanup(srv.Close)
+	return sdkConfig(srv.URL)
+}
+
+// throttleApp serves the gateway's throttle response for the signed service.
+func throttleApp(t *testing.T) aws.Config {
+	t.Helper()
+	c := &dispatchCapture{}
+	b := dispatch.NewBuilder()
+	require.NoError(t, b.Register(envelopeRegistration("sqs", dispatch.ErrorEnvelopeJSON10Query, c,
+		func(r *dispatch.Registration) { r.ErrorNamespace = "com.amazonaws.sqs" })))
+	require.NoError(t, b.Register(envelopeRegistration("lambda", dispatch.ErrorEnvelopeRESTJSON, c, nil)))
+	gw := &GatewayConfig{DisableLogging: true, Services: b.Build()}
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		svc := credentialScopeService(r)
+		gw.writeThrottleError(w, r.WithContext(context.WithValue(r.Context(), ctxService, svc)))
+	}))
+	t.Cleanup(srv.Close)
+	return sdkConfig(srv.URL)
+}
+
+func TestSDKv2_SQSDecodesJSON10QueryErrors(t *testing.T) {
+	registerTestCodes(t, map[string]awserrors.ErrorMessage{
+		"QueueDoesNotExist": {HTTPCode: 400, Message: "The specified queue does not exist."},
+	})
+	cfg := sdkApp(t, awserrors.Errorf("QueueDoesNotExist", "no such queue"))
+
+	_, err := sqs.NewFromConfig(cfg).GetQueueUrl(context.Background(), &sqs.GetQueueUrlInput{QueueName: aws.String("q")})
+	require.Error(t, err)
+
+	var modelled *sqstypes.QueueDoesNotExist
+	require.ErrorAs(t, err, &modelled, "the SDK must map __type to its modelled error: %v", err)
+	assert.Equal(t, "no such queue", aws.ToString(modelled.Message))
+
+	var apiErr smithy.APIError
+	require.ErrorAs(t, err, &apiErr)
+	assert.Equal(t, "AWS.SimpleQueueService.NonExistentQueue", apiErr.ErrorCode(), "awsQueryCompatible clients report the legacy code")
+	assert.Equal(t, smithy.FaultClient, apiErr.ErrorFault())
+}
+
+func TestSDKv2_SQSDecodesRequestThrottled(t *testing.T) {
+	_, err := sqs.NewFromConfig(throttleApp(t)).GetQueueUrl(context.Background(), &sqs.GetQueueUrlInput{QueueName: aws.String("q")})
+
+	var throttled *sqstypes.RequestThrottled
+	require.ErrorAs(t, err, &throttled, "%v", err)
+	assert.Equal(t, "Rate exceeded.", aws.ToString(throttled.Message))
+}
+
+func TestSDKv2_LambdaDecodesThrottleWithReason(t *testing.T) {
+	registerTestCodes(t, map[string]awserrors.ErrorMessage{
+		"TooManyRequestsException": {HTTPCode: 429, Message: "Rate Exceeded."},
+	})
+	cfg := sdkApp(t, awserrors.WithFields(
+		awserrors.Errorf("TooManyRequestsException", "Rate Exceeded."),
+		map[string]string{"Reason": "ReservedFunctionConcurrentInvocationLimitExceeded"}))
+
+	_, err := lambda.NewFromConfig(cfg).GetFunction(context.Background(), &lambda.GetFunctionInput{FunctionName: aws.String("f")})
+	var throttled *lambdatypes.TooManyRequestsException
+	require.ErrorAs(t, err, &throttled, "%v", err)
+	assert.Equal(t, lambdatypes.ThrottleReasonReservedFunctionConcurrentInvocationLimitExceeded, throttled.Reason)
+	assert.Equal(t, "Rate Exceeded.", aws.ToString(throttled.Message))
+}
+
+func TestSDKv2_LambdaDecodesGatewayThrottle(t *testing.T) {
+	_, err := lambda.NewFromConfig(throttleApp(t)).GetFunction(context.Background(), &lambda.GetFunctionInput{FunctionName: aws.String("f")})
+
+	var throttled *lambdatypes.TooManyRequestsException
+	require.ErrorAs(t, err, &throttled, "%v", err)
+	assert.Equal(t, lambdatypes.ThrottleReason("CallerRateLimitExceeded"), throttled.Reason)
+}
+
+func TestSDKv2_LambdaDecodesServiceException(t *testing.T) {
+	cfg := sdkApp(t, errors.New("not a registered code"))
+
+	_, err := lambda.NewFromConfig(cfg).GetFunction(context.Background(), &lambda.GetFunctionInput{FunctionName: aws.String("f")})
+	var svcErr *lambdatypes.ServiceException
+	require.ErrorAs(t, err, &svcErr, "%v", err)
+}
+
+func TestSDKv2_LambdaOversizeBodyDecodesRequestEntityTooLarge(t *testing.T) {
+	cfg := sdkApp(t, nil)
+	oversized := bytes.Repeat([]byte("x"), int(sigv4.MaxPayloadLen)+1)
+
+	_, err := lambda.NewFromConfig(cfg).Invoke(context.Background(), &lambda.InvokeInput{FunctionName: aws.String("f"), Payload: oversized})
+	require.Error(t, err)
+
+	// The Go SDK models this error as RequestTooLargeException, so AWS's own
+	// RequestEntityTooLargeException surfaces as a generic API error.
+	var apiErr smithy.APIError
+	require.ErrorAs(t, err, &apiErr, "%v", err)
+	assert.Equal(t, "RequestEntityTooLargeException", apiErr.ErrorCode())
+}

spinifex/gateway/error_envelope_test.go

@@ -0,0 +1,304 @@
+// Tests that the gateway renders dispatcher, throttle, cluster-unavailable and
+// oversize-body errors through the envelope a registration declares.
+
+package gateway
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/bluebottle/pkg/sigv4"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/envelope"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	restJSONService  = "testrestjson"
+	json10Service    = "testjson10"
+	json11Service    = "testjson11"
+	xmlRegisteredSvc = "testxmlreg"
+)
+
+func envelopeRegistration(service string, env dispatch.ErrorEnvelope, c *dispatchCapture, mod func(*dispatch.Registration)) dispatch.Registration {
+	reg := dispatch.Registration{
+		Service:       service,
+		ResolveAction: func(*http.Request) string { return fakeAction },
+		Dispatch:      c.dispatcher(),
+		Errors:        env,
+		Inventory:     dispatch.Inventory{Registered: []string{fakeAction}},
+	}
+	if mod != nil {
+		mod(&reg)
+	}
+	return reg
+}
+
+func bodyString(t *testing.T, resp *http.Response) string {
+	t.Helper()
+	b, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	return string(b)
+}
+
+func throttleResponse(t *testing.T, regs ...dispatch.Registration) func(svc string) *http.Response {
+	t.Helper()
+	b := dispatch.NewBuilder()
+	for _, reg := range regs {
+		require.NoError(t, b.Register(reg))
+	}
+	gw := &GatewayConfig{DisableLogging: true, Services: b.Build()}
+	return func(svc string) *http.Response {
+		req := httptest.NewRequest(http.MethodPost, "/", nil)
+		req = req.WithContext(context.WithValue(req.Context(), ctxService, svc))
+		w := httptest.NewRecorder()
+		gw.writeThrottleError(w, req)
+		return w.Result()
+	}
+}
+
+func TestErrorEnvelope_Throttle(t *testing.T) {
+	c := &dispatchCapture{}
+	throttle := throttleResponse(t,
+		envelopeRegistration(restJSONService, dispatch.ErrorEnvelopeRESTJSON, c, nil),
+		envelopeRegistration(json10Service, dispatch.ErrorEnvelopeJSON10Query, c, nil),
+		envelopeRegistration(json11Service, dispatch.ErrorEnvelopeJSON, c, nil),
+		envelopeRegistration(xmlRegisteredSvc, dispatch.ErrorEnvelopeXML, c, nil),
+	)
+
+	namespaced := throttleResponse(t, envelopeRegistration(json10Service, dispatch.ErrorEnvelopeJSON10Query, c,
+		func(r *dispatch.Registration) { r.ErrorNamespace = "com.amazonaws.sqs" }))
+
+	t.Run("REST-JSON registration gets TooManyRequestsException with Reason", func(t *testing.T) {
+		resp := throttle(restJSONService)
+		assert.Equal(t, http.StatusTooManyRequests, resp.StatusCode)
+		assert.Equal(t, "application/json", resp.Header.Get("Content-Type"))
+		assert.Equal(t, "TooManyRequestsException", resp.Header.Get("X-Amzn-Errortype"))
+		assert.Empty(t, resp.Header.Get("Retry-After"))
+		assert.JSONEq(t, `{"Reason":"CallerRateLimitExceeded","Type":"User","message":"Rate Exceeded."}`, bodyString(t, resp))
+	})
+
+	t.Run("JSON 1.0 registration gets RequestThrottled with the query header", func(t *testing.T) {
+		resp := throttle(json10Service)
+		assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
+		assert.Equal(t, "application/x-amz-json-1.0", resp.Header.Get("Content-Type"))
+		assert.Equal(t, "RequestThrottled;Sender", resp.Header.Get("X-Amzn-Query-Error"))
+		assert.JSONEq(t, `{"__type":"RequestThrottled","message":"Rate exceeded."}`, bodyString(t, resp))
+	})
+
+	t.Run("JSON 1.0 namespaced throttle", func(t *testing.T) {
+		resp := namespaced(json10Service)
+		assert.Equal(t, "RequestThrottled;Sender", resp.Header.Get("X-Amzn-Query-Error"))
+		assert.JSONEq(t, `{"__type":"com.amazonaws.sqs#RequestThrottled","message":"Rate exceeded."}`, bodyString(t, resp))
+	})
+
+	t.Run("JSON 1.1 registration keeps its current throttle", func(t *testing.T) {
+		resp := throttle(json11Service)
+		assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode)
+		assert.Equal(t, "application/x-amz-json-1.1", resp.Header.Get("Content-Type"))
+		assert.Equal(t, "RequestLimitExceededException", resp.Header.Get("X-Amzn-Errortype"))
+		assert.Contains(t, bodyString(t, resp), `"__type":"RequestLimitExceededException"`)
+	})
+
+	t.Run("XML registration keeps its current throttle", func(t *testing.T) {
+		resp := throttle(xmlRegisteredSvc)
+		assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode)
+		assert.Equal(t, "application/xml", resp.Header.Get("Content-Type"))
+		assert.Contains(t, bodyString(t, resp), "<Code>RequestLimitExceeded</Code>")
+	})
+}
+
+func TestErrorEnvelope_ThrottleOverride(t *testing.T) {
+	c := &dispatchCapture{}
+	throttle := throttleResponse(t, envelopeRegistration(restJSONService, dispatch.ErrorEnvelopeRESTJSON, c,
+		func(r *dispatch.Registration) {
+			r.Throttle = &dispatch.ErrorSpec{
+				Code: "TooManyRequestsException", Message: "Slow down.", Status: http.StatusTooManyRequests,
+				Fields: map[string]string{"Reason": "FunctionInvocationRateLimitExceeded"},
+			}
+		}))
+	resp := throttle(restJSONService)
+	assert.Equal(t, http.StatusTooManyRequests, resp.StatusCode)
+	assert.JSONEq(t, `{"Reason":"FunctionInvocationRateLimitExceeded","Type":"User","message":"Slow down."}`, bodyString(t, resp))
+}
+
+func TestErrorEnvelope_DispatcherError(t *testing.T) {
+	notFound := awserrors.WithFields(
+		awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "bad value"),
+		map[string]string{"Reason": "Because"})
+
+	t.Run("REST-JSON uses the code as the type and carries service fields", func(t *testing.T) {
+		c := &dispatchCapture{dispatchErr: notFound}
+		_, app := dispatchSeamApp(t, allowAllECRDocs(), envelopeRegistration(restJSONService, dispatch.ErrorEnvelopeRESTJSON, c, nil))
+		resp := doRequest(app, signDispatchRequest(t, restJSONService, []byte("{}")))
+
+		assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
+		assert.Equal(t, "application/json", resp.Header.Get("Content-Type"))
+		assert.Equal(t, "InvalidParameterValue", resp.Header.Get("X-Amzn-Errortype"))
+		assert.Equal(t, `{"Reason":"Because","Type":"User","message":"bad value"}`, bodyString(t, resp))
+	})
+
+	t.Run("REST-JSON server fault is Type Service with its own status", func(t *testing.T) {
+		c := &dispatchCapture{dispatchErr: awserrors.Errorf(awserrors.ErrorInternalError, "boom")}
+		_, app := dispatchSeamApp(t, allowAllECRDocs(), envelopeRegistration(restJSONService, dispatch.ErrorEnvelopeRESTJSON, c, nil))
+		resp := doRequest(app, signDispatchRequest(t, restJSONService, []byte("{}")))
+
+		assert.Equal(t, http.StatusInternalServerError, resp.StatusCode)
+		assert.Equal(t, "ServiceException", resp.Header.Get("X-Amzn-Errortype"))
+		assert.JSONEq(t, `{"Type":"Service","message":"boom"}`, bodyString(t, resp))
+	})
+
+	t.Run("JSON 1.0 sends __type unsuffixed and maps the query code", func(t *testing.T) {
+		c := &dispatchCapture{dispatchErr: awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "bad value")}
+		_, app := dispatchSeamApp(t, allowAllECRDocs(), envelopeRegistration(json10Service, dispatch.ErrorEnvelopeJSON10Query, c,
+			func(r *dispatch.Registration) {
+				r.QueryErrorCodes = map[string]string{awserrors.ErrorInvalidParameterValue: "AWS.Simple.InvalidParameterValue"}
+			}))
+		resp := doRequest(app, signDispatchRequest(t, json10Service, []byte("{}")))
+
+		assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
+		assert.Equal(t, "application/x-amz-json-1.0", resp.Header.Get("Content-Type"))
+		assert.Equal(t, "AWS.Simple.InvalidParameterValue;Sender", resp.Header.Get("X-Amzn-Query-Error"))
+		assert.Empty(t, resp.Header.Get("X-Amzn-Errortype"))
+		assert.Equal(t, `{"__type":"InvalidParameterValue","message":"bad value"}`, bodyString(t, resp))
+	})
+
+	t.Run("JSON 1.0 namespaces __type but not the query header", func(t *testing.T) {
+		c := &dispatchCapture{dispatchErr: awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "bad value")}
+		_, app := dispatchSeamApp(t, allowAllECRDocs(), envelopeRegistration(json10Service, dispatch.ErrorEnvelopeJSON10Query, c,
+			func(r *dispatch.Registration) { r.ErrorNamespace = "com.amazonaws.sqs" }))
+		resp := doRequest(app, signDispatchRequest(t, json10Service, []byte("{}")))
+
+		assert.Equal(t, "InvalidParameterValue;Sender", resp.Header.Get("X-Amzn-Query-Error"))
+		assert.Equal(t, `{"__type":"com.amazonaws.sqs#InvalidParameterValue","message":"bad value"}`, bodyString(t, resp))
+	})
+
+	t.Run("JSON 1.0 per-code namespace and query code for a framework error", func(t *testing.T) {
+		c := &dispatchCapture{dispatchErr: awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "bad value")}
+		_, app := dispatchSeamApp(t, allowAllECRDocs(), envelopeRegistration(json10Service, dispatch.ErrorEnvelopeJSON10Query, c,
+			func(r *dispatch.Registration) {
+				r.ErrorNamespace = "com.amazonaws.sqs"
+				r.ErrorNamespaces = map[string]string{awserrors.ErrorInvalidParameterValue: envelope.CoralNamespace}
+			}))
+		resp := doRequest(app, signDispatchRequest(t, json10Service, []byte("{}")))
+		assert.Equal(t, `{"__type":"com.amazon.coral.service#InvalidParameterValue","message":"bad value"}`, bodyString(t, resp))
+	})
+
+	t.Run("JSON 1.0 unmapped server fault is Receiver with InternalFailure", func(t *testing.T) {
+		c := &dispatchCapture{dispatchErr: awserrors.Errorf(awserrors.ErrorInternalError, "boom")}
+		_, app := dispatchSeamApp(t, allowAllECRDocs(), envelopeRegistration(json10Service, dispatch.ErrorEnvelopeJSON10Query, c, nil))
+		resp := doRequest(app, signDispatchRequest(t, json10Service, []byte("{}")))
+
+		assert.Equal(t, http.StatusInternalServerError, resp.StatusCode)
+		assert.Equal(t, "InternalFailure;Receiver", resp.Header.Get("X-Amzn-Query-Error"))
+		assert.Contains(t, bodyString(t, resp), `"__type":"InternalFailure"`)
+	})
+
+	t.Run("unknown code takes the registration's internal error code", func(t *testing.T) {
+		c := &dispatchCapture{dispatchErr: errors.New("not a registered code")}
+		_, app := dispatchSeamApp(t, allowAllECRDocs(), envelopeRegistration(restJSONService, dispatch.ErrorEnvelopeRESTJSON, c,
+			func(r *dispatch.Registration) { r.InternalErrorCode = "CustomInternal" }))
+		resp := doRequest(app, signDispatchRequest(t, restJSONService, []byte("{}")))
+
+		assert.Equal(t, http.StatusInternalServerError, resp.StatusCode)
+		assert.Equal(t, "CustomInternal", resp.Header.Get("X-Amzn-Errortype"))
+	})
+
+	t.Run("JSON 1.1 and XML keep InternalError", func(t *testing.T) {
+		c := &dispatchCapture{dispatchErr: errors.New("not a registered code")}
+		_, app := dispatchSeamApp(t, allowAllECRDocs(), envelopeRegistration(json11Service, dispatch.ErrorEnvelopeJSON, c, nil))
+		resp := doRequest(app, signDispatchRequest(t, json11Service, []byte("{}")))
+		assert.Equal(t, "InternalErrorException", resp.Header.Get("X-Amzn-Errortype"))
+	})
+
+	t.Run("JSON 1.1 still appends Exception", func(t *testing.T) {
+		c := &dispatchCapture{dispatchErr: awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "bad value")}
+		_, app := dispatchSeamApp(t, allowAllECRDocs(), envelopeRegistration(json11Service, dispatch.ErrorEnvelopeJSON, c, nil))
+		resp := doRequest(app, signDispatchRequest(t, json11Service, []byte("{}")))
+
+		assert.Equal(t, "application/x-amz-json-1.1", resp.Header.Get("Content-Type"))
+		assert.Equal(t, "InvalidParameterValueException", resp.Header.Get("X-Amzn-Errortype"))
+		assert.Equal(t, `{"__type":"InvalidParameterValueException","message":"bad value"}`, bodyString(t, resp))
+	})
+}
+
+func TestErrorEnvelope_ClusterUnavailable(t *testing.T) {
+	c := &dispatchCapture{}
+	b := dispatch.NewBuilder()
+	require.NoError(t, b.Register(envelopeRegistration(restJSONService, dispatch.ErrorEnvelopeRESTJSON, c, nil)))
+	require.NoError(t, b.Register(envelopeRegistration(json10Service, dispatch.ErrorEnvelopeJSON10Query, c, nil)))
+	gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService(), Services: b.Build()}
+
+	call := func(svc string) *http.Response {
+		req := httptest.NewRequest(http.MethodPost, "/", nil)
+		req = req.WithContext(context.WithValue(req.Context(), ctxService, svc))
+		w := httptest.NewRecorder()
+		gw.Request(w, req)
+		return w.Result()
+	}
+
+	resp := call(restJSONService)
+	assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode)
+	assert.Equal(t, "application/json", resp.Header.Get("Content-Type"))
+	assert.Equal(t, "ServiceUnavailableException", resp.Header.Get("X-Amzn-Errortype"))
+	assert.JSONEq(t, `{"Type":"Service","message":"`+clusterUnavailableMsg+`"}`, bodyString(t, resp))
+
+	resp = call(json10Service)
+	assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode)
+	assert.Equal(t, "application/x-amz-json-1.0", resp.Header.Get("Content-Type"))
+	assert.Equal(t, "ServiceUnavailable;Receiver", resp.Header.Get("X-Amzn-Query-Error"))
+	assert.JSONEq(t, `{"__type":"com.amazon.coral.service#ServiceUnavailableException","message":"`+clusterUnavailableMsg+`"}`, bodyString(t, resp))
+}
+
+func TestErrorEnvelope_OversizeBody(t *testing.T) {
+	oversized := bytes.Repeat([]byte("x"), int(sigv4.MaxPayloadLen)+1)
+	send := func(t *testing.T, app http.Handler, service string) *http.Response {
+		req := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(oversized))
+		req.Host = "localhost:9999"
+		signTestRequestAs(t, req, oversized, testAccessKey, testSecretKey, service, time.Now().UTC())
+		return doRequest(app, req)
+	}
+	c := &dispatchCapture{}
+	_, app := dispatchSeamApp(t, allowAllECRDocs(),
+		envelopeRegistration(restJSONService, dispatch.ErrorEnvelopeRESTJSON, c, nil),
+		envelopeRegistration(json10Service, dispatch.ErrorEnvelopeJSON10Query, c, nil))
+
+	resp := send(t, app, restJSONService)
+	assert.Equal(t, http.StatusRequestEntityTooLarge, resp.StatusCode)
+	assert.Equal(t, "application/json", resp.Header.Get("Content-Type"))
+	assert.Equal(t, "RequestEntityTooLargeException", resp.Header.Get("X-Amzn-Errortype"))
+	assert.JSONEq(t, `{"Type":"User","message":"Request body exceeds the maximum allowed size."}`, bodyString(t, resp))
+
+	resp = send(t, app, json10Service)
+	assert.Equal(t, http.StatusRequestEntityTooLarge, resp.StatusCode)
+	assert.Equal(t, "text; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.Empty(t, resp.Header.Get("X-Amzn-Query-Error"))
+	assert.Equal(t, "HTTP content length exceeded "+strconv.Itoa(int(sigv4.MaxPayloadLen))+" bytes.", bodyString(t, resp))
+}
+
+func TestCredentialScopeService(t *testing.T) {
+	for name, tc := range map[string]struct {
+		header, want string
+	}{
+		"scope":     {"AWS4-HMAC-SHA256 Credential=AK/20260101/us-east-1/lambda/aws4_request, SignedHeaders=host, Signature=x", "lambda"},
+		"no header": {"", ""},
+		"truncated": {"AWS4-HMAC-SHA256 Credential=AK/20260101/us-east-1", ""},
+	} {
+		t.Run(name, func(t *testing.T) {
+			req := httptest.NewRequest(http.MethodPost, "/", nil)
+			if tc.header != "" {
+				req.Header.Set("Authorization", tc.header)
+			}
+			assert.Equal(t, tc.want, credentialScopeService(req))
+		})
+	}
+}

spinifex/gateway/gateway.go

@@ -419,13 +419,23 @@ func jsonErrorService(svc string) bool {
 	return false
 }
 
-// jsonErrorService reports whether svc's errors render in the AWS JSON 1.1
-// envelope: a registered service's own declared envelope, or the legacy
-// table for every service not yet registered through the seam. Every error
-// emitter calls this method, never the free function, so the two can never
-// disagree on a registered service.
-func (gw *GatewayConfig) jsonErrorService(svc string) bool {
-	return gw.selector().JSONErrors(svc)
+// errorProtocol reports the JSON wire format svc's errors use; ok is false when
+// they render as XML. A registered service's declared envelope wins.
+func (gw *GatewayConfig) errorProtocol(svc string) (p envelope.Protocol, ok bool) {
+	return gw.selector().ErrorProtocol(svc)
+}
+
+// writeJSONError renders a gateway-raised error in svc's JSON protocol. The
+// code gets the common-error "Exception" suffix unless verbatim.
+func (gw *GatewayConfig) writeJSONError(w http.ResponseWriter, p envelope.Protocol, svc string, e envelope.JSONError) {
+	if entry, ok := gw.registered(svc); ok {
+		if internal, own := entry.InternalErrorCode(); own && e.Code == awserrors.ErrorInternalError {
+			e.Code, e.Verbatim = internal, true
+		}
+		e.QueryCode = entry.QueryErrorCode(e.Code)
+		e.Namespace = entry.ErrorNamespace(e.Code)
+	}
+	envelope.WriteJSON(w, p, e)
 }
 
 // clusterUnavailableMsg is the 503 body when NATS is disconnected. Points
@@ -437,15 +447,13 @@ const clusterUnavailableMsg = "cluster unavailable: NATS disconnected — check
 func (gw *GatewayConfig) writeClusterUnavailable(w http.ResponseWriter, r *http.Request, svc string) {
 	requestID := uuid.NewV4().String()
 
-	// AWS JSON 1.1 services (EKS/ECS/bedrock family, …) get a JSON body.
-	if gw.jsonErrorService(svc) {
-		body := envelope.JSONBody(awserrors.ErrorServiceUnavailable, clusterUnavailableMsg)
-		w.Header().Set("Content-Type", envelope.JSONContentType)
-		w.Header().Set("X-Amzn-Errortype", envelope.JSONErrorType(awserrors.ErrorServiceUnavailable))
-		w.WriteHeader(http.StatusServiceUnavailable)
-		if _, err := w.Write(body); err != nil {
-			slog.Error("Failed to write JSON cluster-unavailable response", "err", err)
-		}
+	// JSON services get a body in their own protocol.
+	if p, ok := gw.errorProtocol(svc); ok {
+		gw.writeJSONError(w, p, svc, envelope.JSONError{
+			Code:    awserrors.ErrorServiceUnavailable,
+			Message: clusterUnavailableMsg,
+			Status:  http.StatusServiceUnavailable,
+		})
 		return
 	}
 
@@ -469,15 +477,18 @@ func (gw *GatewayConfig) writeThrottleError(w http.ResponseWriter, r *http.Reque
 	}
 	errorMsg := awserrors.LookupErrorMessage(svc, errorCode)
 
-	// AWS JSON 1.1 services (EKS/ECS/bedrock family, …) get a JSON body.
-	if gw.jsonErrorService(svc) {
-		body := envelope.JSONBody(errorCode, errorMsg.Message)
-		w.Header().Set("Content-Type", envelope.JSONContentType)
-		w.Header().Set("X-Amzn-Errortype", envelope.JSONErrorType(errorCode))
-		w.WriteHeader(errorMsg.HTTPCode)
-		if _, err := w.Write(body); err != nil {
-			slog.Error("Failed to write JSON throttle error response", "err", err)
+	// JSON services get a body in their own protocol.
+	if p, ok := gw.errorProtocol(svc); ok {
+		je := envelope.JSONError{Code: errorCode, Message: errorMsg.Message, Status: errorMsg.HTTPCode}
+		if entry, reg := gw.registered(svc); reg {
+			if spec, own := entry.ThrottleError(); own {
+				je = envelope.JSONError{
+					Code: spec.Code, Verbatim: true, Message: spec.Message,
+					Status: spec.Status, Fields: spec.Fields,
+				}
+			}
 		}
+		gw.writeJSONError(w, p, svc, je)
 		return
 	}
 
@@ -945,17 +956,15 @@ func (gw *GatewayConfig) ErrorHandler(w http.ResponseWriter, r *http.Request, er
 		w.Header().Set("Retry-After", strconv.Itoa(int(d.Seconds())))
 	}
 
-	// EKS, ECR, ACM, ECS, tagging, and the bedrock family use AWS JSON 1.1;
-	// query/XML services fall through.
-	if gw.jsonErrorService(svc) {
-		body := envelope.JSONBody(code, errorMsg.Message)
-		slog.Debug("Generated JSON error response", "service", svc, "error", err, "code", code, "json", string(body), "requestId", requestId)
-		w.Header().Set("Content-Type", envelope.JSONContentType)
-		w.Header().Set("X-Amzn-Errortype", envelope.JSONErrorType(code))
-		w.WriteHeader(errorMsg.HTTPCode)
-		if _, err := w.Write(body); err != nil {
-			slog.Error("Failed to write EKS error response", "err", err)
-		}
+	// JSON services answer in their own protocol; query/XML services fall
+	// through.
+	if p, ok := gw.errorProtocol(svc); ok {
+		slog.Debug("Generated JSON error response", "service", svc, "error", err, "code", code, "requestId", requestId)
+		gw.writeJSONError(w, p, svc, envelope.JSONError{
+			Code: code, Verbatim: p != envelope.ProtocolJSON11,
+			Message: errorMsg.Message, Status: errorMsg.HTTPCode,
+			Fields: awserrors.ResolveFields(err),
+		})
 		return
 	}
 

spinifex/ingress/aws/dispatch/registry.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/mulgadc/bluebottle/pkg/iampolicy"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/envelope"
 )
 
 // ErrorEnvelope selects which generic error rendering a service's errors use.
@@ -25,8 +26,43 @@ const (
 	ErrorEnvelopeXML
 	// ErrorEnvelopeJSON is the AWS JSON 1.1 error body with X-Amzn-Errortype.
 	ErrorEnvelopeJSON
+	// ErrorEnvelopeRESTJSON is the REST-JSON error body: application/json,
+	// x-amzn-ErrorType, Type and message keys and any service-supplied fields.
+	ErrorEnvelopeRESTJSON
+	// ErrorEnvelopeJSON10Query is the AWS JSON 1.0 error body with the
+	// awsQueryCompatible x-amzn-query-error header.
+	ErrorEnvelopeJSON10Query
 )
 
+// IsJSON reports whether the envelope is one of the JSON protocols.
+func (e ErrorEnvelope) IsJSON() bool {
+	return e == ErrorEnvelopeJSON || e == ErrorEnvelopeRESTJSON || e == ErrorEnvelopeJSON10Query
+}
+
+// Protocol returns the JSON wire format for a JSON envelope; ok is false for
+// XML and for an unset envelope.
+func (e ErrorEnvelope) Protocol() (p envelope.Protocol, ok bool) {
+	switch e {
+	case ErrorEnvelopeJSON:
+		return envelope.ProtocolJSON11, true
+	case ErrorEnvelopeRESTJSON:
+		return envelope.ProtocolRESTJSON, true
+	case ErrorEnvelopeJSON10Query:
+		return envelope.ProtocolJSON10Query, true
+	}
+	return envelope.ProtocolJSON11, false
+}
+
+// ErrorSpec is an error a registration defines for ingress to raise before its
+// dispatcher runs. The code is sent as the wire error type unchanged.
+type ErrorSpec struct {
+	Code    string
+	Message string
+	Status  int
+	// Fields are extra top-level body keys on protocols that carry them.
+	Fields map[string]string
+}
+
 // Inventory classifies a service's dispatched operations. Registered minus
 // Stubbed and Unsupported are implemented; Stubbed and Unsupported are
 // disjoint subsets of Registered.
@@ -82,7 +118,26 @@ type Registration struct {
 	ResolveAction ActionResolver
 	Dispatch      Dispatcher
 	Errors        ErrorEnvelope
-	Inventory     Inventory
+	// Throttle replaces the error returned when a request is rate limited.
+	// Unset, a REST-JSON or JSON 1.0 registration gets the protocol's own
+	// throttle error and any other keeps the gateway default. JSON only.
+	Throttle *ErrorSpec
+	// QueryErrorCodes maps an error code to the legacy query code sent in
+	// x-amzn-query-error; a code absent from it is sent as itself. JSON 1.0
+	// only.
+	QueryErrorCodes map[string]string
+	// InternalErrorCode is the wire code sent for an error with no registered
+	// code. Unset, a REST-JSON registration sends ServiceException and a JSON
+	// 1.0 one InternalFailure. JSON 1.1 and XML keep InternalError.
+	InternalErrorCode string
+	// ErrorNamespace is the Smithy shape namespace (e.g. "com.amazonaws.sqs")
+	// prefixed to the __type of service-defined errors as "<namespace>#<code>".
+	// Unset, __type is the bare code. JSON 1.0 only.
+	ErrorNamespace string
+	// ErrorNamespaces overrides ErrorNamespace for individual codes, e.g. to
+	// send a framework error under "com.amazon.coral.service". JSON 1.0 only.
+	ErrorNamespaces map[string]string
+	Inventory       Inventory
 }
 
 // UnknownService returns the client-visible code and message for a credential
@@ -137,10 +192,15 @@ func (b *Builder) Register(reg Registration) error {
 	}
 
 	e := &Entry{
-		service: reg.Service,
-		resolve: reg.ResolveAction,
-		handle:  reg.Dispatch,
-		errors:  reg.Errors,
+		service:    reg.Service,
+		resolve:    reg.ResolveAction,
+		handle:     reg.Dispatch,
+		errors:     reg.Errors,
+		throttle:   reg.Throttle,
+		queryCodes: maps.Clone(reg.QueryErrorCodes),
+		internal:   reg.InternalErrorCode,
+		namespace:  reg.ErrorNamespace,
+		namespaces: maps.Clone(reg.ErrorNamespaces),
 	}
 	for _, n := range names {
 		b.entries[n] = e
@@ -178,15 +238,44 @@ func validate(reg Registration) ([]string, error) {
 	if reg.Dispatch == nil {
 		return nil, fmt.Errorf("dispatch: %s: nil dispatcher", reg.Service)
 	}
-	if reg.Errors != ErrorEnvelopeXML && reg.Errors != ErrorEnvelopeJSON {
+	if reg.Errors != ErrorEnvelopeXML && !reg.Errors.IsJSON() {
 		return nil, fmt.Errorf("dispatch: %s: error envelope not set", reg.Service)
 	}
+	if err := validateErrorSpecs(reg); err != nil {
+		return nil, fmt.Errorf("dispatch: %s: %w", reg.Service, err)
+	}
 	if err := validateInventory(reg.Inventory); err != nil {
 		return nil, fmt.Errorf("dispatch: %s: %w", reg.Service, err)
 	}
 	return names, nil
 }
 
+func validateErrorSpecs(reg Registration) error {
+	if t := reg.Throttle; t != nil {
+		if !reg.Errors.IsJSON() {
+			return errors.New("throttle error needs a JSON error envelope")
+		}
+		if t.Code == "" || t.Status < 400 || t.Status > 599 {
+			return fmt.Errorf("throttle error needs a code and a 4xx or 5xx status, got %q and %d", t.Code, t.Status)
+		}
+	}
+	if len(reg.QueryErrorCodes) > 0 && reg.Errors != ErrorEnvelopeJSON10Query {
+		return errors.New("query error codes need the JSON 1.0 error envelope")
+	}
+	if reg.InternalErrorCode != "" && reg.Errors != ErrorEnvelopeRESTJSON && reg.Errors != ErrorEnvelopeJSON10Query {
+		return errors.New("internal error code needs the REST-JSON or JSON 1.0 error envelope")
+	}
+	if (reg.ErrorNamespace != "" || len(reg.ErrorNamespaces) > 0) && reg.Errors != ErrorEnvelopeJSON10Query {
+		return errors.New("error namespace needs the JSON 1.0 error envelope")
+	}
+	for code, legacy := range reg.QueryErrorCodes {
+		if code == "" || legacy == "" {
+			return errors.New("query error codes must not be empty")
+		}
+	}
+	return nil
+}
+
 func validateInventory(inv Inventory) error {
 	if len(inv.Registered) == 0 {
 		return errors.New("inventory has no registered operations")
@@ -265,6 +354,12 @@ type Entry struct {
 	resolve ActionResolver
 	handle  Dispatcher
 	errors  ErrorEnvelope
+
+	throttle   *ErrorSpec
+	queryCodes map[string]string
+	internal   string
+	namespace  string
+	namespaces map[string]string
 }
 
 // Service returns the registration's primary credential-scope name.
@@ -273,6 +368,64 @@ func (e *Entry) Service() string { return e.service }
 // ErrorEnvelope returns the envelope the service's errors render in.
 func (e *Entry) ErrorEnvelope() ErrorEnvelope { return e.errors }
 
+// ThrottleError returns the error to send when a request to this service is
+// rate limited; ok is false when the gateway default applies.
+func (e *Entry) ThrottleError() (spec ErrorSpec, ok bool) {
+	if e.throttle != nil {
+		return cloneSpec(*e.throttle), true
+	}
+	switch e.errors {
+	case ErrorEnvelopeRESTJSON:
+		return ErrorSpec{
+			Code:    "TooManyRequestsException",
+			Message: "Rate Exceeded.",
+			Status:  http.StatusTooManyRequests,
+			Fields:  map[string]string{"Reason": "CallerRateLimitExceeded"},
+		}, true
+	case ErrorEnvelopeJSON10Query:
+		return ErrorSpec{Code: "RequestThrottled", Message: "Rate exceeded.", Status: http.StatusBadRequest}, true
+	}
+	return ErrorSpec{}, false
+}
+
+func cloneSpec(s ErrorSpec) ErrorSpec {
+	s.Fields = maps.Clone(s.Fields)
+	return s
+}
+
+// InternalErrorCode returns the wire code for an error carrying no registered
+// code; ok is false when the gateway's InternalError applies.
+func (e *Entry) InternalErrorCode() (code string, ok bool) {
+	if e.internal != "" {
+		return e.internal, true
+	}
+	switch e.errors {
+	case ErrorEnvelopeRESTJSON:
+		return "ServiceException", true
+	case ErrorEnvelopeJSON10Query:
+		return "InternalFailure", true
+	}
+	return "", false
+}
+
+// ErrorNamespace returns the Smithy namespace prefixed to code's error type,
+// or "" when none is declared.
+func (e *Entry) ErrorNamespace(code string) string {
+	if ns, ok := e.namespaces[code]; ok {
+		return ns
+	}
+	return e.namespace
+}
+
+// QueryErrorCode returns the legacy query code sent in x-amzn-query-error for
+// code, which is code itself unless the registration maps it.
+func (e *Entry) QueryErrorCode(code string) string {
+	if q, ok := e.queryCodes[code]; ok {
+		return q
+	}
+	return code
+}
+
 // ResolveAction returns the advisory action label for r, or "" when the
 // registration has no resolver or cannot name the action.
 func (e *Entry) ResolveAction(r *http.Request) string {

spinifex/ingress/aws/dispatch/registry_test.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/envelope"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -224,3 +225,142 @@ func TestRegistryAllowsConcurrentReads(t *testing.T) {
 	}
 	wg.Wait()
 }
+
+func TestRegisterValidatesErrorSpecs(t *testing.T) {
+	spec := &ErrorSpec{Code: "Throttled", Status: 429}
+	mutate := map[string]func(*Registration){
+		"throttle on XML":         func(r *Registration) { r.Errors = ErrorEnvelopeXML; r.Throttle = spec },
+		"throttle without code":   func(r *Registration) { r.Throttle = &ErrorSpec{Status: 429} },
+		"throttle with 2xx":       func(r *Registration) { r.Throttle = &ErrorSpec{Code: "X", Status: 200} },
+		"query codes on JSON 1.1": func(r *Registration) { r.QueryErrorCodes = map[string]string{"A": "B"} },
+		"namespaces on JSON 1.1":  func(r *Registration) { r.ErrorNamespaces = map[string]string{"A": "ns"} },
+		"namespace on JSON 1.1":   func(r *Registration) { r.ErrorNamespace = "com.amazonaws.sqs" },
+		"empty query code": func(r *Registration) {
+			r.Errors = ErrorEnvelopeJSON10Query
+			r.QueryErrorCodes = map[string]string{"A": ""}
+		},
+	}
+	for name, m := range mutate {
+		t.Run(name, func(t *testing.T) {
+			r := validReg("ecr")
+			m(&r)
+			require.Error(t, NewBuilder().Register(r))
+		})
+	}
+
+	for _, env := range []ErrorEnvelope{ErrorEnvelopeRESTJSON, ErrorEnvelopeJSON10Query} {
+		r := validReg("svc")
+		r.Errors = env
+		r.Throttle = spec
+		require.NoError(t, NewBuilder().Register(r))
+	}
+}
+
+func TestEntryThrottleErrorDefaultsPerEnvelope(t *testing.T) {
+	entry := func(env ErrorEnvelope) *Entry {
+		b := NewBuilder()
+		r := validReg("svc")
+		r.Errors = env
+		require.NoError(t, b.Register(r))
+		e, ok := b.Build().Lookup("svc")
+		require.True(t, ok)
+		return e
+	}
+
+	_, own := entry(ErrorEnvelopeJSON).ThrottleError()
+	assert.False(t, own, "JSON 1.1 keeps the gateway default")
+	_, own = entry(ErrorEnvelopeXML).ThrottleError()
+	assert.False(t, own, "XML keeps the gateway default")
+
+	rest, own := entry(ErrorEnvelopeRESTJSON).ThrottleError()
+	require.True(t, own)
+	assert.Equal(t, ErrorSpec{
+		Code: "TooManyRequestsException", Message: "Rate Exceeded.", Status: 429,
+		Fields: map[string]string{"Reason": "CallerRateLimitExceeded"},
+	}, rest)
+	rest.Fields["Reason"] = "mutated"
+	again, _ := entry(ErrorEnvelopeRESTJSON).ThrottleError()
+	assert.Equal(t, "CallerRateLimitExceeded", again.Fields["Reason"], "a caller cannot mutate the default")
+
+	q, own := entry(ErrorEnvelopeJSON10Query).ThrottleError()
+	require.True(t, own)
+	assert.Equal(t, "RequestThrottled", q.Code)
+	assert.Equal(t, 400, q.Status)
+}
+
+func TestSelectorErrorProtocol(t *testing.T) {
+	b := NewBuilder()
+	for svc, env := range map[string]ErrorEnvelope{
+		"rest": ErrorEnvelopeRESTJSON, "q": ErrorEnvelopeJSON10Query, "j": ErrorEnvelopeJSON, "x": ErrorEnvelopeXML,
+	} {
+		r := validReg(svc)
+		r.Errors = env
+		require.NoError(t, b.Register(r))
+	}
+	sel := NewSelector(b.Build(), nil)
+
+	for svc, want := range map[string]envelope.Protocol{
+		"rest": envelope.ProtocolRESTJSON, "q": envelope.ProtocolJSON10Query, "j": envelope.ProtocolJSON11,
+	} {
+		p, ok := sel.ErrorProtocol(svc)
+		assert.True(t, ok, svc)
+		assert.Equal(t, want, p, svc)
+		assert.True(t, sel.JSONErrors(svc), svc)
+	}
+	_, ok := sel.ErrorProtocol("x")
+	assert.False(t, ok)
+	assert.False(t, sel.JSONErrors("x"))
+	_, ok = sel.ErrorProtocol("unknown")
+	assert.False(t, ok)
+}
+
+func TestEntryQueryErrorCode(t *testing.T) {
+	b := NewBuilder()
+	r := validReg("svc")
+	r.Errors = ErrorEnvelopeJSON10Query
+	r.QueryErrorCodes = map[string]string{"QueueDoesNotExist": "AWS.SimpleQueueService.NonExistentQueue"}
+	require.NoError(t, b.Register(r))
+	e, _ := b.Build().Lookup("svc")
+	assert.Equal(t, "AWS.SimpleQueueService.NonExistentQueue", e.QueryErrorCode("QueueDoesNotExist"))
+	assert.Equal(t, "OverLimit", e.QueryErrorCode("OverLimit"))
+}
+
+func TestEntryInternalErrorCode(t *testing.T) {
+	entry := func(env ErrorEnvelope, code string) *Entry {
+		b := NewBuilder()
+		r := validReg("svc")
+		r.Errors = env
+		r.InternalErrorCode = code
+		require.NoError(t, b.Register(r))
+		e, _ := b.Build().Lookup("svc")
+		return e
+	}
+	code, ok := entry(ErrorEnvelopeRESTJSON, "").InternalErrorCode()
+	assert.True(t, ok)
+	assert.Equal(t, "ServiceException", code)
+	code, ok = entry(ErrorEnvelopeJSON10Query, "").InternalErrorCode()
+	assert.True(t, ok)
+	assert.Equal(t, "InternalFailure", code)
+	code, _ = entry(ErrorEnvelopeRESTJSON, "Mine").InternalErrorCode()
+	assert.Equal(t, "Mine", code)
+	_, ok = entry(ErrorEnvelopeJSON, "").InternalErrorCode()
+	assert.False(t, ok)
+	_, ok = entry(ErrorEnvelopeXML, "").InternalErrorCode()
+	assert.False(t, ok)
+
+	r := validReg("svc")
+	r.InternalErrorCode = "Mine"
+	require.Error(t, NewBuilder().Register(r), "JSON 1.1 cannot set an internal error code")
+}
+
+func TestEntryErrorNamespacePerCodeOverride(t *testing.T) {
+	b := NewBuilder()
+	r := validReg("svc")
+	r.Errors = ErrorEnvelopeJSON10Query
+	r.ErrorNamespace = "com.amazonaws.sqs"
+	r.ErrorNamespaces = map[string]string{"SerializationException": "com.amazon.coral.service"}
+	require.NoError(t, b.Register(r))
+	e, _ := b.Build().Lookup("svc")
+	assert.Equal(t, "com.amazonaws.sqs", e.ErrorNamespace("QueueDoesNotExist"))
+	assert.Equal(t, "com.amazon.coral.service", e.ErrorNamespace("SerializationException"))
+}

spinifex/ingress/aws/dispatch/selector.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"net/http"
 	"slices"
+
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/envelope"
 )
 
 // Legacy answers service selection for signed services not yet registered,
@@ -59,15 +61,27 @@ func (s *Selector) ResolveAction(r *http.Request, service string) string {
 	return s.legacy.ResolveAction(r, service)
 }
 
-// JSONErrors reports whether service's errors render in the AWS JSON 1.1
-// envelope: a registration's declared envelope, else the legacy answer.
+// JSONErrors reports whether service's errors render in a JSON envelope: a
+// registration's declared envelope, else the legacy answer.
 func (s *Selector) JSONErrors(service string) bool {
 	if e, ok := s.Lookup(service); ok {
-		return e.ErrorEnvelope() == ErrorEnvelopeJSON
+		return e.ErrorEnvelope().IsJSON()
 	}
 	return s.legacy.JSONErrors(service)
 }
 
+// ErrorProtocol returns the JSON protocol service's errors use; ok is false
+// when they render as XML. A service served only by the legacy path is JSON 1.1.
+func (s *Selector) ErrorProtocol(service string) (p envelope.Protocol, ok bool) {
+	if e, found := s.Lookup(service); found {
+		return e.ErrorEnvelope().Protocol()
+	}
+	if s.legacy.JSONErrors(service) {
+		return envelope.ProtocolJSON11, true
+	}
+	return envelope.ProtocolJSON11, false
+}
+
 // Validate fails if any name the registry serves, alias included, is also
 // served by the legacy path, so no service is reachable down both at once.
 func (s *Selector) Validate() error {

spinifex/ingress/aws/envelope/envelope.go

@@ -8,6 +8,7 @@ import (
 	"encoding/xml"
 	"fmt"
 	"log/slog"
+	"maps"
 	"net/http"
 	"strings"
 )
@@ -164,3 +165,162 @@ func RequestSignalsJSONProtocol(r *http.Request) bool {
 	}
 	return strings.HasPrefix(r.Header.Get("Content-Type"), "application/x-amz-json")
 }
+
+// REST-JSON and AWS JSON 1.0 content types.
+const (
+	RESTJSONContentType = "application/json"
+	JSON10ContentType   = "application/x-amz-json-1.0"
+)
+
+// QueryErrorHeader carries the legacy query-protocol error code and fault on
+// an awsQueryCompatible JSON 1.0 response, as "<code>;<Sender|Receiver>".
+const QueryErrorHeader = "X-Amzn-Query-Error"
+
+// CoralNamespace is the __type namespace of errors the AWS framework raises
+// ahead of a service, as opposed to errors the service model defines.
+const CoralNamespace = "com.amazon.coral.service"
+
+// coralTypes maps a gateway error code to the framework error AWS JSON 1.0
+// services send for it. A code absent here becomes "<code>Exception".
+var coralTypes = map[string]string{
+	"SignatureDoesNotMatch": "InvalidSignatureException",
+	"InvalidClientTokenId":  "UnrecognizedClientException",
+	"InvalidAction":         "UnknownOperationException",
+}
+
+// Protocol selects a JSON error wire format.
+type Protocol int
+
+const (
+	// ProtocolJSON11 is AWS JSON 1.1: __type and message, X-Amzn-Errortype.
+	ProtocolJSON11 Protocol = iota
+	// ProtocolRESTJSON is REST-JSON: Type and message, x-amzn-ErrorType, plus
+	// any service-supplied fields.
+	ProtocolRESTJSON
+	// ProtocolJSON10Query is AWS JSON 1.0 with the awsQueryCompatible trait:
+	// __type and message, and an x-amzn-query-error header.
+	ProtocolJSON10Query
+)
+
+// JSONError describes one error for WriteJSON.
+type JSONError struct {
+	// Code is the AWS error code.
+	Code string
+	// Verbatim sends Code as the wire error type. Otherwise "Exception" is
+	// appended when absent, as AWS does for errors raised outside a service
+	// model.
+	Verbatim bool
+	Message  string
+	Status   int
+	// Fields are extra top-level body keys; they never replace a reserved key.
+	Fields map[string]string
+	// Namespace is the Smithy shape namespace prefixed to __type as
+	// "<namespace>#<code>" on ProtocolJSON10Query, for a Verbatim code only.
+	// A code that is not Verbatim is a framework error and always takes
+	// CoralNamespace on that protocol.
+	Namespace string
+	// QueryCode is the legacy query code for ProtocolJSON10Query; Code is used
+	// when empty.
+	QueryCode string
+}
+
+// senderFault reports whether the error is the caller's ("Sender"/"User") or
+// the service's ("Receiver"/"Service"), derived from the HTTP status.
+func (e JSONError) senderFault() bool { return e.Status < http.StatusInternalServerError }
+
+// json10Type is the __type of e on JSON 1.0: a framework error is a coral
+// type, a service error its code under the service namespace when it has one.
+func (e JSONError) json10Type() string {
+	if !e.Verbatim {
+		name, ok := coralTypes[e.Code]
+		if !ok {
+			name = JSONErrorType(e.Code)
+		}
+		return CoralNamespace + "#" + name
+	}
+	if e.Namespace != "" {
+		return e.Namespace + "#" + e.Code
+	}
+	return e.Code
+}
+
+func (e JSONError) wireType() string {
+	if e.Verbatim {
+		return e.Code
+	}
+	return JSONErrorType(e.Code)
+}
+
+// JSONContentTypeFor returns the content type p's responses use.
+func JSONContentTypeFor(p Protocol) string {
+	switch p {
+	case ProtocolRESTJSON:
+		return RESTJSONContentType
+	case ProtocolJSON10Query:
+		return JSON10ContentType
+	default:
+		return JSONContentType
+	}
+}
+
+// JSONErrorBody renders e's body for p.
+func JSONErrorBody(p Protocol, e JSONError) []byte {
+	typeKey, typeVal := "__type", e.wireType()
+	if p == ProtocolJSON10Query {
+		typeVal = e.json10Type()
+	}
+	if p == ProtocolRESTJSON {
+		typeKey, typeVal = "Type", "User"
+		if !e.senderFault() {
+			typeVal = "Service"
+		}
+	}
+	body := map[string]string{}
+	maps.Copy(body, e.Fields)
+	body[typeKey] = typeVal
+	// AWS sends no message key at all for some JSON 1.0 errors.
+	if e.Message != "" || p != ProtocolJSON10Query {
+		body["message"] = e.Message
+	}
+	out, err := json.Marshal(body)
+	if err != nil {
+		slog.Error("envelope: failed to marshal JSON error", "code", e.Code, "err", err)
+		return fmt.Appendf(nil, `{"%s":"InternalError","message":%q}`, typeKey, e.Message)
+	}
+	return out
+}
+
+// WriteJSON writes e as a complete response in p's wire format.
+func WriteJSON(w http.ResponseWriter, p Protocol, e JSONError) {
+	h := w.Header()
+	h.Set("Content-Type", JSONContentTypeFor(p))
+	if p == ProtocolJSON10Query {
+		fault := "Sender"
+		if !e.senderFault() {
+			fault = "Receiver"
+		}
+		code := e.QueryCode
+		if code == "" {
+			code = e.Code
+		}
+		h.Set(QueryErrorHeader, code+";"+fault)
+	} else {
+		h.Set("X-Amzn-Errortype", e.wireType())
+	}
+	w.WriteHeader(e.Status)
+	if _, err := w.Write(JSONErrorBody(p, e)); err != nil {
+		slog.Error("envelope: failed to write JSON error response", "err", err)
+	}
+}
+
+// WriteTooLargeText writes the oversize-request rejection AWS JSON 1.0
+// services send: a plain-text 413 with no error type and no query header.
+func WriteTooLargeText(w http.ResponseWriter, limit int64) {
+	h := w.Header()
+	h.Set("Content-Type", "text; charset=utf-8")
+	h.Set("Connection", "close")
+	w.WriteHeader(http.StatusRequestEntityTooLarge)
+	if _, err := fmt.Fprintf(w, "HTTP content length exceeded %d bytes.", limit); err != nil {
+		slog.Error("envelope: failed to write oversize response", "err", err)
+	}
+}

spinifex/ingress/aws/envelope/envelope_test.go

@@ -270,3 +270,146 @@ func TestRequestSignalsJSONProtocol(t *testing.T) {
 		})
 	}
 }
+
+func TestWriteTooLargeText(t *testing.T) {
+	w := httptest.NewRecorder()
+	WriteTooLargeText(w, 2097152)
+	resp := w.Result()
+	assert.Equal(t, http.StatusRequestEntityTooLarge, resp.StatusCode)
+	assert.Equal(t, "text; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.Equal(t, "close", resp.Header.Get("Connection"))
+	assert.Empty(t, resp.Header.Get(QueryErrorHeader))
+	assert.Equal(t, "HTTP content length exceeded 2097152 bytes.", w.Body.String())
+}
+
+func TestWriteJSON_WireBytesPerProtocol(t *testing.T) {
+	cases := []struct {
+		name         string
+		p            Protocol
+		e            JSONError
+		contentType  string
+		headers      map[string]string
+		absentHeader string
+		body         string
+	}{
+		{
+			name:        "JSON 1.1 appends Exception",
+			p:           ProtocolJSON11,
+			e:           JSONError{Code: "Throttling", Message: "slow", Status: 400},
+			contentType: "application/x-amz-json-1.1",
+			headers:     map[string]string{"X-Amzn-Errortype": "ThrottlingException"},
+			body:        `{"__type":"ThrottlingException","message":"slow"}`,
+		},
+		{
+			name:        "REST-JSON keeps a verbatim type and sorts extra fields",
+			p:           ProtocolRESTJSON,
+			e:           JSONError{Code: "TooManyRequestsException", Verbatim: true, Message: "Rate Exceeded.", Status: 429, Fields: map[string]string{"Reason": "CallerRateLimitExceeded"}},
+			contentType: "application/json",
+			headers:     map[string]string{"X-Amzn-Errortype": "TooManyRequestsException"},
+			body:        `{"Reason":"CallerRateLimitExceeded","Type":"User","message":"Rate Exceeded."}`,
+		},
+		{
+			name:        "REST-JSON common error gets Exception and a 5xx is a Service fault",
+			p:           ProtocolRESTJSON,
+			e:           JSONError{Code: "ServiceUnavailable", Message: "down", Status: 503},
+			contentType: "application/json",
+			headers:     map[string]string{"X-Amzn-Errortype": "ServiceUnavailableException"},
+			body:        `{"Type":"Service","message":"down"}`,
+		},
+		{
+			name:        "REST-JSON extra fields cannot replace reserved keys",
+			p:           ProtocolRESTJSON,
+			e:           JSONError{Code: "X", Verbatim: true, Message: "m", Status: 400, Fields: map[string]string{"Type": "Evil", "message": "evil"}},
+			contentType: "application/json",
+			body:        `{"Type":"User","message":"m"}`,
+		},
+		{
+			name:         "JSON 1.0 never suffixes and sends the query header",
+			p:            ProtocolJSON10Query,
+			e:            JSONError{Code: "QueueDoesNotExist", Verbatim: true, Message: "gone", Status: 400, QueryCode: "AWS.SimpleQueueService.NonExistentQueue"},
+			contentType:  "application/x-amz-json-1.0",
+			headers:      map[string]string{"X-Amzn-Query-Error": "AWS.SimpleQueueService.NonExistentQueue;Sender"},
+			absentHeader: "X-Amzn-Errortype",
+			body:         `{"__type":"QueueDoesNotExist","message":"gone"}`,
+		},
+		{
+			name:        "JSON 1.0 namespaces a verbatim type only in the body",
+			p:           ProtocolJSON10Query,
+			e:           JSONError{Code: "QueueDoesNotExist", Verbatim: true, Namespace: "com.amazonaws.sqs", Message: "gone", Status: 400},
+			contentType: "application/x-amz-json-1.0",
+			headers:     map[string]string{"X-Amzn-Query-Error": "QueueDoesNotExist;Sender"},
+			body:        `{"__type":"com.amazonaws.sqs#QueueDoesNotExist","message":"gone"}`,
+		},
+		{
+			name:        "JSON 1.0 captured QueueDoesNotExist",
+			p:           ProtocolJSON10Query,
+			e:           JSONError{Code: "QueueDoesNotExist", Verbatim: true, Namespace: "com.amazonaws.sqs", Message: "The specified queue does not exist.", Status: 400, QueryCode: "AWS.SimpleQueueService.NonExistentQueue"},
+			contentType: "application/x-amz-json-1.0",
+			headers:     map[string]string{"X-Amzn-Query-Error": "AWS.SimpleQueueService.NonExistentQueue;Sender"},
+			body:        `{"__type":"com.amazonaws.sqs#QueueDoesNotExist","message":"The specified queue does not exist."}`,
+		},
+		{
+			name:        "JSON 1.0 captured framework error raised by a service",
+			p:           ProtocolJSON10Query,
+			e:           JSONError{Code: "MissingRequiredParameterException", Verbatim: true, Namespace: CoralNamespace, Message: "The request must contain the parameter MessageBody.", Status: 400, QueryCode: "MissingParameter"},
+			contentType: "application/x-amz-json-1.0",
+			headers:     map[string]string{"X-Amzn-Query-Error": "MissingParameter;Sender"},
+			body:        `{"__type":"com.amazon.coral.service#MissingRequiredParameterException","message":"The request must contain the parameter MessageBody."}`,
+		},
+		{
+			name:        "JSON 1.0 captured unknown operation has no message",
+			p:           ProtocolJSON10Query,
+			e:           JSONError{Code: "InvalidAction", Status: 400},
+			contentType: "application/x-amz-json-1.0",
+			headers:     map[string]string{"X-Amzn-Query-Error": "InvalidAction;Sender"},
+			body:        `{"__type":"com.amazon.coral.service#UnknownOperationException"}`,
+		},
+		{
+			name:        "JSON 1.0 captured wrong secret key",
+			p:           ProtocolJSON10Query,
+			e:           JSONError{Code: "SignatureDoesNotMatch", Message: "sig", Status: 403},
+			contentType: "application/x-amz-json-1.0",
+			headers:     map[string]string{"X-Amzn-Query-Error": "SignatureDoesNotMatch;Sender"},
+			body:        `{"__type":"com.amazon.coral.service#InvalidSignatureException","message":"sig"}`,
+		},
+		{
+			name:        "JSON 1.0 captured unknown token",
+			p:           ProtocolJSON10Query,
+			e:           JSONError{Code: "InvalidClientTokenId", Message: "The security token included in the request is invalid", Status: 403},
+			contentType: "application/x-amz-json-1.0",
+			headers:     map[string]string{"X-Amzn-Query-Error": "InvalidClientTokenId;Sender"},
+			body:        `{"__type":"com.amazon.coral.service#UnrecognizedClientException","message":"The security token included in the request is invalid"}`,
+		},
+		{
+			name:        "JSON 1.1 and REST-JSON still send a message when empty",
+			p:           ProtocolRESTJSON,
+			e:           JSONError{Code: "X", Verbatim: true, Status: 400},
+			contentType: "application/json",
+			body:        `{"Type":"User","message":""}`,
+		},
+		{
+			name:        "JSON 1.0 query code defaults to the code and a 5xx is a Receiver",
+			p:           ProtocolJSON10Query,
+			e:           JSONError{Code: "InternalError", Verbatim: true, Message: "boom", Status: 500},
+			contentType: "application/x-amz-json-1.0",
+			headers:     map[string]string{"X-Amzn-Query-Error": "InternalError;Receiver"},
+			body:        `{"__type":"InternalError","message":"boom"}`,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			w := httptest.NewRecorder()
+			WriteJSON(w, tc.p, tc.e)
+			resp := w.Result()
+			assert.Equal(t, tc.e.Status, resp.StatusCode)
+			assert.Equal(t, tc.contentType, resp.Header.Get("Content-Type"))
+			for k, v := range tc.headers {
+				assert.Equal(t, v, resp.Header.Get(k), k)
+			}
+			if tc.absentHeader != "" {
+				assert.Empty(t, resp.Header.Get(tc.absentHeader))
+			}
+			assert.Equal(t, tc.body, w.Body.String())
+		})
+	}
+}

spinifex/operator/admin/images_remove.go

@@ -339,6 +339,9 @@ func deletePrefix(store objectstore.ObjectStore, bucket, prefix string) (int, in
 				Bucket: aws.String(bucket),
 				Key:    obj.Key,
 			}); err != nil {
+				if objectstore.IsNoSuchKeyError(err) {
+					continue
+				}
 				return count, bytes, fmt.Errorf("delete %s: %w", *obj.Key, err)
 			}
 			count++

spinifex/operator/admin/images_remove_test.go

@@ -123,39 +123,46 @@ func TestRemoveSystemImage_HappyPath(t *testing.T) {
 	assert.Equal(t, 0, store.Count())
 }
 
-// strictDeleteStore reports a missing key on delete, the way predastore does.
-// The in-memory store is silently tolerant and AWS S3 answers 204, so neither
-// reproduces the backend this actually runs against.
-type strictDeleteStore struct {
-	*objectstore.MemoryObjectStore
-}
-
-var _ objectstore.ObjectStore = (*strictDeleteStore)(nil)
-
-func (s *strictDeleteStore) DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput) (*awss3.DeleteObjectOutput, error) {
-	key := aws.StringValue(input.Key)
-	if _, err := s.HeadObject(ctx, &awss3.HeadObjectInput{Bucket: input.Bucket, Key: input.Key}); err != nil {
-		return nil, &objectstore.NoSuchKeyError{Key: key}
-	}
-	return s.MemoryObjectStore.DeleteObject(ctx, input)
-}
-
 // TestRemoveSystemImage_OrphanedBlocksAreReclaimable covers the state the
 // orphan report points operators at: blocks in the object store with no
 // document. A live env19 run failed here on NoSuchKey, so the one documented
 // way to reclaim an orphan was broken by the very thing that defines one.
 func TestRemoveSystemImage_OrphanedBlocksAreReclaimable(t *testing.T) {
 	memory := objectstore.NewMemoryObjectStore()
-	store := &strictDeleteStore{MemoryObjectStore: memory}
 	const id = "ami-orphaned0001"
 	putAMIBlocks(t, memory, id, 3, 128)
 
-	res, err := RemoveSystemImage(store, testRemoveBucket, RemoveImageOpts{ImageID: id, Force: true})
+	res, err := RemoveSystemImage(memory, testRemoveBucket, RemoveImageOpts{ImageID: id, Force: true})
 	require.NoError(t, err, "an image whose document is already gone must still have its blocks reclaimable")
 	assert.Equal(t, 3, res.ObjectsDeleted)
 	assert.Equal(t, 0, memory.Count(), "the stranded blocks must actually be released")
 }
 
+// racingDeleteStore lets a concurrent deleter win every DeleteObject, so the
+// caller's own delete answers NoSuchKey as Predastore would.
+type racingDeleteStore struct {
+	*objectstore.MemoryObjectStore
+}
+
+func (s racingDeleteStore) DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput) (*awss3.DeleteObjectOutput, error) {
+	_, _ = s.MemoryObjectStore.DeleteObject(ctx, input)
+	return s.MemoryObjectStore.DeleteObject(ctx, input)
+}
+
+// A block another remover deleted between the listing and this delete is
+// gone, so the sweep carries on rather than stranding the rest of the prefix.
+// It is not counted: this run did not release it.
+func TestRemoveSystemImage_ConcurrentlyDeletedBlocksAreSkipped(t *testing.T) {
+	memory := objectstore.NewMemoryObjectStore()
+	const id = "ami-racing000001"
+	putAMIBlocks(t, memory, id, 3, 128)
+
+	res, err := RemoveSystemImage(racingDeleteStore{MemoryObjectStore: memory}, testRemoveBucket, RemoveImageOpts{ImageID: id, Force: true})
+	require.NoError(t, err)
+	assert.Equal(t, 0, res.ObjectsDeleted)
+	assert.Equal(t, 0, memory.Count())
+}
+
 func TestRemoveSystemImage_AccountOwned_Refused(t *testing.T) {
 	store := objectstore.NewMemoryObjectStore()
 	const id = "ami-acct-001"

spinifex/providers/objectstore/objectstore.go

@@ -266,6 +266,8 @@ func (m *MemoryObjectStore) PutObject(_ context.Context, input *s3.PutObjectInpu
 	return &s3.PutObjectOutput{}, nil
 }
 
+// DeleteObject answers NoSuchKeyError for an absent key, matching what
+// S3ObjectStore returns against Predastore.
 func (m *MemoryObjectStore) DeleteObject(_ context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
 	if err := checkKey(input.Key); err != nil {
 		return nil, err
@@ -274,6 +276,9 @@ func (m *MemoryObjectStore) DeleteObject(_ context.Context, input *s3.DeleteObje
 	defer m.mu.Unlock()
 
 	storageKey := makeKey(*input.Bucket, *input.Key)
+	if _, ok := m.objects[storageKey]; !ok {
+		return nil, &NoSuchKeyError{Key: *input.Key}
+	}
 	delete(m.objects, storageKey)
 	delete(m.modified, storageKey)
 

spinifex/providers/objectstore/objectstore_test.go

@@ -110,13 +110,15 @@ func TestMemoryObjectStore_Delete(t *testing.T) {
 func TestMemoryObjectStore_DeleteNonExistent(t *testing.T) {
 	store := NewMemoryObjectStore()
 
-	// Deleting non-existent should not error (matches S3 behavior)
+	// Predastore answers NoSuchKey for an absent key; the fake must too.
 	deleteInput := &s3.DeleteObjectInput{
 		Bucket: aws.String("test-bucket"),
 		Key:    aws.String("never-existed"),
 	}
 	_, err := store.DeleteObject(t.Context(), deleteInput)
-	assert.NoError(t, err)
+	var noSuchKey *NoSuchKeyError
+	require.ErrorAs(t, err, &noSuchKey)
+	assert.Equal(t, "never-existed", noSuchKey.Key)
 }
 
 func TestMemoryObjectStore_ListObjects(t *testing.T) {

spinifex/runtime/daemon/tagging_current_behaviour_test.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -15,13 +16,13 @@ import (
 // design, and a failure means the behaviour changed.
 
 // A non-instance ID is never resolved to a resource before the central write.
-// Another account's ID, a well-formed ID naming nothing, and an ID with no
-// recognised prefix all succeed, and each is stored under the caller's account.
+// Another account's ID and a well-formed ID naming nothing both succeed and are
+// stored under the caller's account; an unrecognised prefix is InvalidID.
 func TestCurrentBehaviour_Tagging_EC2CreateTagsStoresUnresolvedIDsInCallerNamespace(t *testing.T) {
 	const caller = "999999999999"
 	d := tagTestDaemon(t, "i-tag-cb-unrelated", nil)
 
-	ids := []string{"sg-0a1b2c3d4e5f60718", "vol-00000000000000000", "not-a-real-id"}
+	ids := []string{"sg-0a1b2c3d4e5f60718", "vol-00000000000000000"}
 	resources := make([]*string, 0, len(ids))
 	for _, id := range ids {
 		resources = append(resources, aws.String(id))
@@ -38,10 +39,10 @@ func TestCurrentBehaviour_Tagging_EC2CreateTagsStoresUnresolvedIDsInCallerNamesp
 		assert.Empty(t, centralTags(t, d, testAccountID, id), id)
 	}
 
-	out, err := d.tagsService.DescribeTags(t.Context(), &ec2.DescribeTagsInput{
-		Filters: []*ec2.Filter{{Name: aws.String("resource-id"), Values: []*string{aws.String("not-a-real-id")}}},
+	_, err = d.createTags(context.Background(), &ec2.CreateTagsInput{
+		Resources: []*string{aws.String("not-a-real-id")},
+		Tags:      []*ec2.Tag{{Key: aws.String("env"), Value: aws.String("prod")}},
 	}, caller)
-	require.NoError(t, err)
-	require.Len(t, out.Tags, 1)
-	assert.Equal(t, "unknown", aws.StringValue(out.Tags[0].ResourceType))
+	require.EqualError(t, err, awserrors.ErrorInvalidID)
+	assert.Empty(t, centralTags(t, d, caller, "not-a-real-id"))
 }

spinifex/runtime/roles/awsgw/tagging_current_behaviour_test.go

@@ -121,19 +121,23 @@ func pageARNs(p getResourcesPage) []string {
 	return out
 }
 
-// GetResources lists whatever the central index holds: an ID with no
-// recognised prefix is reported with resource type "unknown", and a
-// well-formed ID naming no resource is reported as that resource.
+// GetResources lists whatever the central index holds: a well-formed ID naming
+// no resource is reported as that resource. An ID with no recognised prefix
+// never reaches the index, because CreateTags rejects it as InvalidID.
 func TestCurrentBehaviour_Tagging_GetResourcesListsUnresolvedEC2IDs(t *testing.T) {
 	agg, tags := newTaggingCBAggregator(t)
 
-	createTags(t, tags, []string{"not-a-real-id", "vol-00000000000000000"}, map[string]string{"env": "prod"})
+	_, err := tags.CreateTags(t.Context(), &ec2.CreateTagsInput{
+		Resources: aws.StringSlice([]string{"not-a-real-id"}),
+		Tags:      []*ec2.Tag{{Key: aws.String("env"), Value: aws.String("prod")}},
+	}, authzAccountID)
+	require.EqualError(t, err, awserrors.ErrorInvalidID)
+	createTags(t, tags, []string{"vol-00000000000000000"}, map[string]string{"env": "prod"})
 
 	page, err := getResources(t, agg, `{"ResourceTypeFilters":["ec2"]}`)
 	require.NoError(t, err)
 	prefix := "arn:aws:ec2:" + authzRegion + ":" + authzAccountID + ":"
 	assert.ElementsMatch(t, []string{
-		prefix + "unknown/not-a-real-id",
 		prefix + "volume/vol-00000000000000000",
 	}, pageARNs(page))
 }

spinifex/runtime/roles/awsgw/tagging_listers_test.go

@@ -101,7 +101,10 @@ func TestEC2IndexLister_ListsCallerIndexEntriesAsRegionalARNs(t *testing.T) {
 	nc, svc := newEC2IndexFixture(t)
 	seedEC2Tags(t, svc, listerAccount, []string{"sg-0000000000000000a"}, map[string]string{"env": "prod", "Name": "web"})
 	seedEC2Tags(t, svc, listerAccount, []string{"subnet-0000000000000000b"}, map[string]string{"env": "dev"})
-	seedEC2Tags(t, svc, listerAccount, []string{"not-a-real-id", "vol-00000000000000000"}, map[string]string{"env": "orphan"})
+	seedEC2Tags(t, svc, listerAccount, []string{"vol-00000000000000000"}, map[string]string{"env": "orphan"})
+	// CreateTags rejects an unrecognised prefix, but an entry stored before it
+	// did is still in the index and still listed.
+	require.NoError(t, svc.PutResourceTags(t.Context(), listerAccount, "not-a-real-id", map[string]string{"env": "orphan"}))
 	seedEC2Tags(t, svc, foreignAccount, []string{"sg-0000000000000000f"}, map[string]string{"env": "other"})
 
 	got, err := newEC2IndexLister(nc, listerRegion).ListTagged(t.Context(), listerAccount, nil)