feat: add a dedicated SQS message store JetStream server (#1299 )

* feat: add a dedicated SQS message store JetStream server separate from control-plane NATS * test: cover the SQS store server start, validation and memory limit
fix: send ALB health checks as HTTP/1.1 with Host and ELB-HealthChecker User-Agent (#1296 )
2026-10-27 06:27:38 +00:00 · 2026-10-27 15:05:27 +10:00 · 2026-10-27 15:41:44 +11:00 · 2026-10-27 15:41:33 +11:00 · 2026-10-27 15:40:03 +11:00 · 2026-10-27 15:39:51 +11:00
61 changed files with 2452 additions and 169 deletions
@@ -0,0 +1,57 @@
+# spinifex-unit-version: 1
+[Unit]
+Description=Spinifex SQS Message Store (dedicated JetStream)
+PartOf=spinifex.target
+ConditionPathExists=/etc/spinifex/sqs-store/sqs-store.conf
+
+[Service]
+EnvironmentFile=-/etc/spinifex/telemetry.env
+Type=simple
+Slice=spinifex-system.slice
+User=spinifex-sqsstore
+Group=spinifex
+ExecStart=/usr/local/bin/spx service sqs-store start
+Restart=on-failure
+RestartSec=5
+# JetStream store shutdown and raft stepdown have no internal deadline.
+TimeoutStopSec=20
+OOMScoreAdjust=-500
+# Soft Go heap limit for the store; MemoryHigh/Max bound a runaway on top.
+Environment=GOMEMLIMIT=512MiB
+MemoryHigh=10%
+MemoryMax=20%
+Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/sqs-store/sqs-store.conf
+
+# Filesystem access
+NoNewPrivileges=yes
+ProtectSystem=strict
+PrivateDevices=yes
+ReadOnlyPaths=/etc/spinifex/sqs-store /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem
+ReadWritePaths=/var/lib/spinifex/sqs-store /var/log/spinifex
+
+# Hardening
+ProtectHome=yes
+PrivateTmp=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RestrictRealtime=yes
+RestrictNamespaces=yes
+ProtectClock=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectProc=invisible
+SystemCallArchitectures=native
+MemoryDenyWriteExecute=yes
+RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK
+CapabilityBoundingSet=
+ProtectHostname=yes
+KeyringMode=private
+RemoveIPC=yes
+LimitCORE=0
+LimitNOFILE=65536
+LimitNPROC=4096
+
+[Install]
+WantedBy=spinifex.target
@@ -1,9 +1,9 @@
-# spinifex-unit-version: 1
+# spinifex-unit-version: 2
 [Unit]
 Description=Spinifex Infrastructure Platform
 After=network-online.target
 Wants=network-online.target
-Wants=spinifex-nats.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service
+Wants=spinifex-nats.service spinifex-sqs-store.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service

 [Install]
 WantedBy=multi-user.target
@@ -73,6 +73,7 @@ The cluster plane is peer-scoped. On a single-NIC node the planes collapse onto
 | Ports | Serving |
 |---|---|
 | TCP 4222, 4248 | NATS client and cluster routes |
+| TCP 4223, 4249 | SQS message store client and cluster routes |
 | TCP 4432 | Formation, then the daemon cluster manager |
 | TCP and UDP 5300 | Internal DNS |
 | TCP 6641, 6642 | OVN northbound and southbound |
@@ -99,6 +99,9 @@ behavior in other words.
 | 4432 | Formation server | HTTPS | Cluster (bootstrap only) | Cluster join coordination; active only while a join token is valid. Binds the node's `--bind` (lan) address. See *Formation port lifecycle* below. | Short-lived bearer token + TLS¹ |
 | 4222 | spinifex-nats (client) | NATS + TLS | Cluster | Internal service bus for EC2/EBS/VPC/S3 handlers | Token + mutual TLS (cluster CA) |
 | 4248 | spinifex-nats (cluster) | NATS + TLS | Cluster | Inter-node NATS federation | Token + mutual TLS (cluster CA) |
+| 4223 | spinifex-sqs-store (client) | NATS + TLS | Cluster | SQS message data, separate from the control-plane bus | Store-only token + TLS |
+| 4249 | spinifex-sqs-store (cluster) | NATS + TLS | Cluster | Inter-node SQS store replication | Store-only token + mutual TLS (cluster CA) |
+| 8223 | spinifex-sqs-store (monitor) | HTTP | Loopback | varz and jsz for the store | Loopback only |
 | 5300 | northstar | DNS (UDP + TCP) | Cluster | Forward target for every node's per-instance DNS shim, dialled cross-node. Binds the wildcard by design. | None — restrict by firewall |
 | 6660 | predastore (blob node) | QUIC / UDP | Cluster | Erasure-coded object shard transport between hosts. Multi-node clusters only — see *Predastore ports* below. | TLS 1.3, server certificate verified against the cluster CA |
 | 7660 | predastore (meta node) | QUIC / UDP | Cluster | Raft consensus over global state — buckets and the object index — between hosts. Multi-node clusters only. | TLS 1.3, server certificate verified against the cluster CA |
@@ -208,7 +211,7 @@ table inet spinifex_filter {

    # Cluster, from peer nodes only. Replace with your nodes' lan-plane
    # addresses — a CIDR does not generalise to nodes on different subnets.
-    ip saddr { 10.0.1.1, 10.0.1.2, 10.0.1.3 } tcp dport { 4222, 4248, 4432, 5300, 6641, 6642, 6643, 6644, 8660 } accept
+    ip saddr { 10.0.1.1, 10.0.1.2, 10.0.1.3 } tcp dport { 4222, 4223, 4248, 4249, 4432, 5300, 6641, 6642, 6643, 6644, 8660 } accept
    ip saddr { 10.0.1.1, 10.0.1.2, 10.0.1.3 } udp dport { 5300, 6660, 7660 } accept

    # Encap, from peer chassis only. Replace with your nodes' vpc-plane
@@ -4,12 +4,15 @@ go 1.27.3

 require (
 	github.com/aws/aws-sdk-go v1.55.8
-	github.com/aws/aws-sdk-go-v2 v1.47.3
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.23
+	github.com/aws/aws-sdk-go-v2 v1.47.4
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.24
 	github.com/aws/aws-sdk-go-v2/config v1.33.9
 	github.com/aws/aws-sdk-go-v2/credentials v1.20.9
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3
+	github.com/aws/aws-sdk-go-v2/service/lambda v1.113.3
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.114.4
+	github.com/aws/aws-sdk-go-v2/service/sqs v1.52.6
+	github.com/aws/smithy-go v1.28.6
 	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/charmbracelet/bubbles v1.0.0
 	github.com/charmbracelet/bubbletea v1.3.10
@@ -64,8 +67,8 @@ require (
 	github.com/Microsoft/hcsshim v0.15.0-rc.4 // indirect
 	github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect
 	github.com/atotto/clipboard v0.1.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.7 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.7 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 // indirect
@@ -75,7 +78,6 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 // indirect
-	github.com/aws/smithy-go v1.28.6 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
@@ -22,20 +22,20 @@ github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z
 github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk=
-github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.23 h1:/fGPpKrXTjzkhsDXExBgNT2cGwxiF9rLrQXh8RlXMhI=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.23/go.mod h1:QWUDuRi9PFjBHIU2iH1Pkldbe7zqLfXJC4wWLbF8nuQ=
+github.com/aws/aws-sdk-go-v2 v1.47.4 h1:bTvk6UdSSSvaQsfLVj1JRDEEqj/PDRwGf1KanaRzau0=
+github.com/aws/aws-sdk-go-v2 v1.47.4/go.mod h1:XkrjAZvr705Nytv6PkPUhoSB2W9+vVHBp5oAWpkh8Ak=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.24 h1:t7FxqJ5KHDeKceyD4HtGaG6z7hMSeVGjs5/4c+L5ArM=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.24/go.mod h1:5XK6NRVlK8s8yj4zKcM43HLluGA/rAz+azLLvWDMRr0=
 github.com/aws/aws-sdk-go-v2/config v1.33.9 h1:Hv6bcQUJVnmuWaKjqD6Ot8sE75NrWsWV20oGT5aWle0=
 github.com/aws/aws-sdk-go-v2/config v1.33.9/go.mod h1:ivowmrZNfBQLcg4QkaItgi5wae4Bzot9RzIdbY1hfTg=
 github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0=
 github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 h1:CqL8NN3p4I0JTwwV5MEoatIl4Y2amZ2X9dRD+0hbRL0=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6/go.mod h1:exdJ32hMqXl0Wyo6NwqEC5yIhtua3+PDaKE8P29m4qE=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 h1:FpBq24TG6WVSObTNhQOv7Q0088rounUUlwn3zlNItkM=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6/go.mod h1:ONzdPVYC95oBOu0VWNEsKEntcoemP3xVKjT+qWml1s4=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.7 h1:fSNAt6hs4qLjb9tMZc6vj3PHlOCYGXK2JjJZ7AW2tf4=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.7/go.mod h1:TMav+EuXQu5kNpZZz/6rIC8xx4mCsW+6AKHrjvOSAsE=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.7 h1:7NS6kqBjkWIN8P/AEdmtyq6eQGAhN1oHTZdzXclpH6Y=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.7/go.mod h1:1PFqtAUsIgF1nEVojzRJSf/v88zjzqrGW+w+UAtkXyQ=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3VvGMOdbGemtUhX4=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 h1:e6YW5uME30TkrElR+4/xcY7NB6CCj2dcv7vtmGXhfhQ=
@@ -46,10 +46,14 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 h1:LHHdobjo5
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6/go.mod h1:CBGB0evAiaMD3Lk4wPO7LQCcCZLuhzL5Qf/wfEUHQ5w=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 h1:mYViDr2azcgxOpRrObZOl631bvavqILoE/L+Kxgbwys=
 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6/go.mod h1:OXVdAo9jNhggB+hJRGOyy7WolGTKB6Rg7IIulDeSzPM=
+github.com/aws/aws-sdk-go-v2/service/lambda v1.113.3 h1:oSHldse7GDHpdq81XeAJnhIbsdDKP9bMIt00G8W6uXI=
+github.com/aws/aws-sdk-go-v2/service/lambda v1.113.3/go.mod h1:Y//K046w4uhiR3o77beo/y52Jv2uEY13iJ+3+jicLO8=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.114.4 h1:MrkWRB58Qw5TEkb51QVooVLjlM3IevO2iVXEz74B8pg=
 github.com/aws/aws-sdk-go-v2/service/s3 v1.114.4/go.mod h1:yfIxaKRHasvfy6tNEm3/LpqKsupMzQjMaWj9Y0jMVbo=
 github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A=
 github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M=
+github.com/aws/aws-sdk-go-v2/service/sqs v1.52.6 h1:Suq1LNhMBpsmShEv3vO+J3H8HEeK+GwYMJDYLE4IJSs=
+github.com/aws/aws-sdk-go-v2/service/sqs v1.52.6/go.mod h1:rHGuzFg4AfVnWZa3/533tn5GQd1LzVkTXI9WMT8LqAc=
 github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk=
 github.com/aws/aws-sdk-go-v2/service/sso v1.38.4/go.mod h1:+FEO9jrVMpc9oN3CdAxJbtlQEu3z8UddDPSedpeM3PY=
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 h1:j6c+HA4AoS0EBEjArPjavFEBrucJEryfrQU0TffmACk=
@@ -421,7 +421,7 @@ done
 # Matched on the service argv so the ssh shell running this cannot match itself.
 for host in "${HOSTS[@]}"; do
    $DRY_RUN && { echo "  [dry-run] $host: wait for every 'spx service' process to exit"; continue; }
-    on "$host" "for u in spinifex-predastore spinifex-viperblock spinifex-nats; do
+    on "$host" "for u in spinifex-predastore spinifex-viperblock spinifex-sqs-store spinifex-nats; do
            systemctl is-active --quiet \$u && sudo systemctl stop \$u
        done
        for _ in \$(seq 1 45); do
@@ -112,7 +112,7 @@ run sudo systemctl reset-failed 'spinifex-*' 2>/dev/null || true
 # the wipe and before the verification, failing the reset on a file the script
 # had already deleted. update-nodes.sh carries the same wait.
 if ! $DRY_RUN; then
-    for unit in spinifex-predastore spinifex-viperblock spinifex-nats; do
+    for unit in spinifex-predastore spinifex-viperblock spinifex-sqs-store spinifex-nats; do
        if systemctl is-active --quiet "$unit"; then
            sudo systemctl stop "$unit" || true
        fi
@@ -299,6 +299,10 @@ if ! $KEEP_DATA && [ -d "$DATA_DIR/nats/jetstream" ]; then
    log "removing the JetStream store"
    run sudo rm -rf "$DATA_DIR/nats/jetstream"
 fi
+if ! $KEEP_DATA && [ -d "$DATA_DIR/sqs-store/jetstream" ]; then
+    log "removing the SQS store JetStream data"
+    run sudo rm -rf "$DATA_DIR/sqs-store/jetstream"
+fi

 # Directories may survive — a mountpoint cannot be removed, and neither can the
 # path leading down to one. Files may not: a surviving file is state that would
@@ -161,6 +161,7 @@ create_service_users() {
    # Create per-service users with correct home directories
    declare -A SERVICE_HOMES=(
        [nats]="/var/lib/spinifex/nats"
+        [sqsstore]="/var/lib/spinifex/sqs-store"
        [gw]="/var/lib/spinifex/awsgw"
        [daemon]="/var/lib/spinifex/spinifex"
        [storage]="/var/lib/spinifex/predastore"
@@ -169,7 +170,7 @@ create_service_users() {
        [vpcd]="/var/lib/spinifex"
        [ui]="/var/lib/spinifex"
    )
-    for svc in nats gw daemon storage northstar viperblock vpcd ui; do
+    for svc in nats sqsstore gw daemon storage northstar viperblock vpcd ui; do
        local user="spinifex-${svc}"
        if ! id "$user" > /dev/null 2>&1; then
            $SUDO useradd --system --no-create-home \
@@ -201,7 +202,7 @@ create_service_users() {
    # Daemon consumes viperblock's NBD socket — join the producer-typed group.
    $SUDO usermod -aG spinifex-viperblock spinifex-daemon

-    info "Service users created (spinifex-{nats,gw,daemon,storage,northstar,viperblock,vpcd,ui})"
+    info "Service users created (spinifex-{nats,sqsstore,gw,daemon,storage,northstar,viperblock,vpcd,ui})"
 }

 # --- Install scoped sudoers rules ---
@@ -429,7 +430,7 @@ table inet spinifex_filter {
        # inventory and running instances to anyone who asks. 8660 is
        # predastore's unauthenticated /healthz and /readyz, which name the
        # peers a gate cannot reach, so it belongs here for the same reason.
-        ip saddr $spinifex_peers tcp dport { 4222, 4248, 4432, 5300, 6641, 6642, 6643, 6644, 8660 } accept
+        ip saddr $spinifex_peers tcp dport { 4222, 4223, 4248, 4249, 4432, 5300, 6641, 6642, 6643, 6644, 8660 } accept
        ip saddr $spinifex_peers udp dport { 5300, 6660, 7660 } accept

        # Encap plane, peer-scoped: Geneve, IKE, NAT-T and ESP. Geneve is a
@@ -981,6 +982,12 @@ TMPEOF
    $SUDO chown "spinifex-nats:$SPINIFEX_GROUP" /etc/spinifex/nats
    $SUDO chmod 0750 /etc/spinifex/nats

+    # The SQS store holds its own token: 0700 keeps every other spinifex-*
+    # service in the shared group out.
+    $SUDO mkdir -p /etc/spinifex/sqs-store
+    $SUDO chown "spinifex-sqsstore:$SPINIFEX_GROUP" /etc/spinifex/sqs-store
+    $SUDO chmod 0700 /etc/spinifex/sqs-store
+
    $SUDO mkdir -p /etc/spinifex/predastore
    $SUDO chown "spinifex-storage:$SPINIFEX_GROUP" /etc/spinifex/predastore
    $SUDO chmod 0750 /etc/spinifex/predastore
@@ -1008,6 +1015,10 @@ TMPEOF
    $SUDO chown "spinifex-nats:$SPINIFEX_GROUP" /var/lib/spinifex/nats
    $SUDO chmod 0700 /var/lib/spinifex/nats

+    $SUDO mkdir -p /var/lib/spinifex/sqs-store
+    $SUDO chown "spinifex-sqsstore:$SPINIFEX_GROUP" /var/lib/spinifex/sqs-store
+    $SUDO chmod 0700 /var/lib/spinifex/sqs-store
+
    $SUDO mkdir -p /var/lib/spinifex/spinifex
    $SUDO chown "spinifex-daemon:$SPINIFEX_GROUP" /var/lib/spinifex/spinifex
    $SUDO chmod 0700 /var/lib/spinifex/spinifex
@@ -1099,6 +1110,7 @@ fix_file_ownership() {
    # Per-service data dirs — recursive chown so existing files are accessible
    for entry in \
        nats:spinifex-nats \
+        sqs-store:spinifex-sqsstore \
        predastore:spinifex-storage \
        northstar:spinifex-northstar \
        spinifex:spinifex-daemon \
@@ -1117,6 +1129,11 @@ fix_file_ownership() {
        $SUDO chown -R "spinifex-nats:$SPINIFEX_GROUP" /etc/spinifex/nats \
            || fatal "Failed to set ownership on /etc/spinifex/nats"
    fi
+    if [ -d /etc/spinifex/sqs-store ]; then
+        $SUDO chown -R "spinifex-sqsstore:$SPINIFEX_GROUP" /etc/spinifex/sqs-store \
+            || fatal "Failed to set ownership on /etc/spinifex/sqs-store"
+        $SUDO chmod 0700 /etc/spinifex/sqs-store
+    fi
    if [ -d /etc/spinifex/predastore ]; then
        $SUDO chown -R "spinifex-storage:$SPINIFEX_GROUP" /etc/spinifex/predastore \
            || fatal "Failed to set ownership on /etc/spinifex/predastore"
@@ -184,7 +184,7 @@ fi
 log "stopping services"
 run sudo systemctl stop spinifex.target 2>/dev/null || true
 if ! $DRY_RUN; then
-    for unit in spinifex-predastore spinifex-viperblock spinifex-nats; do
+    for unit in spinifex-predastore spinifex-viperblock spinifex-sqs-store spinifex-nats; do
        if systemctl is-active --quiet "$unit"; then
            sudo systemctl stop "$unit" || true
        fi
@@ -415,7 +415,7 @@ fi
 # userdel warns about a missing home. Order here is cosmetic, not correctness.
 # ---------------------------------------------------------------------------
 log "removing service users and groups"
-for svc in nats gw daemon storage northstar viperblock vpcd ui; do
+for svc in nats sqsstore gw daemon storage northstar viperblock vpcd ui; do
    user="spinifex-${svc}"
    if id -u "$user" >/dev/null 2>&1; then
        run sudo userdel "$user" 2>/dev/null || log "  NOTE: could not remove user $user"
@@ -512,7 +512,7 @@ if ! $DRY_RUN; then
    for u in openvswitch-switch ovn-controller; do
        systemctl is-failed --quiet "$u" 2>/dev/null && LEFT+=("$u.service is failed")
    done
-    for svc in nats gw daemon storage northstar viperblock vpcd ui; do
+    for svc in nats sqsstore gw daemon storage northstar viperblock vpcd ui; do
        id -u "spinifex-${svc}" >/dev/null 2>&1 && LEFT+=("user spinifex-${svc}")
    done
    if command -v nft >/dev/null 2>&1 && sudo nft list table inet spinifex_filter >/dev/null 2>&1; then
@@ -12,21 +12,6 @@ import (
 	"github.com/stretchr/testify/require"
 )

-// strictDeleteStore answers a delete of an absent key with NoSuchKey, which is
-// what Predastore does. MemoryObjectStore silently succeeds instead, so a
-// handler that treats a missing key as a failure passes every test and still
-// breaks in production.
-type strictDeleteStore struct {
-	*objectstore.MemoryObjectStore
-}
-
-func (s strictDeleteStore) DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput) (*awss3.DeleteObjectOutput, error) {
-	if _, err := s.HeadObject(ctx, &awss3.HeadObjectInput{Bucket: input.Bucket, Key: input.Key}); err != nil {
-		return nil, err
-	}
-	return s.MemoryObjectStore.DeleteObject(ctx, input)
-}
-
 // TestDeregisterImage_NoLegacyConfig covers every AMI created since the
 // metadata store landed: they have no {imageID}/config.json, so deleting one
 // answers NoSuchKey. Treating that as a failure returned InternalError after
@@ -34,7 +19,7 @@ func (s strictDeleteStore) DeleteObject(ctx context.Context, input *awss3.Delete
 // InvalidAMIID.NotFound for an image it had just been told it could not delete.
 func TestDeregisterImage_NoLegacyConfig(t *testing.T) {
 	mem := objectstore.NewMemoryObjectStore()
-	svc := NewImageServiceImplWithStore(strictDeleteStore{MemoryObjectStore: mem}, testBucket)
+	svc := NewImageServiceImplWithStore(mem, testBucket)

 	amiID := "ami-nolegacy001"
 	createTestAMIConfigWithOwner(t, mem, amiID, "no-legacy-config", testAccountID)
@@ -50,3 +35,27 @@ func TestDeregisterImage_NoLegacyConfig(t *testing.T) {
 	require.Error(t, getErr)
 	assert.True(t, objectstore.IsNoSuchKeyError(getErr), "the AMI document must still be deleted")
 }
+
+// racingDeleteStore lets a concurrent deleter win every DeleteObject, so the
+// caller's own delete answers NoSuchKey as Predastore would.
+type racingDeleteStore struct {
+	*objectstore.MemoryObjectStore
+}
+
+func (s racingDeleteStore) DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput) (*awss3.DeleteObjectOutput, error) {
+	_, _ = s.MemoryObjectStore.DeleteObject(ctx, input)
+	return s.MemoryObjectStore.DeleteObject(ctx, input)
+}
+
+// Two deregisters that both pass the ownership read race to delete the
+// document; the loser's NoSuchKey means the image is already gone.
+func TestDeregisterImage_ConcurrentlyDeletedDocumentIsSuccess(t *testing.T) {
+	mem := objectstore.NewMemoryObjectStore()
+	svc := NewImageServiceImplWithStore(racingDeleteStore{MemoryObjectStore: mem}, testBucket)
+
+	amiID := "ami-racing00001"
+	createTestAMIConfigWithOwner(t, mem, amiID, "racing-deregister", testAccountID)
+
+	_, err := svc.DeregisterImage(context.Background(), &ec2.DeregisterImageInput{ImageId: aws.String(amiID)}, testAccountID)
+	require.NoError(t, err)
+}
@@ -1161,7 +1161,7 @@ func (s *ImageServiceImpl) DeregisterImage(ctx context.Context, input *ec2.Dereg
 		return nil, err
 	}

-	if err := s.metadata.DeleteAMI(ctx, imageID); err != nil {
+	if err := s.metadata.DeleteAMI(ctx, imageID); err != nil && !objectstore.IsNoSuchKeyError(err) {
 		slog.ErrorContext(ctx, "DeregisterImage: failed to delete AMI document", "imageId", imageID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
@@ -793,7 +793,7 @@ func (s *SnapshotServiceImpl) DeleteSnapshot(ctx context.Context, input *ec2.Del
 			Bucket: aws.String(s.config.Predastore.Bucket),
 			Key:    obj.Key,
 		})
-		if err != nil {
+		if err != nil && !objectstore.IsNoSuchKeyError(err) {
 			slog.ErrorContext(ctx, "DeleteSnapshot failed to delete object", "key", *obj.Key, "err", err)
 			sweepErr = errors.Join(sweepErr, err)
 		}
@@ -808,7 +808,7 @@ func (s *SnapshotServiceImpl) DeleteSnapshot(ctx context.Context, input *ec2.Del
 	// explicitly, and last: a crash with it still present leaves a snapshot that
 	// DescribeSnapshots shows and DeleteSnapshot can retry, rather than chunks
 	// nothing enumerates.
-	if err := s.metadata.DeleteSnapshot(ctx, accountID, snapshotID); err != nil {
+	if err := s.metadata.DeleteSnapshot(ctx, accountID, snapshotID); err != nil && !objectstore.IsNoSuchKeyError(err) {
 		slog.ErrorContext(ctx, "DeleteSnapshot failed to delete metadata document", "snapshotId", snapshotID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
@@ -155,6 +155,48 @@ func (s *deleteFailingStore) DeleteObject(ctx context.Context, input *s3.DeleteO
 	return s.MemoryObjectStore.DeleteObject(ctx, input)
 }

+// racingDeleteStore lets a concurrent deleter win every DeleteObject once race
+// is set, so the caller's own delete answers NoSuchKey as Predastore would.
+type racingDeleteStore struct {
+	*objectstore.MemoryObjectStore
+
+	race bool
+}
+
+func (s *racingDeleteStore) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+	if s.race {
+		_, _ = s.MemoryObjectStore.DeleteObject(ctx, input)
+	}
+	return s.MemoryObjectStore.DeleteObject(ctx, input)
+}
+
+// A chunk or document another deleter already removed is deleted, not a
+// failure: the sweep and the document delete must both finish.
+func TestDeleteSnapshot_ConcurrentlyDeletedKeysAreDone(t *testing.T) {
+	ctx := context.Background()
+	objects := &racingDeleteStore{MemoryObjectStore: objectstore.NewMemoryObjectStore()}
+	cfg := &config.Config{Predastore: config.PredastoreConfig{Bucket: "test-bucket"}}
+	svc := NewSnapshotServiceImplWithStore(cfg, objects, nil)
+	svc.SetEBSProvider(ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{}))
+	createTestVolume(t, svc, objects.MemoryObjectStore, "vol-race", 8)
+
+	snap, err := svc.CreateSnapshot(ctx, &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-race")}, testAccountID)
+	require.NoError(t, err)
+	snapshotID := aws.StringValue(snap.SnapshotId)
+	_, err = objects.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String("test-bucket"), Key: aws.String(snapshotID + "/chunk-0"),
+		Body: strings.NewReader("blocks"),
+	})
+	require.NoError(t, err)
+	objects.race = true
+
+	_, err = svc.DeleteSnapshot(ctx, &ec2.DeleteSnapshotInput{SnapshotId: snap.SnapshotId}, testAccountID)
+	require.NoError(t, err)
+
+	_, err = svc.metadata.GetSnapshot(ctx, testAccountID, snapshotID)
+	assert.True(t, objectstore.IsNoSuchKeyError(err), "the document must be gone, got %v", err)
+}
+
 // Isolation must be enforced *by* the read, not by a filter after it. A
 // whole-tree listing plus a correct in-memory filter would return the same
 // snapshots, so this asserts the access pattern: the listing prefix is the
@@ -9,6 +9,7 @@ import (
 	"io"
 	"log/slog"
 	"maps"
+	"regexp"
 	"slices"
 	"strings"

@@ -89,6 +90,29 @@ func describeTagsResourceType(resourceID string) string {
 	return string(kind)
 }

+// Taggable resources whose ids arn.EC2TypeForID does not type, because their
+// ARN type is named by a parameter rather than derived from the id.
+var untypedTaggablePrefixes = []string{"lt-", "cr-", "sir-"}
+
+// A recognised prefix followed by characters that cannot break the tag store's
+// KV key, which a wildcard, space or dot in the id would.
+var taggableIDShape = regexp.MustCompile(`^[a-z]+-[A-Za-z0-9_-]+$`)
+
+// isTaggableID reports whether CreateTags accepts resourceID. Only new writes
+// are checked: entries already stored under an untypable id are left in place
+// and stay visible to DescribeTags and removable by DeleteTags.
+func isTaggableID(resourceID string) bool {
+	if !taggableIDShape.MatchString(resourceID) {
+		return false
+	}
+	if _, ok := arn.EC2TypeForID(resourceID); ok {
+		return true
+	}
+	return slices.ContainsFunc(untypedTaggablePrefixes, func(p string) bool {
+		return strings.HasPrefix(resourceID, p)
+	})
+}
+
 // getTagsKey returns the S3 key for storing tags for a resource, scoped by account.
 func getTagsKey(accountID, resourceID string) string {
 	return "tags/" + accountID + "/" + resourceID + ".json"
@@ -207,6 +231,13 @@ func (s *TagsServiceImpl) CreateTags(ctx context.Context, input *ec2.CreateTagsI

 	slog.InfoContext(ctx, "CreateTags request", "resources", len(input.Resources), "tags", len(input.Tags))

+	for _, resourceID := range input.Resources {
+		if resourceID != nil && !isTaggableID(*resourceID) {
+			slog.InfoContext(ctx, "CreateTags rejected unrecognised resource id", "resourceId", *resourceID)
+			return nil, errors.New(awserrors.ErrorInvalidID)
+		}
+	}
+
 	for _, resourceID := range input.Resources {
 		if resourceID == nil {
 			continue
@@ -140,6 +140,63 @@ func TestCreateTags_MissingResources(t *testing.T) {
 	assert.Contains(t, err.Error(), awserrors.ErrorMissingParameter)
 }

+// An id with no recognised prefix is rejected before any resource in the
+// request is written, so a valid id alongside it is not tagged either.
+func TestCreateTags_UnrecognisedPrefixIsInvalidID(t *testing.T) {
+	svc, _ := setupTestTagsService(t)
+
+	for _, id := range []string{"not-a-real-id", "", "vpc", "vpc-", "vpc-*", "vpc-a b", "vpc->", "vpc-a.b"} {
+		_, err := svc.CreateTags(context.Background(), &ec2.CreateTagsInput{
+			Resources: []*string{aws.String("vpc-0abc"), aws.String(id)},
+			Tags:      []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("x")}},
+		}, testAccountID)
+		require.Error(t, err, id)
+		assert.Equal(t, awserrors.ErrorInvalidID, err.Error(), id)
+	}
+
+	out, err := svc.DescribeTags(context.Background(), &ec2.DescribeTagsInput{}, testAccountID)
+	require.NoError(t, err)
+	assert.Empty(t, out.Tags)
+}
+
+// Launch templates, capacity reservations and spot requests have no ARN-type
+// prefix mapping but are still taggable resources. A nil entry is skipped.
+func TestCreateTags_AcceptsTaggablePrefixesWithoutARNType(t *testing.T) {
+	svc, _ := setupTestTagsService(t)
+
+	ids := []string{"lt-0abc", "cr-0abc", "sir-0abc", "eipalloc-0abc", "pg-0abc"}
+	resources := []*string{nil}
+	for _, id := range ids {
+		resources = append(resources, aws.String(id))
+	}
+	_, err := svc.CreateTags(context.Background(), &ec2.CreateTagsInput{
+		Resources: resources,
+		Tags:      []*ec2.Tag{{Key: aws.String("Name"), Value: aws.String("x")}},
+	}, testAccountID)
+	require.NoError(t, err)
+
+	out, err := svc.DescribeTags(context.Background(), &ec2.DescribeTagsInput{}, testAccountID)
+	require.NoError(t, err)
+	assert.Len(t, out.Tags, len(ids))
+}
+
+// Tags stored for an untypable id before validation existed stay removable.
+func TestDeleteTags_UntypableIDStillDeletable(t *testing.T) {
+	svc, _ := setupTestTagsService(t)
+	require.NoError(t, svc.mutateTags(context.Background(), testAccountID, "not-a-real-id", func(tags map[string]string) {
+		tags["Name"] = "residue"
+	}))
+
+	_, err := svc.DeleteTags(context.Background(), &ec2.DeleteTagsInput{
+		Resources: []*string{aws.String("not-a-real-id")},
+	}, testAccountID)
+	require.NoError(t, err)
+
+	out, err := svc.DescribeTags(context.Background(), &ec2.DescribeTagsInput{}, testAccountID)
+	require.NoError(t, err)
+	assert.Empty(t, out.Tags)
+}
+
 // TestCreateTags_MissingTags tests creating tags without tags.
 func TestCreateTags_MissingTags(t *testing.T) {
 	svc, _ := setupTestTagsService(t)
@@ -92,6 +92,42 @@ func TestDeleteVolume_UnreadableDocumentIsInternalError(t *testing.T) {
 	require.NotNil(t, provVol, "the provider's data must survive a failed delete")
 }

+// racingDeleteStore lets a concurrent deleter win every DeleteObject, so the
+// caller's own delete answers NoSuchKey as Predastore would.
+type racingDeleteStore struct {
+	*objectstore.MemoryObjectStore
+}
+
+func (s *racingDeleteStore) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+	_, _ = s.MemoryObjectStore.DeleteObject(ctx, input)
+	return s.MemoryObjectStore.DeleteObject(ctx, input)
+}
+
+// A document another delete removed after this one read it is deleted, so the
+// call must succeed rather than answer InternalError for a finished delete.
+func TestDeleteVolume_ConcurrentlyDeletedDocumentIsSuccess(t *testing.T) {
+	memory := objectstore.NewMemoryObjectStore()
+	svc := newTestVolumeServiceWithStore("ap-southeast-2a", memory)
+	svc.snapshotKV = setupTestVolumeKV(t)
+
+	volumeID := "vol-racingdelete00"
+	createVolumeInStoreWithMeta(t, svc, memory, volumeID, ebsmetadata.Volume{
+		VolumeID: volumeID, CapacityGiB: 10, State: "available",
+	})
+
+	racing := &racingDeleteStore{MemoryObjectStore: memory}
+	svc.store = racing
+	svc.metadata = ebsmetadata.NewStore(racing, "test-bucket")
+
+	_, err := svc.DeleteVolume(context.Background(), &ec2.DeleteVolumeInput{
+		VolumeId: aws.String(volumeID),
+	}, testVolAccountID)
+	require.NoError(t, err)
+
+	_, getErr := ebsmetadata.NewStore(memory, "test-bucket").GetVolume(context.Background(), testVolAccountID, volumeID)
+	assert.True(t, objectstore.IsNoSuchKeyError(getErr), "the document must be gone, got %v", getErr)
+}
+
 // TestDeleteVolume_AbsentDocumentIsStillNotFound guards the other side of the
 // fix: absence (NoSuchKey) must keep returning InvalidVolume.NotFound, not be
 // swept up into the new InternalError branch.
@@ -1135,7 +1135,7 @@ func (s *VolumeServiceImpl) DeleteVolume(ctx context.Context, input *ec2.DeleteV
 		// it gets back, so a bare code makes a persistent fault unreadable.
 		return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "delete volume %s: %v", volumeID, err)
 	}
-	if err := s.metadata.DeleteVolume(ctx, accountID, volumeID); err != nil {
+	if err := s.metadata.DeleteVolume(ctx, accountID, volumeID); err != nil && !objectstore.IsNoSuchKeyError(err) {
 		return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "delete volume metadata %s: %v", volumeID, err)
 	}

@@ -853,7 +853,10 @@ func (reg *Registry) deleteUploadBytesAt(ctx context.Context, key string) error
 		Bucket: aws.String(reg.bucket()),
 		Key:    aws.String(key),
 	})
-	return err
+	if err != nil && !objectstore.IsNoSuchKeyError(err) {
+		return err
+	}
+	return nil
 }

 func (reg *Registry) writeJSON(ctx context.Context, w http.ResponseWriter, status int, v any) {
@@ -273,7 +273,7 @@ func (reg *Registry) reclaimManifest(ctx context.Context, account, repo, digest
 	if _, err := reg.Store.DeleteObject(ctx, &s3.DeleteObjectInput{
 		Bucket: aws.String(ecr.AccountBucket(account)),
 		Key:    aws.String(ecr.ManifestKey(repo, digest)),
-	}); err != nil {
+	}); err != nil && !objectstore.IsNoSuchKeyError(err) {
 		slog.WarnContext(ctx, "ECR/GC: manifest object delete failed", "repo", repo, "digest", digest, "err", err)
 	}

@@ -292,7 +292,7 @@ func (reg *Registry) reclaimManifest(ctx context.Context, account, repo, digest
 		if _, err := reg.Store.DeleteObject(ctx, &s3.DeleteObjectInput{
 			Bucket: aws.String(ecr.AccountBucket(account)),
 			Key:    aws.String(ecr.BlobKey(child)),
-		}); err != nil {
+		}); err != nil && !objectstore.IsNoSuchKeyError(err) {
 			slog.WarnContext(ctx, "ECR/GC: blob delete failed", "digest", child, "err", err)
 		}
 	}
@@ -91,13 +91,14 @@ func containerDefsFromAWS(in []*ecs.ContainerDefinition) []taskdefinition.Contai
 		if c == nil {
 			continue
 		}
+		// AWS assumes a container is essential when the parameter is omitted.
 		def := taskdefinition.Container{
 			Name:      aws.StringValue(c.Name),
 			Image:     aws.StringValue(c.Image),
 			CPU:       int(aws.Int64Value(c.Cpu)),
 			MemoryMiB: int(aws.Int64Value(c.Memory)),
 			GPU:       gpuCountFromResourceRequirements(c.ResourceRequirements),
-			Essential: aws.BoolValue(c.Essential),
+			Essential: c.Essential == nil || *c.Essential,
 			Command:   awsStringSlice(c.Command),
 		}
 		if c.Environment != nil {
@@ -163,3 +163,20 @@ func TestTaskDefReservedGPU(t *testing.T) {
 	empty := &taskdefinition.Record{Containers: []taskdefinition.Container{{CPU: 128}}}
 	assert.Zero(t, empty.ReservedGPU())
 }
+
+// AWS assumes a container is essential when the essential parameter is omitted,
+// so an omitted value must describe back as true while explicit values are kept.
+func TestContainerDefsFromAWS_EssentialDefaultsTrue(t *testing.T) {
+	defs := containerDefsFromAWS([]*ecs.ContainerDefinition{
+		{Name: aws.String("omitted"), Image: aws.String("registry/app:1")},
+		{Name: aws.String("sidecar"), Image: aws.String("registry/side:1"), Essential: aws.Bool(false)},
+		{Name: aws.String("explicit"), Image: aws.String("registry/app:1"), Essential: aws.Bool(true)},
+	})
+	require.Len(t, defs, 3)
+	assert.True(t, defs[0].Essential)
+	assert.True(t, aws.BoolValue(containerToAWS(defs[0]).Essential))
+	assert.True(t, containerToAssign(defs[0]).Essential)
+	assert.False(t, defs[1].Essential)
+	assert.False(t, aws.BoolValue(containerToAWS(defs[1]).Essential))
+	assert.True(t, defs[2].Essential)
+}
@@ -153,6 +153,13 @@ type ClusterRecord struct {
 	// an ASG primitive).
 	CapacityProviders               []string                       `json:"capacityProviders,omitempty"`
 	DefaultCapacityProviderStrategy []CapacityProviderStrategyItem `json:"defaultCapacityProviderStrategy,omitempty"`
+	Settings                        []ClusterSetting               `json:"settings,omitempty"`
+}
+
+// ClusterSetting is one CreateCluster setting, stored and described back.
+type ClusterSetting struct {
+	Name  string `json:"name"`
+	Value string `json:"value"`
 }

 // CapacityProviderStrategyItem is one entry of a capacity-provider strategy:
@@ -258,6 +258,11 @@ func (s *Service) CreateCluster(ctx context.Context, input *ecs.CreateClusterInp
 			Status:    ClusterStatusActive,
 			CreatedAt: time.Now().UTC(),
 		}
+		settings, err := clusterSettingsFromAWS(input.Settings)
+		if err != nil {
+			return nil, err
+		}
+		rec.Settings = settings
 		if len(input.Tags) > 0 {
 			rec.Tags = map[string]string{}
 			for _, t := range input.Tags {
@@ -271,8 +276,8 @@ func (s *Service) CreateCluster(ctx context.Context, input *ecs.CreateClusterInp
 	return &ecs.CreateClusterOutput{Cluster: rec.toAWS()}, nil
 }

-// DescribeClusters returns meta for the named clusters; unknown names are
-// silently skipped (AWS returns them under "failures", omitted here in v1).
+// DescribeClusters returns meta for the named clusters; unknown names surface
+// as MISSING failures. Settings are only returned under Include=SETTINGS.
 func (s *Service) DescribeClusters(ctx context.Context, input *ecs.DescribeClustersInput, accountID string) (*ecs.DescribeClustersOutput, error) {
 	kv, err := s.bucket(ctx, accountID)
 	if err != nil {
@@ -282,6 +287,7 @@ func (s *Service) DescribeClusters(ctx context.Context, input *ecs.DescribeClust
 	if len(names) == 0 {
 		names = []string{defaultCluster}
 	}
+	includeSettings := slices.Contains(awsStringSlice(input.Include), ecs.ClusterFieldSettings)
 	out := &ecs.DescribeClustersOutput{}
 	for _, name := range names {
 		name = ClusterShortName(name)
@@ -291,9 +297,13 @@ func (s *Service) DescribeClusters(ctx context.Context, input *ecs.DescribeClust
 			return nil, err
 		}
 		if !found {
+			out.Failures = append(out.Failures, &ecs.Failure{Arn: aws.String(ClusterARN(s.region, accountID, name)), Reason: aws.String("MISSING")})
 			continue
 		}
 		c := rec.toAWS()
+		if !includeSettings {
+			c.Settings = nil
+		}
 		if err := s.applyClusterCounts(ctx, kv, name, c); err != nil {
 			return nil, err
 		}
@@ -382,9 +392,31 @@ func (r *ClusterRecord) toAWS() *ecs.Cluster {
 	for _, item := range r.DefaultCapacityProviderStrategy {
 		c.DefaultCapacityProviderStrategy = append(c.DefaultCapacityProviderStrategy, item.toAWS())
 	}
+	for _, st := range r.Settings {
+		c.Settings = append(c.Settings, &ecs.ClusterSetting{Name: aws.String(st.Name), Value: aws.String(st.Value)})
+	}
 	return c
 }

+// clusterSettingsFromAWS validates CreateCluster settings against the names
+// and values AWS accepts, so an unsupported setting is refused, not dropped.
+func clusterSettingsFromAWS(in []*ecs.ClusterSetting) ([]ClusterSetting, error) {
+	var out []ClusterSetting
+	for _, st := range in {
+		if st == nil {
+			continue
+		}
+		name, value := aws.StringValue(st.Name), aws.StringValue(st.Value)
+		if name != ecs.ClusterSettingNameContainerInsights || !slices.Contains(containerInsightsValues, value) {
+			return nil, errors.New(awserrors.ErrorECSInvalidParameter)
+		}
+		out = append(out, ClusterSetting{Name: name, Value: value})
+	}
+	return out, nil
+}
+
+var containerInsightsValues = []string{"enabled", "disabled", "enhanced"}
+
 // tagsToAWS converts a stored tag map into the AWS list form with a stable
 // key order so Describe output is deterministic.
 func tagsToAWS(tags map[string]string) []*ecs.Tag {
@@ -548,7 +580,16 @@ func (s *Service) DescribeTaskDefinition(ctx context.Context, input *ecs.Describ
 	if err != nil {
 		return nil, err
 	}
-	rec, err := s.taskDefs.Resolve(ctx, kv, aws.StringValue(input.TaskDefinition))
+	ref := aws.StringValue(input.TaskDefinition)
+	if family, _ := taskdefinition.ParseRef(ref); family == "" {
+		return nil, errors.New(awserrors.ErrorECSInvalidParameter)
+	}
+	// AWS answers an unknown revision with ClientException, not the
+	// InvalidParameterException that launches get for the same reference.
+	rec, err := s.taskDefs.Resolve(ctx, kv, ref)
+	if err != nil && err.Error() == awserrors.ErrorECSInvalidParameter {
+		return nil, errors.New(awserrors.ErrorECSClientException)
+	}
 	if err != nil {
 		return nil, err
 	}
@@ -11,6 +11,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/ecs"
 	ecsv1 "github.com/mulgadc/spinifex/contracts/ecs/v1"
 	"github.com/mulgadc/spinifex/internal/testkit"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -61,6 +62,48 @@ func TestService_DescribeClusters_DefaultAndMissing(t *testing.T) {
 	miss, err := svc.DescribeClusters(context.Background(), &ecs.DescribeClustersInput{Clusters: []*string{aws.String("nope")}}, testAccountID)
 	require.NoError(t, err)
 	assert.Empty(t, miss.Clusters)
+	require.Len(t, miss.Failures, 1)
+	assert.Equal(t, ClusterARN(testRegion, testAccountID, "nope"), aws.StringValue(miss.Failures[0].Arn))
+	assert.Equal(t, "MISSING", aws.StringValue(miss.Failures[0].Reason))
+}
+
+func TestService_CreateCluster_StoresSettings(t *testing.T) {
+	svc, _ := newTestService(t)
+	setting := &ecs.ClusterSetting{Name: aws.String(ecs.ClusterSettingNameContainerInsights), Value: aws.String("enabled")}
+	created, err := svc.CreateCluster(context.Background(), &ecs.CreateClusterInput{
+		ClusterName: aws.String("web"), Settings: []*ecs.ClusterSetting{setting},
+	}, testAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, []*ecs.ClusterSetting{setting}, created.Cluster.Settings)
+
+	withSettings, err := svc.DescribeClusters(context.Background(), &ecs.DescribeClustersInput{
+		Clusters: []*string{aws.String("web")}, Include: []*string{aws.String(ecs.ClusterFieldSettings)},
+	}, testAccountID)
+	require.NoError(t, err)
+	require.Len(t, withSettings.Clusters, 1)
+	assert.Equal(t, []*ecs.ClusterSetting{setting}, withSettings.Clusters[0].Settings)
+
+	plain, err := svc.DescribeClusters(context.Background(), &ecs.DescribeClustersInput{Clusters: []*string{aws.String("web")}}, testAccountID)
+	require.NoError(t, err)
+	require.Len(t, plain.Clusters, 1)
+	assert.Empty(t, plain.Clusters[0].Settings)
+}
+
+func TestService_CreateCluster_RejectsUnsupportedSetting(t *testing.T) {
+	svc, _ := newTestService(t)
+	for _, st := range []*ecs.ClusterSetting{
+		{Name: aws.String("unknownSetting"), Value: aws.String("enabled")},
+		{Name: aws.String(ecs.ClusterSettingNameContainerInsights), Value: aws.String("sometimes")},
+	} {
+		_, err := svc.CreateCluster(context.Background(), &ecs.CreateClusterInput{
+			ClusterName: aws.String("web"), Settings: []*ecs.ClusterSetting{st},
+		}, testAccountID)
+		require.Error(t, err)
+		assert.Equal(t, awserrors.ErrorECSInvalidParameter, err.Error())
+	}
+	list, err := svc.ListClusters(context.Background(), &ecs.ListClustersInput{}, testAccountID)
+	require.NoError(t, err)
+	assert.Empty(t, list.ClusterArns)
 }

 func TestService_DescribeClusters_ReturnsTags(t *testing.T) {
@@ -71,11 +71,15 @@ func TestDescribeTaskDefinition_ResolvesEveryReferenceForm(t *testing.T) {
 		assert.Equal(t, want, aws.Int64Value(out.TaskDefinition.Revision), ref)
 	}

-	for _, ref := range []string{"app:3", "ghost", ""} {
+	for ref, want := range map[string]string{
+		"app:3": awserrors.ErrorECSClientException,
+		"ghost": awserrors.ErrorECSClientException,
+		"":      awserrors.ErrorECSInvalidParameter,
+	} {
 		_, err := svc.DescribeTaskDefinition(context.Background(),
 			&ecs.DescribeTaskDefinitionInput{TaskDefinition: aws.String(ref)}, testAccountID)
 		require.Error(t, err, ref)
-		assert.Equal(t, awserrors.ErrorECSInvalidParameter, err.Error(), ref)
+		assert.Equal(t, want, err.Error(), ref)
 	}
 }

@@ -46,7 +46,8 @@ backend {{.Name}}
    http-request return status {{.FixedResponse.StatusCode}}{{if .FixedResponse.ContentType}} content-type "{{.FixedResponse.ContentType}}"{{end}}{{if .FixedResponse.Body}} string "{{.FixedResponse.Body}}"{{end}}
 {{- else}}
    balance roundrobin{{if .HealthCheckPath}}
-    option httpchk GET {{.HealthCheckPath}}
+    option httpchk
+    http-check send meth GET ver HTTP/1.1 hdr Host %[bc_dst]%[bc_dst_port,regsub(^,:),regsub(^:{{.HealthCheckDefaultPort}}$,)] hdr User-Agent ELB-HealthChecker/2.0 uri {{.HealthCheckPath}}
    http-check expect status {{.HealthCheckMatcher}}{{end}}
 {{- range .Servers}}
    server {{.Name}} {{.Addr}}:{{.Port}} check{{if .Secure}} check-ssl ssl verify none{{end}} inter {{.CheckInterval}}s fall {{.Fall}} rise {{.Rise}}
@@ -102,9 +103,12 @@ type HAProxyBackend struct {
 	Name               string
 	HealthCheckPath    string
 	HealthCheckMatcher string
-	Servers            []HAProxyServer
-	FixedResponse      *HAProxyFixedResponse
-	Redirect           *HAProxyRedirect
+	// HealthCheckDefaultPort is left out of the check's Host header, as AWS
+	// sends "Host: <ip>" on the scheme's default port and "<ip>:<port>" otherwise.
+	HealthCheckDefaultPort int64
+	Servers                []HAProxyServer
+	FixedResponse          *HAProxyFixedResponse
+	Redirect               *HAProxyRedirect
 }

 // HAProxyFixedResponse renders an `http-request return` directive. All fields are validated before use.
@@ -185,10 +189,15 @@ func buildHAProxyConfig(lb *LoadBalancerRecord, listeners []*ListenerRecord, tgB
 			return
 		}

+		secure := strings.EqualFold(tg.Protocol, ProtocolHTTPS)
 		backend := HAProxyBackend{
-			Name:               backendName,
-			HealthCheckPath:    tg.HealthCheck.Path,
-			HealthCheckMatcher: tg.HealthCheck.Matcher,
+			Name:                   backendName,
+			HealthCheckPath:        tg.HealthCheck.Path,
+			HealthCheckMatcher:     tg.HealthCheck.Matcher,
+			HealthCheckDefaultPort: 80,
+		}
+		if secure {
+			backend.HealthCheckDefaultPort = 443
 		}

 		for _, target := range tg.Targets {
@@ -202,7 +211,7 @@ func buildHAProxyConfig(lb *LoadBalancerRecord, listeners []*ListenerRecord, tgB
 				CheckInterval: tg.HealthCheck.IntervalSeconds,
 				Fall:          tg.HealthCheck.UnhealthyThreshold,
 				Rise:          tg.HealthCheck.HealthyThreshold,
-				Secure:        strings.EqualFold(tg.Protocol, ProtocolHTTPS),
+				Secure:        secure,
 			})
 		}
 		cfg.Backends = append(cfg.Backends, backend)
@@ -57,8 +57,13 @@ func TestGenerateHAProxyConfig_SingleListenerAndBackend(t *testing.T) {
 	assert.Contains(t, config, "default_backend")

 	// Verify backend
-	assert.Contains(t, config, "option httpchk GET /health")
-	assert.Contains(t, config, "http-check expect status 200")
+	// AWS shape: HTTP/1.1, Host is the target IP (plus port when not 80), ELB User-Agent.
+	assert.Contains(t, config, "    option httpchk\n"+
+		"    http-check send meth GET ver HTTP/1.1"+
+		" hdr Host %[bc_dst]%[bc_dst_port,regsub(^,:),regsub(^:80$,)]"+
+		" hdr User-Agent ELB-HealthChecker/2.0 uri /health\n"+
+		"    http-check expect status 200\n")
+	assert.NotContains(t, config, "option httpchk GET")
 	assert.Contains(t, config, "10.0.1.10:8080")
 	assert.Contains(t, config, "10.0.1.11:8080")
 	assert.Contains(t, config, "check inter 30s fall 2 rise 5")
@@ -97,6 +102,28 @@ func TestGenerateHAProxyConfig_HTTPSBackendReencrypts(t *testing.T) {
 	config, _, err := GenerateHAProxyConfigWithCerts(lb, listeners, tgByArn, nil, "10.0.2.5", nil)
 	require.NoError(t, err)
 	assert.Contains(t, config, "10.0.2.10:30081 check check-ssl ssl verify none inter 10s")
+	// 443 is the HTTPS default port, so AWS omits it from the Host header.
+	assert.Contains(t, config, "hdr Host %[bc_dst]%[bc_dst_port,regsub(^,:),regsub(^:443$,)]")
+}
+
+// A '#' starts a comment in haproxy config, so uri must come last on the send
+// line: a fragment in the path then only trims the uri, not the headers.
+func TestGenerateHAProxyConfig_HealthCheckURIIsLastOnSendLine(t *testing.T) {
+	t.Parallel()
+	lb := &LoadBalancerRecord{LoadBalancerID: "lb-frag"}
+	tgArn := "arn:tg-frag"
+	listeners := []*ListenerRecord{
+		{ListenerArn: "arn:lst-frag", Port: 80, DefaultActions: []ListenerAction{{Type: ActionTypeForward, TargetGroupArn: tgArn}}},
+	}
+	hc := DefaultHealthCheck()
+	hc.Path = "/healthz?probe=lb#frag"
+	tgByArn := map[string]*TargetGroupRecord{
+		tgArn: {HealthCheck: hc, Targets: []Target{{Id: "i-f", Port: 80, PrivateIP: "10.0.0.9", HealthState: TargetHealthHealthy}}},
+	}
+
+	config, _, err := GenerateHAProxyConfigWithCerts(lb, listeners, tgByArn, nil, "0.0.0.0", nil)
+	require.NoError(t, err)
+	assert.Contains(t, config, "hdr User-Agent ELB-HealthChecker/2.0 uri /healthz?probe=lb#frag\n")
 }

 func TestGenerateHAProxyConfig_MultipleListeners(t *testing.T) {
@@ -519,8 +546,8 @@ func TestGenerateALBConfig_StillWorks(t *testing.T) {

 	assert.Contains(t, config, "mode http")
 	assert.Contains(t, config, "option httplog")
-	assert.Contains(t, config, "option httpchk GET /health")
-	assert.Contains(t, config, "http-check expect status 200")
+	assert.Contains(t, config, "ver HTTP/1.1 hdr Host %[bc_dst]")
+	assert.Contains(t, config, "uri /health\n    http-check expect status 200\n")
 	assert.NotContains(t, config, "mode tcp")
 	assert.NotContains(t, config, "option tcplog")
 }
@@ -424,6 +424,7 @@ var (
 	ErrorRoute53TooManyTagsForZone = "TooManyTags"
 	// ECS JSON-1.1 exception codes. ECS clients key on the "Exception"-suffixed
 	// __type; do not cross-wire with the EC2/ACM not-found codes above.
+	ErrorECSClientException                             = "ClientException"
 	ErrorECSClusterNotFound                             = "ClusterNotFoundException"
 	ErrorECSInvalidParameter                            = "InvalidParameterException"
 	ErrorECSServiceNotFound                             = "ServiceNotFoundException"
@@ -724,6 +725,33 @@ func ResolveRetryAfter(err error) (time.Duration, bool) {
 	return 0, false
 }

+// fieldsError attaches extra response-body fields to an error without changing
+// its text, for protocols that carry them (e.g. a REST-JSON throttle Reason).
+type fieldsError struct {
+	err    error
+	fields map[string]string
+}
+
+func (e *fieldsError) Error() string { return e.err.Error() }
+func (e *fieldsError) Unwrap() error { return e.err }
+
+// WithFields attaches extra top-level response-body fields to err. Protocols
+// without extra fields ignore them.
+func WithFields(err error, fields map[string]string) error {
+	if err == nil || len(fields) == 0 {
+		return err
+	}
+	return &fieldsError{err: err, fields: fields}
+}
+
+// ResolveFields returns the fields attached to err via WithFields, if any.
+func ResolveFields(err error) map[string]string {
+	if fe, ok := errors.AsType[*fieldsError](err); ok {
+		return fe.fields
+	}
+	return nil
+}
+
 // errorLookupByService overrides ErrorLookup's message and HTTP status for a
 // (service, code) pair whose wire code is shared by services with different
 // canonical wording — e.g. ACM and EKS both use "ResourceInUseException".
@@ -1214,6 +1242,7 @@ var ErrorLookup = map[string]ErrorMessage{
 	ErrorResourceNotFound:                                      {HTTPCode: 404, Message: "The specified resource was not found."},
 	ErrorEKSResourceInUse:                                      {HTTPCode: 409, Message: "A cluster with this name already exists in this account and Region. Delete the existing cluster, or choose a different name."},
 	ErrorEKSResourceNotFound:                                   {HTTPCode: 404, Message: "The specified cluster could not be found. You can view your available clusters with ListClusters. Amazon EKS clusters are Region specific."},
+	ErrorECSClientException:                                    {HTTPCode: 400, Message: "Unable to describe task definition."},
 	ErrorECSClusterNotFound:                                    {HTTPCode: 400, Message: "The specified cluster was not found. You can view your available clusters with ListClusters. Amazon ECS clusters are Region specific."},
 	ErrorECSInvalidParameter:                                   {HTTPCode: 400, Message: "The specified parameter is not valid. Review the available parameters for the API request."},
 	ErrorECSServiceNotFound:                                    {HTTPCode: 400, Message: "The specified service was not found. You can view your available services with ListServices. Amazon ECS services are cluster specific and Region specific."},
@@ -335,3 +335,21 @@ func TestIsNotFound(t *testing.T) {
 		})
 	}
 }
+
+func TestWithFields_SurvivesWrappingAndKeepsTheCode(t *testing.T) {
+	err := fmt.Errorf("handler: %w", WithFields(Errorf(ErrorInvalidParameterValue, "bad"), map[string]string{"Reason": "R"}))
+
+	code, message, ok := ResolveErrorDetail(err)
+	if !ok || code != ErrorInvalidParameterValue || message != "bad" {
+		t.Errorf("ResolveErrorDetail = %q, %q, %v", code, message, ok)
+	}
+	if got := ResolveFields(err); len(got) != 1 || got["Reason"] != "R" {
+		t.Errorf("ResolveFields = %v", got)
+	}
+	if got := ResolveFields(errors.New(ErrorInvalidParameterValue)); got != nil {
+		t.Errorf("unattached ResolveFields = %v", got)
+	}
+	if WithFields(nil, map[string]string{"a": "b"}) != nil {
+		t.Error("WithFields(nil) must stay nil")
+	}
+}
@@ -532,16 +532,31 @@ func (gw *GatewayConfig) writeSigV4Error(w http.ResponseWriter, r *http.Request,
 	}

 	svc := signedService(r)
+	if svc == "" {
+		// A request rejected while its signature was still being parsed has no
+		// service in context; its claimed scope still picks a registration's
+		// envelope, and nothing else.
+		if hint := credentialScopeService(r); hint != "" {
+			if _, ok := gw.registered(hint); ok {
+				svc = hint
+			}
+		}
+	}

-	// AWS JSON 1.1 services (bedrock family, EKS, …) need a JSON error body, or
-	// the SDK chokes deserializing our XML into its shape. An unserved scope is
-	// absent from jsonErrorService by definition, so its protocol is read off
-	// the request itself instead.
-	if gw.jsonErrorService(svc) || envelope.RequestSignalsJSONProtocol(r) {
-		w.Header().Set("Content-Type", envelope.JSONContentType)
-		w.Header().Set("X-Amzn-Errortype", envelope.JSONErrorType(errorCode))
-		w.WriteHeader(errorMsg.HTTPCode)
-		_, _ = w.Write(envelope.JSONBody(errorCode, errorMsg.Message))
+	// JSON services (bedrock family, EKS, registered JSON services, …) need a
+	// JSON error body in their own protocol, or the SDK chokes deserializing
+	// our XML into its shape. An unserved scope is absent from the tables by
+	// definition, so its protocol is read off the request itself instead.
+	p, isJSON := gw.errorProtocol(svc)
+	if !isJSON && envelope.RequestSignalsJSONProtocol(r) {
+		p, isJSON = envelope.ProtocolJSON11, true
+	}
+	if isJSON {
+		if p == envelope.ProtocolJSON10Query && errorCode == awserrors.ErrorRequestEntityTooLarge {
+			envelope.WriteTooLargeText(w, int64(sigv4.MaxPayloadLen))
+			return
+		}
+		gw.writeJSONError(w, p, svc, envelope.JSONError{Code: errorCode, Message: errorMsg.Message, Status: errorMsg.HTTPCode})
 		return
 	}

@@ -552,6 +567,21 @@ func (gw *GatewayConfig) writeSigV4Error(w http.ResponseWriter, r *http.Request,
 	_, _ = w.Write(xmlError)
 }

+// credentialScopeService returns the service named in the request's
+// Authorization credential scope, unverified, or "" when it has none.
+func credentialScopeService(r *http.Request) string {
+	_, rest, ok := strings.Cut(r.Header.Get("Authorization"), "Credential=")
+	if !ok {
+		return ""
+	}
+	scope, _, _ := strings.Cut(rest, ",")
+	parts := strings.Split(strings.TrimSpace(scope), "/")
+	if len(parts) != 5 {
+		return ""
+	}
+	return parts[3]
+}
+
 // signedService returns the credential-scope service the client signed with,
 // including a scope the gateway does not serve. A rejection has to render in
 // the client's format, and GetService reports those scopes as empty.
@@ -0,0 +1,153 @@
+// Drives the real AWS SDK v2 clients against the gateway's error envelopes and
+// asserts they decode them into their modelled errors.
+
+package gateway
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"maps"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/lambda"
+	lambdatypes "github.com/aws/aws-sdk-go-v2/service/lambda/types"
+	"github.com/aws/aws-sdk-go-v2/service/sqs"
+	sqstypes "github.com/aws/aws-sdk-go-v2/service/sqs/types"
+	"github.com/aws/smithy-go"
+	"github.com/mulgadc/bluebottle/pkg/sigv4"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// registerTestCodes adds codes only the SDK fixtures raise, as the Lambda and
+// SQS domains would, and removes them afterwards.
+func registerTestCodes(t *testing.T, codes map[string]awserrors.ErrorMessage) {
+	t.Helper()
+	maps.Copy(awserrors.ErrorLookup, codes)
+	t.Cleanup(func() {
+		for code := range codes {
+			delete(awserrors.ErrorLookup, code)
+		}
+	})
+}
+
+func sdkConfig(url string) aws.Config {
+	return aws.Config{
+		Region:           testRegion,
+		Credentials:      credentials.NewStaticCredentialsProvider(testAccessKey, testSecretKey, ""),
+		RetryMaxAttempts: 1,
+		BaseEndpoint:     aws.String(url),
+	}
+}
+
+// sdkApp serves fixture sqs (JSON 1.0) and lambda (REST-JSON) registrations
+// through the full gateway pipeline; their dispatchers fail with dispatchErr.
+func sdkApp(t *testing.T, dispatchErr error) aws.Config {
+	t.Helper()
+	c := &dispatchCapture{dispatchErr: dispatchErr}
+	_, app := dispatchSeamApp(t, allowAllECRDocs(),
+		envelopeRegistration("sqs", dispatch.ErrorEnvelopeJSON10Query, c, func(r *dispatch.Registration) {
+			r.QueryErrorCodes = map[string]string{"QueueDoesNotExist": "AWS.SimpleQueueService.NonExistentQueue"}
+			r.ErrorNamespace = "com.amazonaws.sqs"
+		}),
+		envelopeRegistration("lambda", dispatch.ErrorEnvelopeRESTJSON, c, nil))
+	srv := httptest.NewServer(app)
+	t.Cleanup(srv.Close)
+	return sdkConfig(srv.URL)
+}
+
+// throttleApp serves the gateway's throttle response for the signed service.
+func throttleApp(t *testing.T) aws.Config {
+	t.Helper()
+	c := &dispatchCapture{}
+	b := dispatch.NewBuilder()
+	require.NoError(t, b.Register(envelopeRegistration("sqs", dispatch.ErrorEnvelopeJSON10Query, c,
+		func(r *dispatch.Registration) { r.ErrorNamespace = "com.amazonaws.sqs" })))
+	require.NoError(t, b.Register(envelopeRegistration("lambda", dispatch.ErrorEnvelopeRESTJSON, c, nil)))
+	gw := &GatewayConfig{DisableLogging: true, Services: b.Build()}
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		svc := credentialScopeService(r)
+		gw.writeThrottleError(w, r.WithContext(context.WithValue(r.Context(), ctxService, svc)))
+	}))
+	t.Cleanup(srv.Close)
+	return sdkConfig(srv.URL)
+}
+
+func TestSDKv2_SQSDecodesJSON10QueryErrors(t *testing.T) {
+	registerTestCodes(t, map[string]awserrors.ErrorMessage{
+		"QueueDoesNotExist": {HTTPCode: 400, Message: "The specified queue does not exist."},
+	})
+	cfg := sdkApp(t, awserrors.Errorf("QueueDoesNotExist", "no such queue"))
+
+	_, err := sqs.NewFromConfig(cfg).GetQueueUrl(context.Background(), &sqs.GetQueueUrlInput{QueueName: aws.String("q")})
+	require.Error(t, err)
+
+	var modelled *sqstypes.QueueDoesNotExist
+	require.ErrorAs(t, err, &modelled, "the SDK must map __type to its modelled error: %v", err)
+	assert.Equal(t, "no such queue", aws.ToString(modelled.Message))
+
+	var apiErr smithy.APIError
+	require.ErrorAs(t, err, &apiErr)
+	assert.Equal(t, "AWS.SimpleQueueService.NonExistentQueue", apiErr.ErrorCode(), "awsQueryCompatible clients report the legacy code")
+	assert.Equal(t, smithy.FaultClient, apiErr.ErrorFault())
+}
+
+func TestSDKv2_SQSDecodesRequestThrottled(t *testing.T) {
+	_, err := sqs.NewFromConfig(throttleApp(t)).GetQueueUrl(context.Background(), &sqs.GetQueueUrlInput{QueueName: aws.String("q")})
+
+	var throttled *sqstypes.RequestThrottled
+	require.ErrorAs(t, err, &throttled, "%v", err)
+	assert.Equal(t, "Rate exceeded.", aws.ToString(throttled.Message))
+}
+
+func TestSDKv2_LambdaDecodesThrottleWithReason(t *testing.T) {
+	registerTestCodes(t, map[string]awserrors.ErrorMessage{
+		"TooManyRequestsException": {HTTPCode: 429, Message: "Rate Exceeded."},
+	})
+	cfg := sdkApp(t, awserrors.WithFields(
+		awserrors.Errorf("TooManyRequestsException", "Rate Exceeded."),
+		map[string]string{"Reason": "ReservedFunctionConcurrentInvocationLimitExceeded"}))
+
+	_, err := lambda.NewFromConfig(cfg).GetFunction(context.Background(), &lambda.GetFunctionInput{FunctionName: aws.String("f")})
+	var throttled *lambdatypes.TooManyRequestsException
+	require.ErrorAs(t, err, &throttled, "%v", err)
+	assert.Equal(t, lambdatypes.ThrottleReasonReservedFunctionConcurrentInvocationLimitExceeded, throttled.Reason)
+	assert.Equal(t, "Rate Exceeded.", aws.ToString(throttled.Message))
+}
+
+func TestSDKv2_LambdaDecodesGatewayThrottle(t *testing.T) {
+	_, err := lambda.NewFromConfig(throttleApp(t)).GetFunction(context.Background(), &lambda.GetFunctionInput{FunctionName: aws.String("f")})
+
+	var throttled *lambdatypes.TooManyRequestsException
+	require.ErrorAs(t, err, &throttled, "%v", err)
+	assert.Equal(t, lambdatypes.ThrottleReason("CallerRateLimitExceeded"), throttled.Reason)
+}
+
+func TestSDKv2_LambdaDecodesServiceException(t *testing.T) {
+	cfg := sdkApp(t, errors.New("not a registered code"))
+
+	_, err := lambda.NewFromConfig(cfg).GetFunction(context.Background(), &lambda.GetFunctionInput{FunctionName: aws.String("f")})
+	var svcErr *lambdatypes.ServiceException
+	require.ErrorAs(t, err, &svcErr, "%v", err)
+}
+
+func TestSDKv2_LambdaOversizeBodyDecodesRequestEntityTooLarge(t *testing.T) {
+	cfg := sdkApp(t, nil)
+	oversized := bytes.Repeat([]byte("x"), int(sigv4.MaxPayloadLen)+1)
+
+	_, err := lambda.NewFromConfig(cfg).Invoke(context.Background(), &lambda.InvokeInput{FunctionName: aws.String("f"), Payload: oversized})
+	require.Error(t, err)
+
+	// The Go SDK models this error as RequestTooLargeException, so AWS's own
+	// RequestEntityTooLargeException surfaces as a generic API error.
+	var apiErr smithy.APIError
+	require.ErrorAs(t, err, &apiErr, "%v", err)
+	assert.Equal(t, "RequestEntityTooLargeException", apiErr.ErrorCode())
+}
@@ -0,0 +1,304 @@
+// Tests that the gateway renders dispatcher, throttle, cluster-unavailable and
+// oversize-body errors through the envelope a registration declares.
+
+package gateway
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/bluebottle/pkg/sigv4"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/envelope"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	restJSONService  = "testrestjson"
+	json10Service    = "testjson10"
+	json11Service    = "testjson11"
+	xmlRegisteredSvc = "testxmlreg"
+)
+
+func envelopeRegistration(service string, env dispatch.ErrorEnvelope, c *dispatchCapture, mod func(*dispatch.Registration)) dispatch.Registration {
+	reg := dispatch.Registration{
+		Service:       service,
+		ResolveAction: func(*http.Request) string { return fakeAction },
+		Dispatch:      c.dispatcher(),
+		Errors:        env,
+		Inventory:     dispatch.Inventory{Registered: []string{fakeAction}},
+	}
+	if mod != nil {
+		mod(&reg)
+	}
+	return reg
+}
+
+func bodyString(t *testing.T, resp *http.Response) string {
+	t.Helper()
+	b, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	return string(b)
+}
+
+func throttleResponse(t *testing.T, regs ...dispatch.Registration) func(svc string) *http.Response {
+	t.Helper()
+	b := dispatch.NewBuilder()
+	for _, reg := range regs {
+		require.NoError(t, b.Register(reg))
+	}
+	gw := &GatewayConfig{DisableLogging: true, Services: b.Build()}
+	return func(svc string) *http.Response {
+		req := httptest.NewRequest(http.MethodPost, "/", nil)
+		req = req.WithContext(context.WithValue(req.Context(), ctxService, svc))
+		w := httptest.NewRecorder()
+		gw.writeThrottleError(w, req)
+		return w.Result()
+	}
+}
+
+func TestErrorEnvelope_Throttle(t *testing.T) {
+	c := &dispatchCapture{}
+	throttle := throttleResponse(t,
+		envelopeRegistration(restJSONService, dispatch.ErrorEnvelopeRESTJSON, c, nil),
+		envelopeRegistration(json10Service, dispatch.ErrorEnvelopeJSON10Query, c, nil),
+		envelopeRegistration(json11Service, dispatch.ErrorEnvelopeJSON, c, nil),
+		envelopeRegistration(xmlRegisteredSvc, dispatch.ErrorEnvelopeXML, c, nil),
+	)
+
+	namespaced := throttleResponse(t, envelopeRegistration(json10Service, dispatch.ErrorEnvelopeJSON10Query, c,
+		func(r *dispatch.Registration) { r.ErrorNamespace = "com.amazonaws.sqs" }))
+
+	t.Run("REST-JSON registration gets TooManyRequestsException with Reason", func(t *testing.T) {
+		resp := throttle(restJSONService)
+		assert.Equal(t, http.StatusTooManyRequests, resp.StatusCode)
+		assert.Equal(t, "application/json", resp.Header.Get("Content-Type"))
+		assert.Equal(t, "TooManyRequestsException", resp.Header.Get("X-Amzn-Errortype"))
+		assert.Empty(t, resp.Header.Get("Retry-After"))
+		assert.JSONEq(t, `{"Reason":"CallerRateLimitExceeded","Type":"User","message":"Rate Exceeded."}`, bodyString(t, resp))
+	})
+
+	t.Run("JSON 1.0 registration gets RequestThrottled with the query header", func(t *testing.T) {
+		resp := throttle(json10Service)
+		assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
+		assert.Equal(t, "application/x-amz-json-1.0", resp.Header.Get("Content-Type"))
+		assert.Equal(t, "RequestThrottled;Sender", resp.Header.Get("X-Amzn-Query-Error"))
+		assert.JSONEq(t, `{"__type":"RequestThrottled","message":"Rate exceeded."}`, bodyString(t, resp))
+	})
+
+	t.Run("JSON 1.0 namespaced throttle", func(t *testing.T) {
+		resp := namespaced(json10Service)
+		assert.Equal(t, "RequestThrottled;Sender", resp.Header.Get("X-Amzn-Query-Error"))
+		assert.JSONEq(t, `{"__type":"com.amazonaws.sqs#RequestThrottled","message":"Rate exceeded."}`, bodyString(t, resp))
+	})
+
+	t.Run("JSON 1.1 registration keeps its current throttle", func(t *testing.T) {
+		resp := throttle(json11Service)
+		assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode)
+		assert.Equal(t, "application/x-amz-json-1.1", resp.Header.Get("Content-Type"))
+		assert.Equal(t, "RequestLimitExceededException", resp.Header.Get("X-Amzn-Errortype"))
+		assert.Contains(t, bodyString(t, resp), `"__type":"RequestLimitExceededException"`)
+	})
+
+	t.Run("XML registration keeps its current throttle", func(t *testing.T) {
+		resp := throttle(xmlRegisteredSvc)
+		assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode)
+		assert.Equal(t, "application/xml", resp.Header.Get("Content-Type"))
+		assert.Contains(t, bodyString(t, resp), "<Code>RequestLimitExceeded</Code>")
+	})
+}
+
+func TestErrorEnvelope_ThrottleOverride(t *testing.T) {
+	c := &dispatchCapture{}
+	throttle := throttleResponse(t, envelopeRegistration(restJSONService, dispatch.ErrorEnvelopeRESTJSON, c,
+		func(r *dispatch.Registration) {
+			r.Throttle = &dispatch.ErrorSpec{
+				Code: "TooManyRequestsException", Message: "Slow down.", Status: http.StatusTooManyRequests,
+				Fields: map[string]string{"Reason": "FunctionInvocationRateLimitExceeded"},
+			}
+		}))
+	resp := throttle(restJSONService)
+	assert.Equal(t, http.StatusTooManyRequests, resp.StatusCode)
+	assert.JSONEq(t, `{"Reason":"FunctionInvocationRateLimitExceeded","Type":"User","message":"Slow down."}`, bodyString(t, resp))
+}
+
+func TestErrorEnvelope_DispatcherError(t *testing.T) {
+	notFound := awserrors.WithFields(
+		awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "bad value"),
+		map[string]string{"Reason": "Because"})
+
+	t.Run("REST-JSON uses the code as the type and carries service fields", func(t *testing.T) {
+		c := &dispatchCapture{dispatchErr: notFound}
+		_, app := dispatchSeamApp(t, allowAllECRDocs(), envelopeRegistration(restJSONService, dispatch.ErrorEnvelopeRESTJSON, c, nil))
+		resp := doRequest(app, signDispatchRequest(t, restJSONService, []byte("{}")))
+
+		assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
+		assert.Equal(t, "application/json", resp.Header.Get("Content-Type"))
+		assert.Equal(t, "InvalidParameterValue", resp.Header.Get("X-Amzn-Errortype"))
+		assert.Equal(t, `{"Reason":"Because","Type":"User","message":"bad value"}`, bodyString(t, resp))
+	})
+
+	t.Run("REST-JSON server fault is Type Service with its own status", func(t *testing.T) {
+		c := &dispatchCapture{dispatchErr: awserrors.Errorf(awserrors.ErrorInternalError, "boom")}
+		_, app := dispatchSeamApp(t, allowAllECRDocs(), envelopeRegistration(restJSONService, dispatch.ErrorEnvelopeRESTJSON, c, nil))
+		resp := doRequest(app, signDispatchRequest(t, restJSONService, []byte("{}")))
+
+		assert.Equal(t, http.StatusInternalServerError, resp.StatusCode)
+		assert.Equal(t, "ServiceException", resp.Header.Get("X-Amzn-Errortype"))
+		assert.JSONEq(t, `{"Type":"Service","message":"boom"}`, bodyString(t, resp))
+	})
+
+	t.Run("JSON 1.0 sends __type unsuffixed and maps the query code", func(t *testing.T) {
+		c := &dispatchCapture{dispatchErr: awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "bad value")}
+		_, app := dispatchSeamApp(t, allowAllECRDocs(), envelopeRegistration(json10Service, dispatch.ErrorEnvelopeJSON10Query, c,
+			func(r *dispatch.Registration) {
+				r.QueryErrorCodes = map[string]string{awserrors.ErrorInvalidParameterValue: "AWS.Simple.InvalidParameterValue"}
+			}))
+		resp := doRequest(app, signDispatchRequest(t, json10Service, []byte("{}")))
+
+		assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
+		assert.Equal(t, "application/x-amz-json-1.0", resp.Header.Get("Content-Type"))
+		assert.Equal(t, "AWS.Simple.InvalidParameterValue;Sender", resp.Header.Get("X-Amzn-Query-Error"))
+		assert.Empty(t, resp.Header.Get("X-Amzn-Errortype"))
+		assert.Equal(t, `{"__type":"InvalidParameterValue","message":"bad value"}`, bodyString(t, resp))
+	})
+
+	t.Run("JSON 1.0 namespaces __type but not the query header", func(t *testing.T) {
+		c := &dispatchCapture{dispatchErr: awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "bad value")}
+		_, app := dispatchSeamApp(t, allowAllECRDocs(), envelopeRegistration(json10Service, dispatch.ErrorEnvelopeJSON10Query, c,
+			func(r *dispatch.Registration) { r.ErrorNamespace = "com.amazonaws.sqs" }))
+		resp := doRequest(app, signDispatchRequest(t, json10Service, []byte("{}")))
+
+		assert.Equal(t, "InvalidParameterValue;Sender", resp.Header.Get("X-Amzn-Query-Error"))
+		assert.Equal(t, `{"__type":"com.amazonaws.sqs#InvalidParameterValue","message":"bad value"}`, bodyString(t, resp))
+	})
+
+	t.Run("JSON 1.0 per-code namespace and query code for a framework error", func(t *testing.T) {
+		c := &dispatchCapture{dispatchErr: awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "bad value")}
+		_, app := dispatchSeamApp(t, allowAllECRDocs(), envelopeRegistration(json10Service, dispatch.ErrorEnvelopeJSON10Query, c,
+			func(r *dispatch.Registration) {
+				r.ErrorNamespace = "com.amazonaws.sqs"
+				r.ErrorNamespaces = map[string]string{awserrors.ErrorInvalidParameterValue: envelope.CoralNamespace}
+			}))
+		resp := doRequest(app, signDispatchRequest(t, json10Service, []byte("{}")))
+		assert.Equal(t, `{"__type":"com.amazon.coral.service#InvalidParameterValue","message":"bad value"}`, bodyString(t, resp))
+	})
+
+	t.Run("JSON 1.0 unmapped server fault is Receiver with InternalFailure", func(t *testing.T) {
+		c := &dispatchCapture{dispatchErr: awserrors.Errorf(awserrors.ErrorInternalError, "boom")}
+		_, app := dispatchSeamApp(t, allowAllECRDocs(), envelopeRegistration(json10Service, dispatch.ErrorEnvelopeJSON10Query, c, nil))
+		resp := doRequest(app, signDispatchRequest(t, json10Service, []byte("{}")))
+
+		assert.Equal(t, http.StatusInternalServerError, resp.StatusCode)
+		assert.Equal(t, "InternalFailure;Receiver", resp.Header.Get("X-Amzn-Query-Error"))
+		assert.Contains(t, bodyString(t, resp), `"__type":"InternalFailure"`)
+	})
+
+	t.Run("unknown code takes the registration's internal error code", func(t *testing.T) {
+		c := &dispatchCapture{dispatchErr: errors.New("not a registered code")}
+		_, app := dispatchSeamApp(t, allowAllECRDocs(), envelopeRegistration(restJSONService, dispatch.ErrorEnvelopeRESTJSON, c,
+			func(r *dispatch.Registration) { r.InternalErrorCode = "CustomInternal" }))
+		resp := doRequest(app, signDispatchRequest(t, restJSONService, []byte("{}")))
+
+		assert.Equal(t, http.StatusInternalServerError, resp.StatusCode)
+		assert.Equal(t, "CustomInternal", resp.Header.Get("X-Amzn-Errortype"))
+	})
+
+	t.Run("JSON 1.1 and XML keep InternalError", func(t *testing.T) {
+		c := &dispatchCapture{dispatchErr: errors.New("not a registered code")}
+		_, app := dispatchSeamApp(t, allowAllECRDocs(), envelopeRegistration(json11Service, dispatch.ErrorEnvelopeJSON, c, nil))
+		resp := doRequest(app, signDispatchRequest(t, json11Service, []byte("{}")))
+		assert.Equal(t, "InternalErrorException", resp.Header.Get("X-Amzn-Errortype"))
+	})
+
+	t.Run("JSON 1.1 still appends Exception", func(t *testing.T) {
+		c := &dispatchCapture{dispatchErr: awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "bad value")}
+		_, app := dispatchSeamApp(t, allowAllECRDocs(), envelopeRegistration(json11Service, dispatch.ErrorEnvelopeJSON, c, nil))
+		resp := doRequest(app, signDispatchRequest(t, json11Service, []byte("{}")))
+
+		assert.Equal(t, "application/x-amz-json-1.1", resp.Header.Get("Content-Type"))
+		assert.Equal(t, "InvalidParameterValueException", resp.Header.Get("X-Amzn-Errortype"))
+		assert.Equal(t, `{"__type":"InvalidParameterValueException","message":"bad value"}`, bodyString(t, resp))
+	})
+}
+
+func TestErrorEnvelope_ClusterUnavailable(t *testing.T) {
+	c := &dispatchCapture{}
+	b := dispatch.NewBuilder()
+	require.NoError(t, b.Register(envelopeRegistration(restJSONService, dispatch.ErrorEnvelopeRESTJSON, c, nil)))
+	require.NoError(t, b.Register(envelopeRegistration(json10Service, dispatch.ErrorEnvelopeJSON10Query, c, nil)))
+	gw := &GatewayConfig{DisableLogging: true, IAMService: allowAllIAMService(), Services: b.Build()}
+
+	call := func(svc string) *http.Response {
+		req := httptest.NewRequest(http.MethodPost, "/", nil)
+		req = req.WithContext(context.WithValue(req.Context(), ctxService, svc))
+		w := httptest.NewRecorder()
+		gw.Request(w, req)
+		return w.Result()
+	}
+
+	resp := call(restJSONService)
+	assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode)
+	assert.Equal(t, "application/json", resp.Header.Get("Content-Type"))
+	assert.Equal(t, "ServiceUnavailableException", resp.Header.Get("X-Amzn-Errortype"))
+	assert.JSONEq(t, `{"Type":"Service","message":"`+clusterUnavailableMsg+`"}`, bodyString(t, resp))
+
+	resp = call(json10Service)
+	assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode)
+	assert.Equal(t, "application/x-amz-json-1.0", resp.Header.Get("Content-Type"))
+	assert.Equal(t, "ServiceUnavailable;Receiver", resp.Header.Get("X-Amzn-Query-Error"))
+	assert.JSONEq(t, `{"__type":"com.amazon.coral.service#ServiceUnavailableException","message":"`+clusterUnavailableMsg+`"}`, bodyString(t, resp))
+}
+
+func TestErrorEnvelope_OversizeBody(t *testing.T) {
+	oversized := bytes.Repeat([]byte("x"), int(sigv4.MaxPayloadLen)+1)
+	send := func(t *testing.T, app http.Handler, service string) *http.Response {
+		req := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(oversized))
+		req.Host = "localhost:9999"
+		signTestRequestAs(t, req, oversized, testAccessKey, testSecretKey, service, time.Now().UTC())
+		return doRequest(app, req)
+	}
+	c := &dispatchCapture{}
+	_, app := dispatchSeamApp(t, allowAllECRDocs(),
+		envelopeRegistration(restJSONService, dispatch.ErrorEnvelopeRESTJSON, c, nil),
+		envelopeRegistration(json10Service, dispatch.ErrorEnvelopeJSON10Query, c, nil))
+
+	resp := send(t, app, restJSONService)
+	assert.Equal(t, http.StatusRequestEntityTooLarge, resp.StatusCode)
+	assert.Equal(t, "application/json", resp.Header.Get("Content-Type"))
+	assert.Equal(t, "RequestEntityTooLargeException", resp.Header.Get("X-Amzn-Errortype"))
+	assert.JSONEq(t, `{"Type":"User","message":"Request body exceeds the maximum allowed size."}`, bodyString(t, resp))
+
+	resp = send(t, app, json10Service)
+	assert.Equal(t, http.StatusRequestEntityTooLarge, resp.StatusCode)
+	assert.Equal(t, "text; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.Empty(t, resp.Header.Get("X-Amzn-Query-Error"))
+	assert.Equal(t, "HTTP content length exceeded "+strconv.Itoa(int(sigv4.MaxPayloadLen))+" bytes.", bodyString(t, resp))
+}
+
+func TestCredentialScopeService(t *testing.T) {
+	for name, tc := range map[string]struct {
+		header, want string
+	}{
+		"scope":     {"AWS4-HMAC-SHA256 Credential=AK/20260101/us-east-1/lambda/aws4_request, SignedHeaders=host, Signature=x", "lambda"},
+		"no header": {"", ""},
+		"truncated": {"AWS4-HMAC-SHA256 Credential=AK/20260101/us-east-1", ""},
+	} {
+		t.Run(name, func(t *testing.T) {
+			req := httptest.NewRequest(http.MethodPost, "/", nil)
+			if tc.header != "" {
+				req.Header.Set("Authorization", tc.header)
+			}
+			assert.Equal(t, tc.want, credentialScopeService(req))
+		})
+	}
+}
@@ -419,13 +419,23 @@ func jsonErrorService(svc string) bool {
 	return false
 }

-// jsonErrorService reports whether svc's errors render in the AWS JSON 1.1
-// envelope: a registered service's own declared envelope, or the legacy
-// table for every service not yet registered through the seam. Every error
-// emitter calls this method, never the free function, so the two can never
-// disagree on a registered service.
-func (gw *GatewayConfig) jsonErrorService(svc string) bool {
-	return gw.selector().JSONErrors(svc)
+// errorProtocol reports the JSON wire format svc's errors use; ok is false when
+// they render as XML. A registered service's declared envelope wins.
+func (gw *GatewayConfig) errorProtocol(svc string) (p envelope.Protocol, ok bool) {
+	return gw.selector().ErrorProtocol(svc)
+}
+
+// writeJSONError renders a gateway-raised error in svc's JSON protocol. The
+// code gets the common-error "Exception" suffix unless verbatim.
+func (gw *GatewayConfig) writeJSONError(w http.ResponseWriter, p envelope.Protocol, svc string, e envelope.JSONError) {
+	if entry, ok := gw.registered(svc); ok {
+		if internal, own := entry.InternalErrorCode(); own && e.Code == awserrors.ErrorInternalError {
+			e.Code, e.Verbatim = internal, true
+		}
+		e.QueryCode = entry.QueryErrorCode(e.Code)
+		e.Namespace = entry.ErrorNamespace(e.Code)
+	}
+	envelope.WriteJSON(w, p, e)
 }

 // clusterUnavailableMsg is the 503 body when NATS is disconnected. Points
@@ -437,15 +447,13 @@ const clusterUnavailableMsg = "cluster unavailable: NATS disconnected — check
 func (gw *GatewayConfig) writeClusterUnavailable(w http.ResponseWriter, r *http.Request, svc string) {
 	requestID := uuid.NewV4().String()

-	// AWS JSON 1.1 services (EKS/ECS/bedrock family, …) get a JSON body.
-	if gw.jsonErrorService(svc) {
-		body := envelope.JSONBody(awserrors.ErrorServiceUnavailable, clusterUnavailableMsg)
-		w.Header().Set("Content-Type", envelope.JSONContentType)
-		w.Header().Set("X-Amzn-Errortype", envelope.JSONErrorType(awserrors.ErrorServiceUnavailable))
-		w.WriteHeader(http.StatusServiceUnavailable)
-		if _, err := w.Write(body); err != nil {
-			slog.Error("Failed to write JSON cluster-unavailable response", "err", err)
-		}
+	// JSON services get a body in their own protocol.
+	if p, ok := gw.errorProtocol(svc); ok {
+		gw.writeJSONError(w, p, svc, envelope.JSONError{
+			Code:    awserrors.ErrorServiceUnavailable,
+			Message: clusterUnavailableMsg,
+			Status:  http.StatusServiceUnavailable,
+		})
 		return
 	}

@@ -469,15 +477,18 @@ func (gw *GatewayConfig) writeThrottleError(w http.ResponseWriter, r *http.Reque
 	}
 	errorMsg := awserrors.LookupErrorMessage(svc, errorCode)

-	// AWS JSON 1.1 services (EKS/ECS/bedrock family, …) get a JSON body.
-	if gw.jsonErrorService(svc) {
-		body := envelope.JSONBody(errorCode, errorMsg.Message)
-		w.Header().Set("Content-Type", envelope.JSONContentType)
-		w.Header().Set("X-Amzn-Errortype", envelope.JSONErrorType(errorCode))
-		w.WriteHeader(errorMsg.HTTPCode)
-		if _, err := w.Write(body); err != nil {
-			slog.Error("Failed to write JSON throttle error response", "err", err)
+	// JSON services get a body in their own protocol.
+	if p, ok := gw.errorProtocol(svc); ok {
+		je := envelope.JSONError{Code: errorCode, Message: errorMsg.Message, Status: errorMsg.HTTPCode}
+		if entry, reg := gw.registered(svc); reg {
+			if spec, own := entry.ThrottleError(); own {
+				je = envelope.JSONError{
+					Code: spec.Code, Verbatim: true, Message: spec.Message,
+					Status: spec.Status, Fields: spec.Fields,
+				}
+			}
 		}
+		gw.writeJSONError(w, p, svc, je)
 		return
 	}

@@ -945,17 +956,15 @@ func (gw *GatewayConfig) ErrorHandler(w http.ResponseWriter, r *http.Request, er
 		w.Header().Set("Retry-After", strconv.Itoa(int(d.Seconds())))
 	}

-	// EKS, ECR, ACM, ECS, tagging, and the bedrock family use AWS JSON 1.1;
-	// query/XML services fall through.
-	if gw.jsonErrorService(svc) {
-		body := envelope.JSONBody(code, errorMsg.Message)
-		slog.Debug("Generated JSON error response", "service", svc, "error", err, "code", code, "json", string(body), "requestId", requestId)
-		w.Header().Set("Content-Type", envelope.JSONContentType)
-		w.Header().Set("X-Amzn-Errortype", envelope.JSONErrorType(code))
-		w.WriteHeader(errorMsg.HTTPCode)
-		if _, err := w.Write(body); err != nil {
-			slog.Error("Failed to write EKS error response", "err", err)
-		}
+	// JSON services answer in their own protocol; query/XML services fall
+	// through.
+	if p, ok := gw.errorProtocol(svc); ok {
+		slog.Debug("Generated JSON error response", "service", svc, "error", err, "code", code, "requestId", requestId)
+		gw.writeJSONError(w, p, svc, envelope.JSONError{
+			Code: code, Verbatim: p != envelope.ProtocolJSON11,
+			Message: errorMsg.Message, Status: errorMsg.HTTPCode,
+			Fields: awserrors.ResolveFields(err),
+		})
 		return
 	}

@@ -13,6 +13,7 @@ import (

 	"github.com/mulgadc/bluebottle/pkg/iampolicy"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/envelope"
 )

 // ErrorEnvelope selects which generic error rendering a service's errors use.
@@ -25,8 +26,43 @@ const (
 	ErrorEnvelopeXML
 	// ErrorEnvelopeJSON is the AWS JSON 1.1 error body with X-Amzn-Errortype.
 	ErrorEnvelopeJSON
+	// ErrorEnvelopeRESTJSON is the REST-JSON error body: application/json,
+	// x-amzn-ErrorType, Type and message keys and any service-supplied fields.
+	ErrorEnvelopeRESTJSON
+	// ErrorEnvelopeJSON10Query is the AWS JSON 1.0 error body with the
+	// awsQueryCompatible x-amzn-query-error header.
+	ErrorEnvelopeJSON10Query
 )

+// IsJSON reports whether the envelope is one of the JSON protocols.
+func (e ErrorEnvelope) IsJSON() bool {
+	return e == ErrorEnvelopeJSON || e == ErrorEnvelopeRESTJSON || e == ErrorEnvelopeJSON10Query
+}
+
+// Protocol returns the JSON wire format for a JSON envelope; ok is false for
+// XML and for an unset envelope.
+func (e ErrorEnvelope) Protocol() (p envelope.Protocol, ok bool) {
+	switch e {
+	case ErrorEnvelopeJSON:
+		return envelope.ProtocolJSON11, true
+	case ErrorEnvelopeRESTJSON:
+		return envelope.ProtocolRESTJSON, true
+	case ErrorEnvelopeJSON10Query:
+		return envelope.ProtocolJSON10Query, true
+	}
+	return envelope.ProtocolJSON11, false
+}
+
+// ErrorSpec is an error a registration defines for ingress to raise before its
+// dispatcher runs. The code is sent as the wire error type unchanged.
+type ErrorSpec struct {
+	Code    string
+	Message string
+	Status  int
+	// Fields are extra top-level body keys on protocols that carry them.
+	Fields map[string]string
+}
+
 // Inventory classifies a service's dispatched operations. Registered minus
 // Stubbed and Unsupported are implemented; Stubbed and Unsupported are
 // disjoint subsets of Registered.
@@ -82,7 +118,26 @@ type Registration struct {
 	ResolveAction ActionResolver
 	Dispatch      Dispatcher
 	Errors        ErrorEnvelope
-	Inventory     Inventory
+	// Throttle replaces the error returned when a request is rate limited.
+	// Unset, a REST-JSON or JSON 1.0 registration gets the protocol's own
+	// throttle error and any other keeps the gateway default. JSON only.
+	Throttle *ErrorSpec
+	// QueryErrorCodes maps an error code to the legacy query code sent in
+	// x-amzn-query-error; a code absent from it is sent as itself. JSON 1.0
+	// only.
+	QueryErrorCodes map[string]string
+	// InternalErrorCode is the wire code sent for an error with no registered
+	// code. Unset, a REST-JSON registration sends ServiceException and a JSON
+	// 1.0 one InternalFailure. JSON 1.1 and XML keep InternalError.
+	InternalErrorCode string
+	// ErrorNamespace is the Smithy shape namespace (e.g. "com.amazonaws.sqs")
+	// prefixed to the __type of service-defined errors as "<namespace>#<code>".
+	// Unset, __type is the bare code. JSON 1.0 only.
+	ErrorNamespace string
+	// ErrorNamespaces overrides ErrorNamespace for individual codes, e.g. to
+	// send a framework error under "com.amazon.coral.service". JSON 1.0 only.
+	ErrorNamespaces map[string]string
+	Inventory       Inventory
 }

 // UnknownService returns the client-visible code and message for a credential
@@ -137,10 +192,15 @@ func (b *Builder) Register(reg Registration) error {
 	}

 	e := &Entry{
-		service: reg.Service,
-		resolve: reg.ResolveAction,
-		handle:  reg.Dispatch,
-		errors:  reg.Errors,
+		service:    reg.Service,
+		resolve:    reg.ResolveAction,
+		handle:     reg.Dispatch,
+		errors:     reg.Errors,
+		throttle:   reg.Throttle,
+		queryCodes: maps.Clone(reg.QueryErrorCodes),
+		internal:   reg.InternalErrorCode,
+		namespace:  reg.ErrorNamespace,
+		namespaces: maps.Clone(reg.ErrorNamespaces),
 	}
 	for _, n := range names {
 		b.entries[n] = e
@@ -178,15 +238,44 @@ func validate(reg Registration) ([]string, error) {
 	if reg.Dispatch == nil {
 		return nil, fmt.Errorf("dispatch: %s: nil dispatcher", reg.Service)
 	}
-	if reg.Errors != ErrorEnvelopeXML && reg.Errors != ErrorEnvelopeJSON {
+	if reg.Errors != ErrorEnvelopeXML && !reg.Errors.IsJSON() {
 		return nil, fmt.Errorf("dispatch: %s: error envelope not set", reg.Service)
 	}
+	if err := validateErrorSpecs(reg); err != nil {
+		return nil, fmt.Errorf("dispatch: %s: %w", reg.Service, err)
+	}
 	if err := validateInventory(reg.Inventory); err != nil {
 		return nil, fmt.Errorf("dispatch: %s: %w", reg.Service, err)
 	}
 	return names, nil
 }

+func validateErrorSpecs(reg Registration) error {
+	if t := reg.Throttle; t != nil {
+		if !reg.Errors.IsJSON() {
+			return errors.New("throttle error needs a JSON error envelope")
+		}
+		if t.Code == "" || t.Status < 400 || t.Status > 599 {
+			return fmt.Errorf("throttle error needs a code and a 4xx or 5xx status, got %q and %d", t.Code, t.Status)
+		}
+	}
+	if len(reg.QueryErrorCodes) > 0 && reg.Errors != ErrorEnvelopeJSON10Query {
+		return errors.New("query error codes need the JSON 1.0 error envelope")
+	}
+	if reg.InternalErrorCode != "" && reg.Errors != ErrorEnvelopeRESTJSON && reg.Errors != ErrorEnvelopeJSON10Query {
+		return errors.New("internal error code needs the REST-JSON or JSON 1.0 error envelope")
+	}
+	if (reg.ErrorNamespace != "" || len(reg.ErrorNamespaces) > 0) && reg.Errors != ErrorEnvelopeJSON10Query {
+		return errors.New("error namespace needs the JSON 1.0 error envelope")
+	}
+	for code, legacy := range reg.QueryErrorCodes {
+		if code == "" || legacy == "" {
+			return errors.New("query error codes must not be empty")
+		}
+	}
+	return nil
+}
+
 func validateInventory(inv Inventory) error {
 	if len(inv.Registered) == 0 {
 		return errors.New("inventory has no registered operations")
@@ -265,6 +354,12 @@ type Entry struct {
 	resolve ActionResolver
 	handle  Dispatcher
 	errors  ErrorEnvelope
+
+	throttle   *ErrorSpec
+	queryCodes map[string]string
+	internal   string
+	namespace  string
+	namespaces map[string]string
 }

 // Service returns the registration's primary credential-scope name.
@@ -273,6 +368,64 @@ func (e *Entry) Service() string { return e.service }
 // ErrorEnvelope returns the envelope the service's errors render in.
 func (e *Entry) ErrorEnvelope() ErrorEnvelope { return e.errors }

+// ThrottleError returns the error to send when a request to this service is
+// rate limited; ok is false when the gateway default applies.
+func (e *Entry) ThrottleError() (spec ErrorSpec, ok bool) {
+	if e.throttle != nil {
+		return cloneSpec(*e.throttle), true
+	}
+	switch e.errors {
+	case ErrorEnvelopeRESTJSON:
+		return ErrorSpec{
+			Code:    "TooManyRequestsException",
+			Message: "Rate Exceeded.",
+			Status:  http.StatusTooManyRequests,
+			Fields:  map[string]string{"Reason": "CallerRateLimitExceeded"},
+		}, true
+	case ErrorEnvelopeJSON10Query:
+		return ErrorSpec{Code: "RequestThrottled", Message: "Rate exceeded.", Status: http.StatusBadRequest}, true
+	}
+	return ErrorSpec{}, false
+}
+
+func cloneSpec(s ErrorSpec) ErrorSpec {
+	s.Fields = maps.Clone(s.Fields)
+	return s
+}
+
+// InternalErrorCode returns the wire code for an error carrying no registered
+// code; ok is false when the gateway's InternalError applies.
+func (e *Entry) InternalErrorCode() (code string, ok bool) {
+	if e.internal != "" {
+		return e.internal, true
+	}
+	switch e.errors {
+	case ErrorEnvelopeRESTJSON:
+		return "ServiceException", true
+	case ErrorEnvelopeJSON10Query:
+		return "InternalFailure", true
+	}
+	return "", false
+}
+
+// ErrorNamespace returns the Smithy namespace prefixed to code's error type,
+// or "" when none is declared.
+func (e *Entry) ErrorNamespace(code string) string {
+	if ns, ok := e.namespaces[code]; ok {
+		return ns
+	}
+	return e.namespace
+}
+
+// QueryErrorCode returns the legacy query code sent in x-amzn-query-error for
+// code, which is code itself unless the registration maps it.
+func (e *Entry) QueryErrorCode(code string) string {
+	if q, ok := e.queryCodes[code]; ok {
+		return q
+	}
+	return code
+}
+
 // ResolveAction returns the advisory action label for r, or "" when the
 // registration has no resolver or cannot name the action.
 func (e *Entry) ResolveAction(r *http.Request) string {
@@ -8,6 +8,7 @@ import (
 	"testing"

 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/envelope"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -224,3 +225,142 @@ func TestRegistryAllowsConcurrentReads(t *testing.T) {
 	}
 	wg.Wait()
 }
+
+func TestRegisterValidatesErrorSpecs(t *testing.T) {
+	spec := &ErrorSpec{Code: "Throttled", Status: 429}
+	mutate := map[string]func(*Registration){
+		"throttle on XML":         func(r *Registration) { r.Errors = ErrorEnvelopeXML; r.Throttle = spec },
+		"throttle without code":   func(r *Registration) { r.Throttle = &ErrorSpec{Status: 429} },
+		"throttle with 2xx":       func(r *Registration) { r.Throttle = &ErrorSpec{Code: "X", Status: 200} },
+		"query codes on JSON 1.1": func(r *Registration) { r.QueryErrorCodes = map[string]string{"A": "B"} },
+		"namespaces on JSON 1.1":  func(r *Registration) { r.ErrorNamespaces = map[string]string{"A": "ns"} },
+		"namespace on JSON 1.1":   func(r *Registration) { r.ErrorNamespace = "com.amazonaws.sqs" },
+		"empty query code": func(r *Registration) {
+			r.Errors = ErrorEnvelopeJSON10Query
+			r.QueryErrorCodes = map[string]string{"A": ""}
+		},
+	}
+	for name, m := range mutate {
+		t.Run(name, func(t *testing.T) {
+			r := validReg("ecr")
+			m(&r)
+			require.Error(t, NewBuilder().Register(r))
+		})
+	}
+
+	for _, env := range []ErrorEnvelope{ErrorEnvelopeRESTJSON, ErrorEnvelopeJSON10Query} {
+		r := validReg("svc")
+		r.Errors = env
+		r.Throttle = spec
+		require.NoError(t, NewBuilder().Register(r))
+	}
+}
+
+func TestEntryThrottleErrorDefaultsPerEnvelope(t *testing.T) {
+	entry := func(env ErrorEnvelope) *Entry {
+		b := NewBuilder()
+		r := validReg("svc")
+		r.Errors = env
+		require.NoError(t, b.Register(r))
+		e, ok := b.Build().Lookup("svc")
+		require.True(t, ok)
+		return e
+	}
+
+	_, own := entry(ErrorEnvelopeJSON).ThrottleError()
+	assert.False(t, own, "JSON 1.1 keeps the gateway default")
+	_, own = entry(ErrorEnvelopeXML).ThrottleError()
+	assert.False(t, own, "XML keeps the gateway default")
+
+	rest, own := entry(ErrorEnvelopeRESTJSON).ThrottleError()
+	require.True(t, own)
+	assert.Equal(t, ErrorSpec{
+		Code: "TooManyRequestsException", Message: "Rate Exceeded.", Status: 429,
+		Fields: map[string]string{"Reason": "CallerRateLimitExceeded"},
+	}, rest)
+	rest.Fields["Reason"] = "mutated"
+	again, _ := entry(ErrorEnvelopeRESTJSON).ThrottleError()
+	assert.Equal(t, "CallerRateLimitExceeded", again.Fields["Reason"], "a caller cannot mutate the default")
+
+	q, own := entry(ErrorEnvelopeJSON10Query).ThrottleError()
+	require.True(t, own)
+	assert.Equal(t, "RequestThrottled", q.Code)
+	assert.Equal(t, 400, q.Status)
+}
+
+func TestSelectorErrorProtocol(t *testing.T) {
+	b := NewBuilder()
+	for svc, env := range map[string]ErrorEnvelope{
+		"rest": ErrorEnvelopeRESTJSON, "q": ErrorEnvelopeJSON10Query, "j": ErrorEnvelopeJSON, "x": ErrorEnvelopeXML,
+	} {
+		r := validReg(svc)
+		r.Errors = env
+		require.NoError(t, b.Register(r))
+	}
+	sel := NewSelector(b.Build(), nil)
+
+	for svc, want := range map[string]envelope.Protocol{
+		"rest": envelope.ProtocolRESTJSON, "q": envelope.ProtocolJSON10Query, "j": envelope.ProtocolJSON11,
+	} {
+		p, ok := sel.ErrorProtocol(svc)
+		assert.True(t, ok, svc)
+		assert.Equal(t, want, p, svc)
+		assert.True(t, sel.JSONErrors(svc), svc)
+	}
+	_, ok := sel.ErrorProtocol("x")
+	assert.False(t, ok)
+	assert.False(t, sel.JSONErrors("x"))
+	_, ok = sel.ErrorProtocol("unknown")
+	assert.False(t, ok)
+}
+
+func TestEntryQueryErrorCode(t *testing.T) {
+	b := NewBuilder()
+	r := validReg("svc")
+	r.Errors = ErrorEnvelopeJSON10Query
+	r.QueryErrorCodes = map[string]string{"QueueDoesNotExist": "AWS.SimpleQueueService.NonExistentQueue"}
+	require.NoError(t, b.Register(r))
+	e, _ := b.Build().Lookup("svc")
+	assert.Equal(t, "AWS.SimpleQueueService.NonExistentQueue", e.QueryErrorCode("QueueDoesNotExist"))
+	assert.Equal(t, "OverLimit", e.QueryErrorCode("OverLimit"))
+}
+
+func TestEntryInternalErrorCode(t *testing.T) {
+	entry := func(env ErrorEnvelope, code string) *Entry {
+		b := NewBuilder()
+		r := validReg("svc")
+		r.Errors = env
+		r.InternalErrorCode = code
+		require.NoError(t, b.Register(r))
+		e, _ := b.Build().Lookup("svc")
+		return e
+	}
+	code, ok := entry(ErrorEnvelopeRESTJSON, "").InternalErrorCode()
+	assert.True(t, ok)
+	assert.Equal(t, "ServiceException", code)
+	code, ok = entry(ErrorEnvelopeJSON10Query, "").InternalErrorCode()
+	assert.True(t, ok)
+	assert.Equal(t, "InternalFailure", code)
+	code, _ = entry(ErrorEnvelopeRESTJSON, "Mine").InternalErrorCode()
+	assert.Equal(t, "Mine", code)
+	_, ok = entry(ErrorEnvelopeJSON, "").InternalErrorCode()
+	assert.False(t, ok)
+	_, ok = entry(ErrorEnvelopeXML, "").InternalErrorCode()
+	assert.False(t, ok)
+
+	r := validReg("svc")
+	r.InternalErrorCode = "Mine"
+	require.Error(t, NewBuilder().Register(r), "JSON 1.1 cannot set an internal error code")
+}
+
+func TestEntryErrorNamespacePerCodeOverride(t *testing.T) {
+	b := NewBuilder()
+	r := validReg("svc")
+	r.Errors = ErrorEnvelopeJSON10Query
+	r.ErrorNamespace = "com.amazonaws.sqs"
+	r.ErrorNamespaces = map[string]string{"SerializationException": "com.amazon.coral.service"}
+	require.NoError(t, b.Register(r))
+	e, _ := b.Build().Lookup("svc")
+	assert.Equal(t, "com.amazonaws.sqs", e.ErrorNamespace("QueueDoesNotExist"))
+	assert.Equal(t, "com.amazon.coral.service", e.ErrorNamespace("SerializationException"))
+}
@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"net/http"
 	"slices"
+
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/envelope"
 )

 // Legacy answers service selection for signed services not yet registered,
@@ -59,15 +61,27 @@ func (s *Selector) ResolveAction(r *http.Request, service string) string {
 	return s.legacy.ResolveAction(r, service)
 }

-// JSONErrors reports whether service's errors render in the AWS JSON 1.1
-// envelope: a registration's declared envelope, else the legacy answer.
+// JSONErrors reports whether service's errors render in a JSON envelope: a
+// registration's declared envelope, else the legacy answer.
 func (s *Selector) JSONErrors(service string) bool {
 	if e, ok := s.Lookup(service); ok {
-		return e.ErrorEnvelope() == ErrorEnvelopeJSON
+		return e.ErrorEnvelope().IsJSON()
 	}
 	return s.legacy.JSONErrors(service)
 }

+// ErrorProtocol returns the JSON protocol service's errors use; ok is false
+// when they render as XML. A service served only by the legacy path is JSON 1.1.
+func (s *Selector) ErrorProtocol(service string) (p envelope.Protocol, ok bool) {
+	if e, found := s.Lookup(service); found {
+		return e.ErrorEnvelope().Protocol()
+	}
+	if s.legacy.JSONErrors(service) {
+		return envelope.ProtocolJSON11, true
+	}
+	return envelope.ProtocolJSON11, false
+}
+
 // Validate fails if any name the registry serves, alias included, is also
 // served by the legacy path, so no service is reachable down both at once.
 func (s *Selector) Validate() error {
@@ -8,6 +8,7 @@ import (
 	"encoding/xml"
 	"fmt"
 	"log/slog"
+	"maps"
 	"net/http"
 	"strings"
 )
@@ -164,3 +165,162 @@ func RequestSignalsJSONProtocol(r *http.Request) bool {
 	}
 	return strings.HasPrefix(r.Header.Get("Content-Type"), "application/x-amz-json")
 }
+
+// REST-JSON and AWS JSON 1.0 content types.
+const (
+	RESTJSONContentType = "application/json"
+	JSON10ContentType   = "application/x-amz-json-1.0"
+)
+
+// QueryErrorHeader carries the legacy query-protocol error code and fault on
+// an awsQueryCompatible JSON 1.0 response, as "<code>;<Sender|Receiver>".
+const QueryErrorHeader = "X-Amzn-Query-Error"
+
+// CoralNamespace is the __type namespace of errors the AWS framework raises
+// ahead of a service, as opposed to errors the service model defines.
+const CoralNamespace = "com.amazon.coral.service"
+
+// coralTypes maps a gateway error code to the framework error AWS JSON 1.0
+// services send for it. A code absent here becomes "<code>Exception".
+var coralTypes = map[string]string{
+	"SignatureDoesNotMatch": "InvalidSignatureException",
+	"InvalidClientTokenId":  "UnrecognizedClientException",
+	"InvalidAction":         "UnknownOperationException",
+}
+
+// Protocol selects a JSON error wire format.
+type Protocol int
+
+const (
+	// ProtocolJSON11 is AWS JSON 1.1: __type and message, X-Amzn-Errortype.
+	ProtocolJSON11 Protocol = iota
+	// ProtocolRESTJSON is REST-JSON: Type and message, x-amzn-ErrorType, plus
+	// any service-supplied fields.
+	ProtocolRESTJSON
+	// ProtocolJSON10Query is AWS JSON 1.0 with the awsQueryCompatible trait:
+	// __type and message, and an x-amzn-query-error header.
+	ProtocolJSON10Query
+)
+
+// JSONError describes one error for WriteJSON.
+type JSONError struct {
+	// Code is the AWS error code.
+	Code string
+	// Verbatim sends Code as the wire error type. Otherwise "Exception" is
+	// appended when absent, as AWS does for errors raised outside a service
+	// model.
+	Verbatim bool
+	Message  string
+	Status   int
+	// Fields are extra top-level body keys; they never replace a reserved key.
+	Fields map[string]string
+	// Namespace is the Smithy shape namespace prefixed to __type as
+	// "<namespace>#<code>" on ProtocolJSON10Query, for a Verbatim code only.
+	// A code that is not Verbatim is a framework error and always takes
+	// CoralNamespace on that protocol.
+	Namespace string
+	// QueryCode is the legacy query code for ProtocolJSON10Query; Code is used
+	// when empty.
+	QueryCode string
+}
+
+// senderFault reports whether the error is the caller's ("Sender"/"User") or
+// the service's ("Receiver"/"Service"), derived from the HTTP status.
+func (e JSONError) senderFault() bool { return e.Status < http.StatusInternalServerError }
+
+// json10Type is the __type of e on JSON 1.0: a framework error is a coral
+// type, a service error its code under the service namespace when it has one.
+func (e JSONError) json10Type() string {
+	if !e.Verbatim {
+		name, ok := coralTypes[e.Code]
+		if !ok {
+			name = JSONErrorType(e.Code)
+		}
+		return CoralNamespace + "#" + name
+	}
+	if e.Namespace != "" {
+		return e.Namespace + "#" + e.Code
+	}
+	return e.Code
+}
+
+func (e JSONError) wireType() string {
+	if e.Verbatim {
+		return e.Code
+	}
+	return JSONErrorType(e.Code)
+}
+
+// JSONContentTypeFor returns the content type p's responses use.
+func JSONContentTypeFor(p Protocol) string {
+	switch p {
+	case ProtocolRESTJSON:
+		return RESTJSONContentType
+	case ProtocolJSON10Query:
+		return JSON10ContentType
+	default:
+		return JSONContentType
+	}
+}
+
+// JSONErrorBody renders e's body for p.
+func JSONErrorBody(p Protocol, e JSONError) []byte {
+	typeKey, typeVal := "__type", e.wireType()
+	if p == ProtocolJSON10Query {
+		typeVal = e.json10Type()
+	}
+	if p == ProtocolRESTJSON {
+		typeKey, typeVal = "Type", "User"
+		if !e.senderFault() {
+			typeVal = "Service"
+		}
+	}
+	body := map[string]string{}
+	maps.Copy(body, e.Fields)
+	body[typeKey] = typeVal
+	// AWS sends no message key at all for some JSON 1.0 errors.
+	if e.Message != "" || p != ProtocolJSON10Query {
+		body["message"] = e.Message
+	}
+	out, err := json.Marshal(body)
+	if err != nil {
+		slog.Error("envelope: failed to marshal JSON error", "code", e.Code, "err", err)
+		return fmt.Appendf(nil, `{"%s":"InternalError","message":%q}`, typeKey, e.Message)
+	}
+	return out
+}
+
+// WriteJSON writes e as a complete response in p's wire format.
+func WriteJSON(w http.ResponseWriter, p Protocol, e JSONError) {
+	h := w.Header()
+	h.Set("Content-Type", JSONContentTypeFor(p))
+	if p == ProtocolJSON10Query {
+		fault := "Sender"
+		if !e.senderFault() {
+			fault = "Receiver"
+		}
+		code := e.QueryCode
+		if code == "" {
+			code = e.Code
+		}
+		h.Set(QueryErrorHeader, code+";"+fault)
+	} else {
+		h.Set("X-Amzn-Errortype", e.wireType())
+	}
+	w.WriteHeader(e.Status)
+	if _, err := w.Write(JSONErrorBody(p, e)); err != nil {
+		slog.Error("envelope: failed to write JSON error response", "err", err)
+	}
+}
+
+// WriteTooLargeText writes the oversize-request rejection AWS JSON 1.0
+// services send: a plain-text 413 with no error type and no query header.
+func WriteTooLargeText(w http.ResponseWriter, limit int64) {
+	h := w.Header()
+	h.Set("Content-Type", "text; charset=utf-8")
+	h.Set("Connection", "close")
+	w.WriteHeader(http.StatusRequestEntityTooLarge)
+	if _, err := fmt.Fprintf(w, "HTTP content length exceeded %d bytes.", limit); err != nil {
+		slog.Error("envelope: failed to write oversize response", "err", err)
+	}
+}
@@ -270,3 +270,146 @@ func TestRequestSignalsJSONProtocol(t *testing.T) {
 		})
 	}
 }
+
+func TestWriteTooLargeText(t *testing.T) {
+	w := httptest.NewRecorder()
+	WriteTooLargeText(w, 2097152)
+	resp := w.Result()
+	assert.Equal(t, http.StatusRequestEntityTooLarge, resp.StatusCode)
+	assert.Equal(t, "text; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.Equal(t, "close", resp.Header.Get("Connection"))
+	assert.Empty(t, resp.Header.Get(QueryErrorHeader))
+	assert.Equal(t, "HTTP content length exceeded 2097152 bytes.", w.Body.String())
+}
+
+func TestWriteJSON_WireBytesPerProtocol(t *testing.T) {
+	cases := []struct {
+		name         string
+		p            Protocol
+		e            JSONError
+		contentType  string
+		headers      map[string]string
+		absentHeader string
+		body         string
+	}{
+		{
+			name:        "JSON 1.1 appends Exception",
+			p:           ProtocolJSON11,
+			e:           JSONError{Code: "Throttling", Message: "slow", Status: 400},
+			contentType: "application/x-amz-json-1.1",
+			headers:     map[string]string{"X-Amzn-Errortype": "ThrottlingException"},
+			body:        `{"__type":"ThrottlingException","message":"slow"}`,
+		},
+		{
+			name:        "REST-JSON keeps a verbatim type and sorts extra fields",
+			p:           ProtocolRESTJSON,
+			e:           JSONError{Code: "TooManyRequestsException", Verbatim: true, Message: "Rate Exceeded.", Status: 429, Fields: map[string]string{"Reason": "CallerRateLimitExceeded"}},
+			contentType: "application/json",
+			headers:     map[string]string{"X-Amzn-Errortype": "TooManyRequestsException"},
+			body:        `{"Reason":"CallerRateLimitExceeded","Type":"User","message":"Rate Exceeded."}`,
+		},
+		{
+			name:        "REST-JSON common error gets Exception and a 5xx is a Service fault",
+			p:           ProtocolRESTJSON,
+			e:           JSONError{Code: "ServiceUnavailable", Message: "down", Status: 503},
+			contentType: "application/json",
+			headers:     map[string]string{"X-Amzn-Errortype": "ServiceUnavailableException"},
+			body:        `{"Type":"Service","message":"down"}`,
+		},
+		{
+			name:        "REST-JSON extra fields cannot replace reserved keys",
+			p:           ProtocolRESTJSON,
+			e:           JSONError{Code: "X", Verbatim: true, Message: "m", Status: 400, Fields: map[string]string{"Type": "Evil", "message": "evil"}},
+			contentType: "application/json",
+			body:        `{"Type":"User","message":"m"}`,
+		},
+		{
+			name:         "JSON 1.0 never suffixes and sends the query header",
+			p:            ProtocolJSON10Query,
+			e:            JSONError{Code: "QueueDoesNotExist", Verbatim: true, Message: "gone", Status: 400, QueryCode: "AWS.SimpleQueueService.NonExistentQueue"},
+			contentType:  "application/x-amz-json-1.0",
+			headers:      map[string]string{"X-Amzn-Query-Error": "AWS.SimpleQueueService.NonExistentQueue;Sender"},
+			absentHeader: "X-Amzn-Errortype",
+			body:         `{"__type":"QueueDoesNotExist","message":"gone"}`,
+		},
+		{
+			name:        "JSON 1.0 namespaces a verbatim type only in the body",
+			p:           ProtocolJSON10Query,
+			e:           JSONError{Code: "QueueDoesNotExist", Verbatim: true, Namespace: "com.amazonaws.sqs", Message: "gone", Status: 400},
+			contentType: "application/x-amz-json-1.0",
+			headers:     map[string]string{"X-Amzn-Query-Error": "QueueDoesNotExist;Sender"},
+			body:        `{"__type":"com.amazonaws.sqs#QueueDoesNotExist","message":"gone"}`,
+		},
+		{
+			name:        "JSON 1.0 captured QueueDoesNotExist",
+			p:           ProtocolJSON10Query,
+			e:           JSONError{Code: "QueueDoesNotExist", Verbatim: true, Namespace: "com.amazonaws.sqs", Message: "The specified queue does not exist.", Status: 400, QueryCode: "AWS.SimpleQueueService.NonExistentQueue"},
+			contentType: "application/x-amz-json-1.0",
+			headers:     map[string]string{"X-Amzn-Query-Error": "AWS.SimpleQueueService.NonExistentQueue;Sender"},
+			body:        `{"__type":"com.amazonaws.sqs#QueueDoesNotExist","message":"The specified queue does not exist."}`,
+		},
+		{
+			name:        "JSON 1.0 captured framework error raised by a service",
+			p:           ProtocolJSON10Query,
+			e:           JSONError{Code: "MissingRequiredParameterException", Verbatim: true, Namespace: CoralNamespace, Message: "The request must contain the parameter MessageBody.", Status: 400, QueryCode: "MissingParameter"},
+			contentType: "application/x-amz-json-1.0",
+			headers:     map[string]string{"X-Amzn-Query-Error": "MissingParameter;Sender"},
+			body:        `{"__type":"com.amazon.coral.service#MissingRequiredParameterException","message":"The request must contain the parameter MessageBody."}`,
+		},
+		{
+			name:        "JSON 1.0 captured unknown operation has no message",
+			p:           ProtocolJSON10Query,
+			e:           JSONError{Code: "InvalidAction", Status: 400},
+			contentType: "application/x-amz-json-1.0",
+			headers:     map[string]string{"X-Amzn-Query-Error": "InvalidAction;Sender"},
+			body:        `{"__type":"com.amazon.coral.service#UnknownOperationException"}`,
+		},
+		{
+			name:        "JSON 1.0 captured wrong secret key",
+			p:           ProtocolJSON10Query,
+			e:           JSONError{Code: "SignatureDoesNotMatch", Message: "sig", Status: 403},
+			contentType: "application/x-amz-json-1.0",
+			headers:     map[string]string{"X-Amzn-Query-Error": "SignatureDoesNotMatch;Sender"},
+			body:        `{"__type":"com.amazon.coral.service#InvalidSignatureException","message":"sig"}`,
+		},
+		{
+			name:        "JSON 1.0 captured unknown token",
+			p:           ProtocolJSON10Query,
+			e:           JSONError{Code: "InvalidClientTokenId", Message: "The security token included in the request is invalid", Status: 403},
+			contentType: "application/x-amz-json-1.0",
+			headers:     map[string]string{"X-Amzn-Query-Error": "InvalidClientTokenId;Sender"},
+			body:        `{"__type":"com.amazon.coral.service#UnrecognizedClientException","message":"The security token included in the request is invalid"}`,
+		},
+		{
+			name:        "JSON 1.1 and REST-JSON still send a message when empty",
+			p:           ProtocolRESTJSON,
+			e:           JSONError{Code: "X", Verbatim: true, Status: 400},
+			contentType: "application/json",
+			body:        `{"Type":"User","message":""}`,
+		},
+		{
+			name:        "JSON 1.0 query code defaults to the code and a 5xx is a Receiver",
+			p:           ProtocolJSON10Query,
+			e:           JSONError{Code: "InternalError", Verbatim: true, Message: "boom", Status: 500},
+			contentType: "application/x-amz-json-1.0",
+			headers:     map[string]string{"X-Amzn-Query-Error": "InternalError;Receiver"},
+			body:        `{"__type":"InternalError","message":"boom"}`,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			w := httptest.NewRecorder()
+			WriteJSON(w, tc.p, tc.e)
+			resp := w.Result()
+			assert.Equal(t, tc.e.Status, resp.StatusCode)
+			assert.Equal(t, tc.contentType, resp.Header.Get("Content-Type"))
+			for k, v := range tc.headers {
+				assert.Equal(t, v, resp.Header.Get(k), k)
+			}
+			if tc.absentHeader != "" {
+				assert.Empty(t, resp.Header.Get(tc.absentHeader))
+			}
+			assert.Equal(t, tc.body, w.Body.String())
+		})
+	}
+}
@@ -55,9 +55,16 @@ type ConfigSettings struct {
 	AccountID string
 	Region    string
 	NatsToken string
-	DataDir   string
-	LogDir    string
-	ConfigDir string
+	// SQSStoreToken authenticates clients and routes of the dedicated SQS
+	// message store. It is distinct from NatsToken and is rendered only into
+	// sqs-store.conf. Empty skips the store's config.
+	SQSStoreToken string
+	// SQSStoreRoutes are the "host:port" cluster routes of the SQS store;
+	// empty on a single node.
+	SQSStoreRoutes []string
+	DataDir        string
+	LogDir         string
+	ConfigDir      string
 	// PredastoreDataDir is the [[host]] data_dir rendered into predastore.toml.
 	// Derived from DataDir via PredastoreDataDir when left unset.
 	PredastoreDataDir string
@@ -316,6 +323,7 @@ func CreateServiceDirectories(spxRoot string) {
 		filepath.Join(spxRoot, "state"),
 		filepath.Join(spxRoot, "logs"),
 		filepath.Join(spxRoot, "nats"),
+		filepath.Join(spxRoot, "sqs-store"),
 		filepath.Join(spxRoot, "predastore"),
 		filepath.Join(spxRoot, "viperblock"),
 		filepath.Join(spxRoot, "vpcd"),
@@ -393,6 +401,8 @@ func ChownRecursive(path, username string) error {
 var servicePaths = map[string]string{
 	"/etc/spinifex/nats":           "spinifex-nats",
 	"/var/lib/spinifex/nats":       "spinifex-nats",
+	"/etc/spinifex/sqs-store":      "spinifex-sqsstore",
+	"/var/lib/spinifex/sqs-store":  "spinifex-sqsstore",
 	"/etc/spinifex/predastore":     "spinifex-storage",
 	"/var/lib/spinifex/predastore": "spinifex-storage",
 	"/etc/spinifex/northstar":      "spinifex-northstar",
@@ -636,6 +646,24 @@ type NorthstarCredentials struct {
 	Bucket    string
 }

+// SQSStoreClusterPort is the route port of the SQS store, distinct from the
+// control-plane NATS cluster port 4248.
+const SQSStoreClusterPort = "4249"
+
+// SQSStoreRoutes maps control-plane NATS cluster routes ("host:4248") to the
+// SQS store's route addresses on the same hosts.
+func SQSStoreRoutes(natsRoutes []string) []string {
+	var out []string
+	for _, r := range natsRoutes {
+		host, _, err := net.SplitHostPort(strings.TrimSpace(r))
+		if err != nil || host == "" {
+			continue
+		}
+		out = append(out, net.JoinHostPort(host, SQSStoreClusterPort))
+	}
+	return out
+}
+
 // GenerateNATSToken generates a secure random token for NATS.
 func GenerateNATSToken() (string, error) {
 	bytes := make([]byte, 32)
@@ -339,6 +339,9 @@ func deletePrefix(store objectstore.ObjectStore, bucket, prefix string) (int, in
 				Bucket: aws.String(bucket),
 				Key:    obj.Key,
 			}); err != nil {
+				if objectstore.IsNoSuchKeyError(err) {
+					continue
+				}
 				return count, bytes, fmt.Errorf("delete %s: %w", *obj.Key, err)
 			}
 			count++
@@ -123,39 +123,46 @@ func TestRemoveSystemImage_HappyPath(t *testing.T) {
 	assert.Equal(t, 0, store.Count())
 }

-// strictDeleteStore reports a missing key on delete, the way predastore does.
-// The in-memory store is silently tolerant and AWS S3 answers 204, so neither
-// reproduces the backend this actually runs against.
-type strictDeleteStore struct {
-	*objectstore.MemoryObjectStore
-}
-
-var _ objectstore.ObjectStore = (*strictDeleteStore)(nil)
-
-func (s *strictDeleteStore) DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput) (*awss3.DeleteObjectOutput, error) {
-	key := aws.StringValue(input.Key)
-	if _, err := s.HeadObject(ctx, &awss3.HeadObjectInput{Bucket: input.Bucket, Key: input.Key}); err != nil {
-		return nil, &objectstore.NoSuchKeyError{Key: key}
-	}
-	return s.MemoryObjectStore.DeleteObject(ctx, input)
-}
-
 // TestRemoveSystemImage_OrphanedBlocksAreReclaimable covers the state the
 // orphan report points operators at: blocks in the object store with no
 // document. A live env19 run failed here on NoSuchKey, so the one documented
 // way to reclaim an orphan was broken by the very thing that defines one.
 func TestRemoveSystemImage_OrphanedBlocksAreReclaimable(t *testing.T) {
 	memory := objectstore.NewMemoryObjectStore()
-	store := &strictDeleteStore{MemoryObjectStore: memory}
 	const id = "ami-orphaned0001"
 	putAMIBlocks(t, memory, id, 3, 128)

-	res, err := RemoveSystemImage(store, testRemoveBucket, RemoveImageOpts{ImageID: id, Force: true})
+	res, err := RemoveSystemImage(memory, testRemoveBucket, RemoveImageOpts{ImageID: id, Force: true})
 	require.NoError(t, err, "an image whose document is already gone must still have its blocks reclaimable")
 	assert.Equal(t, 3, res.ObjectsDeleted)
 	assert.Equal(t, 0, memory.Count(), "the stranded blocks must actually be released")
 }

+// racingDeleteStore lets a concurrent deleter win every DeleteObject, so the
+// caller's own delete answers NoSuchKey as Predastore would.
+type racingDeleteStore struct {
+	*objectstore.MemoryObjectStore
+}
+
+func (s racingDeleteStore) DeleteObject(ctx context.Context, input *awss3.DeleteObjectInput) (*awss3.DeleteObjectOutput, error) {
+	_, _ = s.MemoryObjectStore.DeleteObject(ctx, input)
+	return s.MemoryObjectStore.DeleteObject(ctx, input)
+}
+
+// A block another remover deleted between the listing and this delete is
+// gone, so the sweep carries on rather than stranding the rest of the prefix.
+// It is not counted: this run did not release it.
+func TestRemoveSystemImage_ConcurrentlyDeletedBlocksAreSkipped(t *testing.T) {
+	memory := objectstore.NewMemoryObjectStore()
+	const id = "ami-racing000001"
+	putAMIBlocks(t, memory, id, 3, 128)
+
+	res, err := RemoveSystemImage(racingDeleteStore{MemoryObjectStore: memory}, testRemoveBucket, RemoveImageOpts{ImageID: id, Force: true})
+	require.NoError(t, err)
+	assert.Equal(t, 0, res.ObjectsDeleted)
+	assert.Equal(t, 0, memory.Count())
+}
+
 func TestRemoveSystemImage_AccountOwned_Refused(t *testing.T) {
 	store := objectstore.NewMemoryObjectStore()
 	const id = "ami-acct-001"
@@ -0,0 +1,13 @@
+package admin
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSQSStoreRoutes(t *testing.T) {
+	got := SQSStoreRoutes([]string{"10.0.0.1:4248", " 10.0.0.2:4248", "bad", "[fd00::3]:4248"})
+	assert.Equal(t, []string{"10.0.0.1:4249", "10.0.0.2:4249", "[fd00::3]:4249"}, got)
+	assert.Empty(t, SQSStoreRoutes(nil))
+}
@@ -66,6 +66,9 @@ var predastoreTomlTemplate string
 //go:embed templates/nats.conf
 var natsConfTemplate string

+//go:embed templates/sqs-store.conf
+var sqsStoreConfTemplate string
+
 //go:embed templates/predastore-multinode.toml
 var predastoreMultiNodeTemplate string

@@ -1736,6 +1739,12 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	}
 	fmt.Println("\n🔒 Generated NATS authentication token")

+	sqsStoreToken, err := admin.GenerateNATSToken()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error generating SQS store token: %v\n", err)
+		os.Exit(1)
+	}
+
 	if spxRoot == "" {
 		spxRoot = DefaultDataDir()
 	}
@@ -1875,9 +1884,11 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 		AccountID: accountID,
 		Region:    region,
 		NatsToken: natsToken,
-		DataDir:   spxRoot,
-		LogDir:    LogDirFor(spxRoot),
-		ConfigDir: configDir,
+		// Held only by the SQS store config; never by spinifex.toml.
+		SQSStoreToken: sqsStoreToken,
+		DataDir:       spxRoot,
+		LogDir:        LogDirFor(spxRoot),
+		ConfigDir:     configDir,

 		Node:          node,
 		Az:            az,
@@ -2046,7 +2057,14 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		os.Exit(1)
 	}

+	sqsStoreToken, err := admin.GenerateNATSToken()
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Error generating SQS store token: %v\n", err)
+		os.Exit(1)
+	}
+
 	creds := &formation.SharedCredentials{
+		SQSStoreToken:  sqsStoreToken,
 		AccessKey:      accessKey,
 		SecretKey:      secretKey,
 		AccountID:      accountID,
@@ -2203,6 +2221,9 @@ func runAdminInitMultiNode(cmd *cobra.Command, accessKey, secretKey, accountID,
 		ClusterRoutes: clusterRoutes,
 		ClusterName:   clusterName,

+		SQSStoreToken:  sqsStoreToken,
+		SQSStoreRoutes: admin.SQSStoreRoutes(clusterRoutes),
+
 		PredastoreHostID:          predastoreHostID,
 		PredastoreAdminPort:       admin.PredastoreAdminPort,
 		CompactionIntervalSeconds: compactionInterval,
@@ -2783,6 +2804,9 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 		ClusterRoutes: clusterRoutes,
 		ClusterName:   creds.ClusterName,

+		SQSStoreToken:  creds.SQSStoreToken,
+		SQSStoreRoutes: admin.SQSStoreRoutes(clusterRoutes),
+
 		PredastoreHostID:          predastoreHostID,
 		PredastoreAdminPort:       admin.PredastoreAdminPort,
 		CompactionIntervalSeconds: compactionInterval,
@@ -3148,6 +3172,7 @@ type configDirs struct {
 	Predastore string
 	Viperblock string
 	NATS       string
+	SQSStore   string
 	Spinifex   string
 	Northstar  string
 }
@@ -3159,10 +3184,11 @@ func createConfigSubdirs(configDir string) (configDirs, error) {
 		Predastore: filepath.Join(configDir, "predastore"),
 		Viperblock: filepath.Join(configDir, "viperblock"),
 		NATS:       filepath.Join(configDir, "nats"),
+		SQSStore:   filepath.Join(configDir, "sqs-store"),
 		Spinifex:   filepath.Join(configDir, "spinifex"),
 		Northstar:  filepath.Join(configDir, "northstar"),
 	}
-	for _, dir := range []string{dirs.AWSGW, dirs.Predastore, dirs.Viperblock, dirs.NATS, dirs.Spinifex, dirs.Northstar} {
+	for _, dir := range []string{dirs.AWSGW, dirs.Predastore, dirs.Viperblock, dirs.NATS, dirs.SQSStore, dirs.Spinifex, dirs.Northstar} {
 		if err := os.MkdirAll(dir, 0700); err != nil {
 			return configDirs{}, fmt.Errorf("create directory %s: %w", dir, err)
 		}
@@ -3245,6 +3271,13 @@ func generateAndWriteConfigs(dirs configDirs, spinifexTomlPath string, settings
 		{Name: filepath.Join(dirs.AWSGW, "awsgw.toml"), Path: filepath.Join(dirs.AWSGW, "awsgw.toml"), Template: awsgwTomlTemplate},
 		{Name: filepath.Join(dirs.NATS, "nats.conf"), Path: filepath.Join(dirs.NATS, "nats.conf"), Template: natsConfTemplate},
 	}
+	// A cluster formed by a leader that predates the store distributes no
+	// token, so no store config is rendered rather than one nobody can join.
+	if settings.SQSStoreToken != "" {
+		configs = append(configs, admin.ConfigFile{
+			Name: filepath.Join(dirs.SQSStore, "sqs-store.conf"), Path: filepath.Join(dirs.SQSStore, "sqs-store.conf"), Template: sqsStoreConfTemplate,
+		})
+	}
 	// northstar.toml is only rendered when scoped DNS credentials were
 	// provisioned. A cluster formed by a leader that predates their distribution
 	// leaves the keys empty, yielding no northstar config rather than a partial
@@ -22,6 +22,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/qemunbdd"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/qmpcollector"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/spinifexui"
+	"github.com/mulgadc/spinifex/spinifex/runtime/roles/sqsstore"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/vpcd"
 	"github.com/mulgadc/spinifex/spinifex/runtime/service"
 	"github.com/spf13/cobra"
@@ -71,6 +72,11 @@ var natsCmd = &cobra.Command{
 	Short: "Manage the nats service",
 }

+var sqsStoreCmd = &cobra.Command{
+	Use:   "sqs-store",
+	Short: "Manage the dedicated SQS message store (JetStream) service",
+}
+
 var spinifexCmd = &cobra.Command{
 	Use:   "spinifex",
 	Short: "Manage the spinifex service",
@@ -492,6 +498,27 @@ var natsStartCmd = &cobra.Command{
 	},
 }

+var sqsStoreStartCmd = &cobra.Command{
+	Use:           "start",
+	Short:         "Start the dedicated SQS message store service",
+	SilenceErrors: true,
+	SilenceUsage:  true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		defer initTelemetry("sqs-store", false)()
+
+		service, err := service.New("sqs-store", &sqsstore.Config{
+			ConfigFile: viper.GetString("config"),
+		})
+		if err != nil {
+			return fmt.Errorf("create sqs-store service: %w", err)
+		}
+		if _, err = service.Start(); err != nil {
+			return fmt.Errorf("start sqs-store service: %w", err)
+		}
+		return nil
+	},
+}
+
 // Repeat for spinifex.
 var spinifexStartCmd = &cobra.Command{
 	Use:           "start",
@@ -1149,6 +1176,9 @@ func init() {

 	natsCmd.AddCommand(natsStartCmd)

+	serviceCmd.AddCommand(sqsStoreCmd)
+	sqsStoreCmd.AddCommand(sqsStoreStartCmd)
+
 	// Add NATS flags
 	natsCmd.PersistentFlags().Int("port", 4222, "NATS server port")
 	natsCmd.PersistentFlags().String("host", "0.0.0.0", "NATS server host")
@@ -217,6 +217,14 @@ func TestServiceStartCmdsReturnErrorOnMissingConfig(t *testing.T) {
 			},
 			wantErr: "start nats service",
 		},
+		{
+			name: "sqs-store start (missing config file)",
+			cmd:  sqsStoreStartCmd,
+			setup: func(t *testing.T) {
+				resetGlobalViper(t)
+			},
+			wantErr: "create sqs-store service",
+		},
 		{name: "spinifex start (missing config)", cmd: spinifexStartCmd},
 		{
 			name: "spinifex start (config file load failure)",
@@ -0,0 +1,167 @@
+//test:in-package — the embedded config templates it renders are unexported,
+// and exporting them would widen the package's API for a test's convenience.
+
+package cli
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/sha256"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/hex"
+	"encoding/pem"
+	"math/big"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/operator/admin"
+	"github.com/mulgadc/spinifex/spinifex/runtime/roles/sqsstore"
+	natsserver "github.com/nats-io/nats-server/v2/server"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// writeTestCerts writes server.pem, server.key and ca.pem under dir so the
+// rendered configs, which name them in their tls blocks, load.
+func writeTestCerts(t *testing.T, dir string) {
+	t.Helper()
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+	tmpl := &x509.Certificate{
+		SerialNumber:          big.NewInt(1),
+		Subject:               pkix.Name{CommonName: "test"},
+		NotBefore:             time.Now().Add(-time.Hour),
+		NotAfter:              time.Now().Add(time.Hour),
+		IsCA:                  true,
+		BasicConstraintsValid: true,
+		KeyUsage:              x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
+	require.NoError(t, err)
+	keyDER, err := x509.MarshalECPrivateKey(key)
+	require.NoError(t, err)
+	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "server.pem"), certPEM, 0o600))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "ca.pem"), certPEM, 0o600))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "server.key"),
+		pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600))
+}
+
+// renderBoth renders the control-plane and SQS store configs from one set of
+// node settings, exactly as admin init does, and parses each with nats-server.
+func renderBoth(t *testing.T, routes []string) (control, store *natsserver.Options) {
+	t.Helper()
+	root := t.TempDir()
+	writeTestCerts(t, root)
+	dirs, err := createConfigSubdirs(root)
+	require.NoError(t, err)
+
+	settings := admin.ConfigSettings{
+		Node: "node1", BindIP: "10.0.0.1", ClusterBindIP: "10.0.0.1", ClusterName: "spinifex",
+		NatsToken: "nats_control", SQSStoreToken: "nats_sqsstore",
+		ConfigDir: root, DataDir: filepath.Join(root, "data"), LogDir: filepath.Join(root, "log"),
+	}
+	if routes != nil {
+		settings.ClusterRoutes = routes
+		settings.SQSStoreRoutes = admin.SQSStoreRoutes(routes)
+	}
+	controlPath := filepath.Join(dirs.NATS, "nats.conf")
+	storePath := filepath.Join(dirs.SQSStore, "sqs-store.conf")
+	require.NoError(t, admin.GenerateConfigFile(controlPath, natsConfTemplate, settings))
+	require.NoError(t, admin.GenerateConfigFile(storePath, sqsStoreConfTemplate, settings))
+
+	control, err = natsserver.ProcessConfigFile(controlPath)
+	require.NoError(t, err)
+	store, err = natsserver.ProcessConfigFile(storePath)
+	require.NoError(t, err)
+	return control, store
+}
+
+func TestSQSStoreConfSharesNothingWithControlPlane(t *testing.T) {
+	control, store := renderBoth(t, []string{"10.0.0.1:4248", "10.0.0.2:4248", "10.0.0.3:4248"})
+
+	assert.NotEqual(t, control.Port, store.Port, "client port")
+	assert.NotEqual(t, control.HTTPPort, store.HTTPPort, "monitoring port")
+	assert.NotEqual(t, control.Cluster.Port, store.Cluster.Port, "cluster port")
+	assert.NotEqual(t, control.Cluster.Name, store.Cluster.Name, "cluster name")
+	assert.NotEqual(t, control.StoreDir, store.StoreDir, "store directory")
+	assert.NotEqual(t, control.ServerName, store.ServerName, "server name")
+	assert.NotEqual(t, control.Authorization, store.Authorization, "client token")
+	assert.NotEqual(t, control.Cluster.Password, store.Cluster.Password, "route credential")
+	assert.NotEqual(t, control.LogFile, store.LogFile, "log file")
+
+	assert.Empty(t, store.LeafNode.Host+store.Gateway.Host, "no leaf or gateway")
+	require.Len(t, store.Routes, 3)
+	for _, r := range store.Routes {
+		assert.Equal(t, "4249", r.Port(), "store routes target the store cluster port")
+	}
+	for _, r := range control.Routes {
+		assert.Equal(t, "4248", r.Port())
+	}
+}
+
+func TestSQSStoreConfDurabilityAndPayload(t *testing.T) {
+	control, store := renderBoth(t, nil)
+
+	assert.True(t, store.SyncAlways, "store must fsync before acknowledging")
+	assert.Equal(t, int32(2*1024*1024), store.MaxPayload)
+	assert.NoError(t, sqsstore.ValidateOptions(store))
+
+	assert.Equal(t, int32(1024*1024), control.MaxPayload, "control-plane max_payload is unchanged")
+	assert.False(t, control.SyncAlways, "control-plane sync_interval is unchanged")
+	assert.Error(t, sqsstore.ValidateOptions(control), "the control-plane config is not a valid store config")
+}
+
+func TestSQSStoreConfOmitsClusterOnSingleNode(t *testing.T) {
+	_, store := renderBoth(t, nil)
+	assert.Empty(t, store.Routes)
+	assert.Zero(t, store.Cluster.Port)
+}
+
+func TestSQSStoreTokenNeverInControlPlaneConfigs(t *testing.T) {
+	root := t.TempDir()
+	dirs, err := createConfigSubdirs(root)
+	require.NoError(t, err)
+	settings := admin.ConfigSettings{
+		Node: "node1", NatsToken: "nats_control", SQSStoreToken: "nats_sqsstore",
+		ConfigDir: root, DataDir: root, LogDir: root, Region: "r", Az: "a", Port: "9999",
+	}
+	require.NoError(t, generateAndWriteConfigs(dirs, filepath.Join(root, "spinifex.toml"), settings, true))
+
+	err = filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error {
+		if err != nil || d.IsDir() {
+			return err
+		}
+		b, rerr := os.ReadFile(p)
+		require.NoError(t, rerr)
+		inStore := strings.HasPrefix(p, dirs.SQSStore+string(filepath.Separator))
+		assert.Equal(t, inStore, strings.Contains(string(b), "nats_sqsstore"), "%s", p)
+		return nil
+	})
+	require.NoError(t, err)
+
+	_, err = os.Stat(filepath.Join(dirs.SQSStore, "sqs-store.conf"))
+	assert.NoError(t, err)
+}
+
+func TestSQSStoreConfSkippedWithoutToken(t *testing.T) {
+	root := t.TempDir()
+	dirs, err := createConfigSubdirs(root)
+	require.NoError(t, err)
+	settings := admin.ConfigSettings{Node: "node1", NatsToken: "t", ConfigDir: root, DataDir: root, LogDir: root, Region: "r", Az: "a", Port: "9999"}
+	require.NoError(t, generateAndWriteConfigs(dirs, filepath.Join(root, "spinifex.toml"), settings, true))
+	_, err = os.Stat(filepath.Join(dirs.SQSStore, "sqs-store.conf"))
+	assert.True(t, os.IsNotExist(err))
+}
+
+// TestControlPlaneNATSTemplateUnchanged pins the control-plane template so a
+// change to it is deliberate and reviewed apart from the SQS store.
+func TestControlPlaneNATSTemplateUnchanged(t *testing.T) {
+	sum := sha256.Sum256([]byte(natsConfTemplate))
+	assert.Equal(t, "afb95c63d6cbe892271656306db8540975a3d839b05ba2cf74895f0d6b6d4729", hex.EncodeToString(sum[:]))
+}
@@ -0,0 +1,68 @@
+# spinifex-config-version: 1
+# Dedicated SQS message store (JetStream)
+# Generated by Spinifex admin init
+# Shares no port, directory, account, token or route with nats.conf.
+
+server_name: spinifex-sqs-store-{{.Node}}
+
+{{- if .BindIP }}
+listen: {{.BindIP}}:4223
+{{- else }}
+listen: 0.0.0.0:4223
+{{- end }}
+
+tls {
+  cert_file: "{{.ConfigDir}}/server.pem"
+  key_file:  "{{.ConfigDir}}/server.key"
+  ca_file:   "{{.ConfigDir}}/ca.pem"
+}
+
+{{- if and .ClusterBindIP .SQSStoreRoutes }}
+
+cluster {
+  name: {{ .ClusterName }}-sqs-store
+  listen: {{.ClusterBindIP}}:4249
+
+  authorization {
+    user: sqs-store-route
+    password: "{{.SQSStoreToken}}"
+  }
+
+  tls {
+    cert_file: "{{.ConfigDir}}/server.pem"
+    key_file:  "{{.ConfigDir}}/server.key"
+    ca_file:   "{{.ConfigDir}}/ca.pem"
+    verify:    true
+  }
+
+  routes = [
+{{- range .SQSStoreRoutes }}
+    "nats-route://sqs-store-route:{{$.SQSStoreToken}}@{{ . }}",
+{{- end }}
+  ]
+}
+{{- end }}
+
+authorization {
+  token: "{{.SQSStoreToken}}"
+}
+
+jetstream {
+  # Every acknowledgement follows a completed fsync.
+  sync_interval: always
+  store_dir: "{{.DataDir}}/sqs-store/"
+  max_memory_store: 256MB
+  max_file_store: 10GB
+}
+
+debug: false
+trace: false
+logtime: true
+log_file: "{{.LogDir}}/sqs-store.log"
+
+# A 1 MiB body with 10 attributes, headers and framing.
+max_connections: 1000
+max_payload: 2MB
+max_pending: 64MB
+
+http: 127.0.0.1:8223
@@ -18,10 +18,11 @@ var Units = map[string]string{
 	"spinifex-predastore.service":      "# spinifex-unit-version: 3\n[Unit]\nDescription=Spinifex Predastore (S3-compatible Object Storage)\nPartOf=spinifex.target\nAfter=spinifex-nats.service\nWants=spinifex-nats.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\n# No MULGA_ROOT_TRACE_RATIO here: the sink's tail sampler sheds guest block I/O\n# instead, and it can see whether a trace errored. A head ratio decides before\n# the error exists, so it discarded failures at the same rate as successes.\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-storage\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service predastore start\nRestart=on-failure\nRestartSec=5\n# Bounds predastore's own graceful stop. viperblock's WAL flush writes into\n# predastore, but After= already stops viperblock first (within its own 150s\n# budget); this is the backstop if that ordering is ever bypassed.\nTimeoutStopSec=30\nOOMScoreAdjust=-500\n# Legitimately the most memory-hungry infra service (S3 buffering/caching);\n# High/Max stay well above idle so normal load isn't throttled, while still\n# bounding a leak short of the slice-wide cap.\nMemoryHigh=40%\nMemoryMax=55%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_PREDASTORE_CONFIG_PATH=/etc/spinifex/predastore/predastore.toml\nEnvironment=SPINIFEX_PREDASTORE_BASE_PATH=/var/lib/spinifex/predastore/\nEnvironment=SPINIFEX_PREDASTORE_TLS_CERT=/etc/spinifex/server.pem\nEnvironment=SPINIFEX_PREDASTORE_TLS_KEY=/etc/spinifex/server.key\nEnvironment=SPINIFEX_PREDASTORE_ENCRYPTION_KEY_FILE=/etc/spinifex/predastore/encryption.key\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/predastore /etc/spinifex/spinifex.toml /etc/spinifex/master.key /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/predastore /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
 	"spinifex-qmp-collector.service":   "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex QMP Telemetry Collector (guest metrics)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-daemon.service\nWants=spinifex-nats.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\n# spinifex-daemon user: the collector reads the daemon's telemetry QMP\n# sockets in /run/spinifex and the daemon-owned QEMU /proc entries.\nUser=spinifex-daemon\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service qmp-collector start\nRestart=on-failure\nRestartSec=5\n# Poll loop selects on ctx.Done() directly and closes NATS locally; shutdown\n# is near-instant, so this only bounds an unexpectedly wedged process.\nTimeoutStopSec=10\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/\n# Same runtime dir as the daemon: telemetry sockets + metadata live there\nEnvironment=XDG_RUNTIME_DIR=/run/spinifex\n\n# RG-9: read-mostly telemetry tier — no capabilities, no device access\nProtectSystem=full\nPrivateDevices=yes\nNoNewPrivileges=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/spinifex /run/spinifex\n\n# RG-9: hardening baseline\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\n# ProtectProc must stay default: hidepid=invisible hides the QEMU vCPU threads\n# (dumpable=0) this collector must read from /proc/<tid>/stat, even same-uid.\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=512\n\n[Install]\nWantedBy=spinifex.target\n",
 	"spinifex-shutdown.service":        "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex graceful guest drain\nPartOf=spinifex.target\n# After= all services so it stops FIRST on target/host shutdown: the drain runs\n# while nats/predastore/viperblock/daemon are still up (QMP, unmount, WAL flush).\nAfter=spinifex-nats.service spinifex-predastore.service spinifex-viperblock.service spinifex-daemon.service\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=/bin/true\n# ExecStop fires on ANY target stop, including the stop half of `systemctl\n# restart spinifex.target` (a rolling deploy) -- PartOf propagates regardless\n# of whether the host is actually going down. --unless-restarting drains on a\n# real shutdown/reboot and on a plain `systemctl stop spinifex.target`; it\n# skips only when a restart of the stack is already queued, so QEMU/nbdkit\n# survive (KillMode=process) and the daemon reconnects on the way back up.\nExecStop=/usr/local/bin/spx admin node drain --local --timeout=120s --unless-restarting\nTimeoutStopSec=180\n\n[Install]\nWantedBy=spinifex.target\n",
+	"spinifex-sqs-store.service":       "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex SQS Message Store (dedicated JetStream)\nPartOf=spinifex.target\nConditionPathExists=/etc/spinifex/sqs-store/sqs-store.conf\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-sqsstore\nGroup=spinifex\nExecStart=/usr/local/bin/spx service sqs-store start\nRestart=on-failure\nRestartSec=5\n# JetStream store shutdown and raft stepdown have no internal deadline.\nTimeoutStopSec=20\nOOMScoreAdjust=-500\n# Soft Go heap limit for the store; MemoryHigh/Max bound a runaway on top.\nEnvironment=GOMEMLIMIT=512MiB\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/sqs-store/sqs-store.conf\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/sqs-store /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/sqs-store /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
 	"spinifex-system.slice":            "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex control/infra tier (NATS, predastore, viperblock, daemon, awsgw, vpcd, ui)\nBefore=slices.target\n\n[Slice]\n# RG-3: child of spinifex.slice (spinifex-system -> spinifex), inherits the\n# parent ceiling, no extra limit. Holds the 7 control/infra services. Guest QEMU\n# moves to the sibling spinifex-guests.slice (per-instance placement lands in\n# 0002 RG-7); until then the daemon-forked guests stay under spinifex-daemon.service.\n",
 	"spinifex-ui.service":              "# spinifex-unit-version: 2\n[Unit]\nDescription=Spinifex Web UI\nPartOf=spinifex.target\nAfter=spinifex-awsgw.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-ui\nGroup=spinifex\nExecStart=/usr/local/bin/spx service spinifex-ui start\nRestart=on-failure\nRestartSec=5\n# server.Shutdown() uses an internal 30s context (WriteTimeout=120s covers\n# long streaming proxy responses, e.g. console passthrough); add margin above it.\nTimeoutStopSec=40\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_UI_TLS_CERT=/etc/spinifex/server.pem\nEnvironment=SPINIFEX_UI_TLS_KEY=/etc/spinifex/server.key\nEnvironment=SPINIFEX_UI_BASE_DIR=/var/lib/spinifex/spinifex-ui/\nEnvironment=SPINIFEX_UI_OCHRE_ENABLED=false\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/server.pem /etc/spinifex/server.key\nReadWritePaths=/var/lib/spinifex/spinifex-ui /var/log/spinifex\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
 	"spinifex-viperblock.service":      "# spinifex-unit-version: 2\n[Unit]\nDescription=Spinifex Viperblock (Block Storage)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-predastore.service\nWants=spinifex-nats.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-viperblock\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service viperblock start\nRestart=on-failure\nRestartSec=5\nOOMScoreAdjust=-500\n# Per-service ceiling bounds a leak in this service alone rather than\n# tripping the slice-wide cap, which would pick an arbitrary victim.\nMemoryHigh=15%\nMemoryMax=25%\n# KillMode=process: nbdkit still serving a guest survives a restart here, the\n# way guest QEMU survives a daemon restart. Tearing one out mid-write corrupts\n# that guest's filesystem; idle nbdkit is reaped explicitly on shutdown.\nKillMode=process\n# One worst-case nbdkit reap: killProcessGracePeriod (120s, utils.KillProcess)\n# plus its own shutdownDrainTimeout (20s, viperblock/nbd) and headroom — must\n# strictly exceed 120s now shutdownVolumes reaps concurrently, not serially.\nTimeoutStopSec=150\nEnvironmentFile=/etc/spinifex/systemd.env\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/viperblock/\n\n# Filesystem access\nNoNewPrivileges=yes\nProtectSystem=strict\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/systemd.env /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/viperblock /var/log/spinifex /run/spinifex/nbd\n\n# Hardening\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\n# ProtectProc must stay default: hidepid=invisible hides spinifex-daemon's\n# QEMU /proc entries (different uid) orphan nbdkit reaping must see.\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK\nCapabilityBoundingSet=\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
 	"spinifex-vpcd.service":            "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex VPC Daemon (OVN Networking)\nPartOf=spinifex.target\nAfter=spinifex-nats.service spinifex-daemon.service systemd-networkd.service openvswitch-switch.service ovn-controller.service\nWants=spinifex-nats.service ovn-controller.service\n\n[Service]\nEnvironmentFile=-/etc/spinifex/telemetry.env\nType=simple\nSlice=spinifex-system.slice\nUser=spinifex-vpcd\nGroup=spinifex\nExecStartPre=/var/lib/spinifex/wait-for-nats.sh\nExecStart=/usr/local/bin/spx service vpcd start\nRestart=on-failure\nRestartSec=5\n# Waits for an in-flight OVN drift-reconcile pass to return before exiting;\n# no sub-timeout bounds that call today, so this covers a normal pass.\nTimeoutStopSec=20\nOOMScoreAdjust=-500\n# Per-service ceiling: bounds a leak here instead of the slice-wide cap.\nMemoryHigh=10%\nMemoryMax=20%\nEnvironment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml\nEnvironment=SPINIFEX_BASE_DIR=/var/lib/spinifex/vpcd/\n\n# RG-9: network tier. Per-tap IMDS removed the in-process setns, so vpcd no\n# longer enters a per-VPC netns: CAP_SYS_ADMIN is dropped and the filesystem\n# sandbox is restored.\n#\n# RG-10: these are ambient so ip/iptables/sysctl/arping inherit them on exec and\n# vpcd holds no sudoers rules at all. NoNewPrivileges stays off pending the ADR\n# update that retires the last escalation path.\nAmbientCapabilities=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID\nCapabilityBoundingSet=CAP_NET_ADMIN CAP_NET_RAW CAP_NET_BIND_SERVICE CAP_SETUID CAP_SETGID\nNoNewPrivileges=no\nProtectSystem=full\nPrivateDevices=yes\nReadOnlyPaths=/etc/spinifex/spinifex.toml /etc/spinifex/server.pem /etc/spinifex/server.key /etc/spinifex/ca.pem\nReadWritePaths=/var/lib/spinifex/vpcd /var/log/spinifex\n\n# RG-9: hardening baseline\nProtectHome=yes\nPrivateTmp=yes\nRestrictSUIDSGID=yes\nLockPersonality=yes\nRestrictRealtime=yes\nRestrictNamespaces=yes\nProtectClock=yes\nProtectKernelTunables=yes\nProtectKernelModules=yes\nProtectKernelLogs=yes\nProtectControlGroups=yes\nProtectProc=invisible\nSystemCallArchitectures=native\nMemoryDenyWriteExecute=yes\n# AF_PACKET: raw L2 BOOTP for the in-process DHCP client before the iface has an IP\nRestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK AF_PACKET\nProtectHostname=yes\nKeyringMode=private\nRemoveIPC=yes\nLimitCORE=0\nLimitNOFILE=65536\nLimitNPROC=4096\n\n[Install]\nWantedBy=spinifex.target\n",
 	"spinifex.slice":                   "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex resource-bounded slice (host ceiling)\nBefore=slices.target\n\n[Slice]\n# RG-1: no CPUWeight — slice roots are siblings; capping below the default would\n# let user.slice out-prioritize the VM fleet. Operator CPU priority lives on\n# system.slice (system.slice.d/spinifex-reserve.conf).\n# RG-1/RG-5: throttle+reclaim at High, hard wall at Max (% of physical RAM).\nMemoryHigh=80%\nMemoryMax=90%\n# RG-5: spill to the full swap device before the MemoryMax wall; matched to swap size.\nMemorySwapMax=8G\n",
-	"spinifex.target":                  "# spinifex-unit-version: 1\n[Unit]\nDescription=Spinifex Infrastructure Platform\nAfter=network-online.target\nWants=network-online.target\nWants=spinifex-nats.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service\n\n[Install]\nWantedBy=multi-user.target\n",
+	"spinifex.target":                  "# spinifex-unit-version: 2\n[Unit]\nDescription=Spinifex Infrastructure Platform\nAfter=network-online.target\nWants=network-online.target\nWants=spinifex-nats.service spinifex-sqs-store.service spinifex-predastore.service spinifex-northstar.service spinifex-viperblock.service spinifex-daemon.service spinifex-awsgw.service spinifex-vpcd.service spinifex-ui.service spinifex-qmp-collector.service spinifex-shutdown.service spinifex-nats-watchdog.timer openvswitch-switch.service ovn-controller.service\n\n[Install]\nWantedBy=multi-user.target\n",
 }
@@ -129,6 +129,7 @@ func TestRG9_TierConfinement(t *testing.T) {
 	// Storage/control tier: near-zero privilege.
 	lockedDown := []string{
 		"spinifex-nats.service",
+		"spinifex-sqs-store.service",
 		"spinifex-predastore.service",
 		"spinifex-viperblock.service",
 		"spinifex-awsgw.service",
@@ -439,6 +440,7 @@ var unitBodyHashes = map[string]map[int]string{
 		3: "f5448bc2aabd3cf1af9dc74f0f12954a9294069da9f5115f8b95b529a070cbdc",
 	},
 	"spinifex-qmp-collector.service": {1: "beb18e6dd9351901f19d992cb2f757fb0e0e4a4d986402ccdb0ebb0a449f225c"},
+	"spinifex-sqs-store.service":     {1: "999641c2fe2592912445ad9e9a733649db6f324d7d8387fee69caeb4cdcbccb4"},
 	"spinifex-shutdown.service":      {1: "bcdc455916f35aa7494b2fe25e691339e8f1e22f031dfd9fd95203a9aa4bdaa4"},
 	"spinifex-system.slice":          {1: "ca450c2b28a8b13dd767957fa9469bd74bd222d7abed79945e83d564d5ce16dd"},
 	"spinifex-ui.service": {
@@ -451,7 +453,10 @@ var unitBodyHashes = map[string]map[int]string{
 	},
 	"spinifex-vpcd.service": {1: "1b722640310145767cd34e87f4804852e63f46d2145ed20f8cc5f5400ebc5965"},
 	"spinifex.slice":        {1: "f73d9343e0e1bedd647835c8bb0c80fb3a3bd66474661234ecac23a4caafc24f"},
-	"spinifex.target":       {1: "0ffba9faee5a477f8ff7466a6bccb4dc7e04f5cf92a405553c242e6548402078"},
+	"spinifex.target": {
+		1: "0ffba9faee5a477f8ff7466a6bccb4dc7e04f5cf92a405553c242e6548402078",
+		2: "16b75e47e6ea724463a7710f3af3705e6cda600df4020ca621e96dad48b150d5",
+	},
 }

 // TestUnitBodyPinnedToVersion asserts each unit's body hash matches the
@@ -266,6 +266,8 @@ func (m *MemoryObjectStore) PutObject(_ context.Context, input *s3.PutObjectInpu
 	return &s3.PutObjectOutput{}, nil
 }

+// DeleteObject answers NoSuchKeyError for an absent key, matching what
+// S3ObjectStore returns against Predastore.
 func (m *MemoryObjectStore) DeleteObject(_ context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
 	if err := checkKey(input.Key); err != nil {
 		return nil, err
@@ -274,6 +276,9 @@ func (m *MemoryObjectStore) DeleteObject(_ context.Context, input *s3.DeleteObje
 	defer m.mu.Unlock()

 	storageKey := makeKey(*input.Bucket, *input.Key)
+	if _, ok := m.objects[storageKey]; !ok {
+		return nil, &NoSuchKeyError{Key: *input.Key}
+	}
 	delete(m.objects, storageKey)
 	delete(m.modified, storageKey)

@@ -110,13 +110,15 @@ func TestMemoryObjectStore_Delete(t *testing.T) {
 func TestMemoryObjectStore_DeleteNonExistent(t *testing.T) {
 	store := NewMemoryObjectStore()

-	// Deleting non-existent should not error (matches S3 behavior)
+	// Predastore answers NoSuchKey for an absent key; the fake must too.
 	deleteInput := &s3.DeleteObjectInput{
 		Bucket: aws.String("test-bucket"),
 		Key:    aws.String("never-existed"),
 	}
 	_, err := store.DeleteObject(t.Context(), deleteInput)
-	assert.NoError(t, err)
+	var noSuchKey *NoSuchKeyError
+	require.ErrorAs(t, err, &noSuchKey)
+	assert.Equal(t, "never-existed", noSuchKey.Key)
 }

 func TestMemoryObjectStore_ListObjects(t *testing.T) {
@@ -6,6 +6,7 @@ import (

 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -15,13 +16,13 @@ import (
 // design, and a failure means the behaviour changed.

 // A non-instance ID is never resolved to a resource before the central write.
-// Another account's ID, a well-formed ID naming nothing, and an ID with no
-// recognised prefix all succeed, and each is stored under the caller's account.
+// Another account's ID and a well-formed ID naming nothing both succeed and are
+// stored under the caller's account; an unrecognised prefix is InvalidID.
 func TestCurrentBehaviour_Tagging_EC2CreateTagsStoresUnresolvedIDsInCallerNamespace(t *testing.T) {
 	const caller = "999999999999"
 	d := tagTestDaemon(t, "i-tag-cb-unrelated", nil)

-	ids := []string{"sg-0a1b2c3d4e5f60718", "vol-00000000000000000", "not-a-real-id"}
+	ids := []string{"sg-0a1b2c3d4e5f60718", "vol-00000000000000000"}
 	resources := make([]*string, 0, len(ids))
 	for _, id := range ids {
 		resources = append(resources, aws.String(id))
@@ -38,10 +39,10 @@ func TestCurrentBehaviour_Tagging_EC2CreateTagsStoresUnresolvedIDsInCallerNamesp
 		assert.Empty(t, centralTags(t, d, testAccountID, id), id)
 	}

-	out, err := d.tagsService.DescribeTags(t.Context(), &ec2.DescribeTagsInput{
-		Filters: []*ec2.Filter{{Name: aws.String("resource-id"), Values: []*string{aws.String("not-a-real-id")}}},
+	_, err = d.createTags(context.Background(), &ec2.CreateTagsInput{
+		Resources: []*string{aws.String("not-a-real-id")},
+		Tags:      []*ec2.Tag{{Key: aws.String("env"), Value: aws.String("prod")}},
 	}, caller)
-	require.NoError(t, err)
-	require.Len(t, out.Tags, 1)
-	assert.Equal(t, "unknown", aws.StringValue(out.Tags[0].ResourceType))
+	require.EqualError(t, err, awserrors.ErrorInvalidID)
+	assert.Empty(t, centralTags(t, d, caller, "not-a-real-id"))
 }
@@ -124,12 +124,16 @@ type NetworkConfig struct {

 // SharedCredentials contains the cluster-wide credentials distributed during formation.
 type SharedCredentials struct {
-	AccessKey   string `json:"access_key"`
-	SecretKey   string `json:"secret_key"`
-	AccountID   string `json:"account_id"`
-	NatsToken   string `json:"nats_token"`
-	ClusterName string `json:"cluster_name"`
-	Region      string `json:"region"`
+	AccessKey string `json:"access_key"`
+	SecretKey string `json:"secret_key"`
+	AccountID string `json:"account_id"`
+	NatsToken string `json:"nats_token"`
+
+	// SQSStoreToken authenticates the dedicated SQS message store's clients
+	// and routes. Omitted by a leader that predates the store.
+	SQSStoreToken string `json:"sqs_store_token,omitempty"`
+	ClusterName   string `json:"cluster_name"`
+	Region        string `json:"region"`

 	// Admin credentials (generated by init node, shared to join nodes)
 	AdminAccessKey string `json:"admin_access_key,omitempty"`
@@ -121,19 +121,23 @@ func pageARNs(p getResourcesPage) []string {
 	return out
 }

-// GetResources lists whatever the central index holds: an ID with no
-// recognised prefix is reported with resource type "unknown", and a
-// well-formed ID naming no resource is reported as that resource.
+// GetResources lists whatever the central index holds: a well-formed ID naming
+// no resource is reported as that resource. An ID with no recognised prefix
+// never reaches the index, because CreateTags rejects it as InvalidID.
 func TestCurrentBehaviour_Tagging_GetResourcesListsUnresolvedEC2IDs(t *testing.T) {
 	agg, tags := newTaggingCBAggregator(t)

-	createTags(t, tags, []string{"not-a-real-id", "vol-00000000000000000"}, map[string]string{"env": "prod"})
+	_, err := tags.CreateTags(t.Context(), &ec2.CreateTagsInput{
+		Resources: aws.StringSlice([]string{"not-a-real-id"}),
+		Tags:      []*ec2.Tag{{Key: aws.String("env"), Value: aws.String("prod")}},
+	}, authzAccountID)
+	require.EqualError(t, err, awserrors.ErrorInvalidID)
+	createTags(t, tags, []string{"vol-00000000000000000"}, map[string]string{"env": "prod"})

 	page, err := getResources(t, agg, `{"ResourceTypeFilters":["ec2"]}`)
 	require.NoError(t, err)
 	prefix := "arn:aws:ec2:" + authzRegion + ":" + authzAccountID + ":"
 	assert.ElementsMatch(t, []string{
-		prefix + "unknown/not-a-real-id",
 		prefix + "volume/vol-00000000000000000",
 	}, pageARNs(page))
 }
@@ -101,7 +101,10 @@ func TestEC2IndexLister_ListsCallerIndexEntriesAsRegionalARNs(t *testing.T) {
 	nc, svc := newEC2IndexFixture(t)
 	seedEC2Tags(t, svc, listerAccount, []string{"sg-0000000000000000a"}, map[string]string{"env": "prod", "Name": "web"})
 	seedEC2Tags(t, svc, listerAccount, []string{"subnet-0000000000000000b"}, map[string]string{"env": "dev"})
-	seedEC2Tags(t, svc, listerAccount, []string{"not-a-real-id", "vol-00000000000000000"}, map[string]string{"env": "orphan"})
+	seedEC2Tags(t, svc, listerAccount, []string{"vol-00000000000000000"}, map[string]string{"env": "orphan"})
+	// CreateTags rejects an unrecognised prefix, but an entry stored before it
+	// did is still in the index and still listed.
+	require.NoError(t, svc.PutResourceTags(t.Context(), listerAccount, "not-a-real-id", map[string]string{"env": "orphan"}))
 	seedEC2Tags(t, svc, foreignAccount, []string{"sg-0000000000000000f"}, map[string]string{"env": "other"})

 	got, err := newEC2IndexLister(nc, listerRegion).ListTagged(t.Context(), listerAccount, nil)
@@ -0,0 +1,48 @@
+package sqsstore
+
+import (
+	"time"
+
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// MessageRetentionBackstop outlasts the longest queue retention (14 days) so
+// the store never expires a message before the domain does.
+const MessageRetentionBackstop = 14*24*time.Hour + time.Hour
+
+// ShardCounts is the number of shared message streams and state buckets.
+type ShardCounts struct {
+	Message int
+	State   int
+}
+
+// DefaultShardCounts returns the shard counts for a topology of the given size.
+func DefaultShardCounts(nodes int) ShardCounts {
+	if nodes >= 3 {
+		return ShardCounts{Message: 8, State: 8}
+	}
+	return ShardCounts{Message: 16, State: 8}
+}
+
+// Replicas returns the JetStream replica count for a topology: R1 on one
+// node, R3 from three.
+func Replicas(nodes int) int {
+	if nodes >= 3 {
+		return 3
+	}
+	return 1
+}
+
+// MessageStreamConfig returns the config of one shared message stream.
+// Persistence stays at the default mode so acknowledgements follow a sync.
+func MessageStreamConfig(name string, subjects []string, replicas int) jetstream.StreamConfig {
+	return jetstream.StreamConfig{
+		Name:        name,
+		Subjects:    subjects,
+		Retention:   jetstream.LimitsPolicy,
+		Storage:     jetstream.FileStorage,
+		Replicas:    replicas,
+		MaxAge:      MessageRetentionBackstop,
+		PersistMode: jetstream.DefaultPersistMode,
+	}
+}
@@ -0,0 +1,125 @@
+// Package sqsstore runs the dedicated NATS JetStream server that holds SQS
+// message data. It is a separate server from the control-plane NATS in
+// package nats and shares none of its ports, store, accounts or routes.
+package sqsstore
+
+import (
+	"fmt"
+	"log/slog"
+	"os"
+	"runtime/debug"
+
+	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
+	"github.com/mulgadc/spinifex/spinifex/runtime/roles/nats"
+	natsserver "github.com/nats-io/nats-server/v2/server"
+)
+
+const (
+	serviceName = "sqs-store"
+
+	// MinMaxPayload admits a 1 MiB body with 10 attributes, headers and framing.
+	MinMaxPayload = 2 * 1024 * 1024
+
+	// MemoryLimitBytes is the soft Go heap limit applied when GOMEMLIMIT is unset.
+	MemoryLimitBytes = 512 * 1024 * 1024
+)
+
+// Config configures the SQS store server from its own config file.
+type Config struct {
+	ConfigFile string `json:"config_file"`
+	DataDir    string `json:"data_dir"`
+}
+
+// Service runs the dedicated SQS JetStream server.
+type Service struct {
+	Config *Config
+}
+
+// New returns the SQS store service. config must be a *Config.
+func New(config any) (*Service, error) {
+	cfg, ok := config.(*Config)
+	if !ok {
+		return nil, fmt.Errorf("invalid config type for sqs-store service")
+	}
+	if cfg.ConfigFile == "" {
+		return nil, fmt.Errorf("sqs-store requires a config file")
+	}
+	return &Service{Config: cfg}, nil
+}
+
+// Start runs the server until it shuts down.
+func (svc *Service) Start() (int, error) {
+	if err := hostprocess.WritePidFileTo(svc.Config.DataDir, serviceName, os.Getpid()); err != nil {
+		return 0, fmt.Errorf("write pid file: %w", err)
+	}
+	if err := launch(svc.Config); err != nil {
+		return 0, err
+	}
+	return os.Getpid(), nil
+}
+
+// ValidateOptions rejects a server configuration that would break the
+// durability or size contract of the SQS store.
+func ValidateOptions(opts *natsserver.Options) error {
+	if !opts.JetStream {
+		return fmt.Errorf("sqs-store requires jetstream to be enabled")
+	}
+	if !opts.SyncAlways {
+		return fmt.Errorf("sqs-store requires sync_interval: always")
+	}
+	if opts.MaxPayload < MinMaxPayload {
+		return fmt.Errorf("sqs-store max_payload %d is below the %d bytes a full SQS message needs", opts.MaxPayload, MinMaxPayload)
+	}
+	return nil
+}
+
+// applyMemoryLimit sets the soft Go heap limit unless the environment already
+// chose one.
+func applyMemoryLimit() {
+	if os.Getenv("GOMEMLIMIT") == "" {
+		debug.SetMemoryLimit(MemoryLimitBytes)
+	}
+}
+
+// startServer validates the config and starts the server, returning it with
+// a function that stops the store-reserve upkeep.
+func startServer(cfg *Config) (*natsserver.Server, func(), error) {
+	opts, err := natsserver.ProcessConfigFile(cfg.ConfigFile)
+	if err != nil {
+		return nil, nil, fmt.Errorf("process sqs-store config: %w", err)
+	}
+	if err := ValidateOptions(opts); err != nil {
+		return nil, nil, err
+	}
+
+	applyMemoryLimit()
+
+	reserve := &nats.StoreReserve{Dir: opts.StoreDir, Size: nats.DefaultStoreReserveBytes}
+	reserve.Apply()
+
+	ns, err := natsserver.NewServer(opts)
+	if err != nil {
+		return nil, nil, fmt.Errorf("create sqs-store server: %w", err)
+	}
+	ns.ConfigureLogger()
+
+	if err := natsserver.Run(ns); err != nil {
+		return nil, nil, fmt.Errorf("run sqs-store server: %w", err)
+	}
+
+	stop := make(chan struct{})
+	go reserve.Maintain(stop)
+	return ns, func() { close(stop) }, nil
+}
+
+func launch(cfg *Config) error {
+	ns, stopReserve, err := startServer(cfg)
+	if err != nil {
+		return err
+	}
+	defer stopReserve()
+
+	slog.Info("SQS store server started", "server_name", ns.Name(), "store_dir", ns.StoreDir())
+	ns.WaitForShutdown()
+	return nil
+}
@@ -0,0 +1,150 @@
+package sqsstore
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime/debug"
+	"testing"
+	"time"
+
+	natsserver "github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func validOpts() *natsserver.Options {
+	return &natsserver.Options{JetStream: true, SyncAlways: true, MaxPayload: MinMaxPayload}
+}
+
+func TestValidateOptions(t *testing.T) {
+	require.NoError(t, ValidateOptions(validOpts()))
+
+	cases := map[string]func(*natsserver.Options){
+		"jetstream off":     func(o *natsserver.Options) { o.JetStream = false },
+		"periodic sync":     func(o *natsserver.Options) { o.SyncAlways = false },
+		"control-plane cap": func(o *natsserver.Options) { o.MaxPayload = 1024 * 1024 },
+	}
+	for name, mutate := range cases {
+		t.Run(name, func(t *testing.T) {
+			o := validOpts()
+			mutate(o)
+			assert.Error(t, ValidateOptions(o))
+		})
+	}
+}
+
+func TestNewRequiresConfigFile(t *testing.T) {
+	_, err := New(&Config{})
+	assert.Error(t, err)
+	_, err = New("not a config")
+	assert.Error(t, err)
+	svc, err := New(&Config{ConfigFile: "/x"})
+	require.NoError(t, err)
+	assert.NotNil(t, svc)
+}
+
+func TestMessageStreamConfigNeverAsyncPersist(t *testing.T) {
+	for _, replicas := range []int{Replicas(1), Replicas(3)} {
+		cfg := MessageStreamConfig("sqs_msg_0", []string{"sqs.msg.0.>"}, replicas)
+		assert.Equal(t, jetstream.DefaultPersistMode, cfg.PersistMode)
+		assert.Equal(t, jetstream.LimitsPolicy, cfg.Retention)
+		assert.Equal(t, jetstream.FileStorage, cfg.Storage)
+		assert.Equal(t, 14*24*time.Hour+time.Hour, cfg.MaxAge)
+		assert.Equal(t, replicas, cfg.Replicas)
+	}
+}
+
+func TestShardCountsPerTopology(t *testing.T) {
+	assert.Equal(t, ShardCounts{Message: 16, State: 8}, DefaultShardCounts(1))
+	assert.Equal(t, ShardCounts{Message: 8, State: 8}, DefaultShardCounts(3))
+	assert.Equal(t, 1, Replicas(1))
+	assert.Equal(t, 3, Replicas(3))
+}
+
+func writeConf(t *testing.T, body string) string {
+	t.Helper()
+	p := filepath.Join(t.TempDir(), "sqs-store.conf")
+	require.NoError(t, os.WriteFile(p, []byte(body), 0o600))
+	return p
+}
+
+func storeConf(t *testing.T, syncLine string, payload string) string {
+	t.Helper()
+	return writeConf(t, `
+server_name: test-sqs-store
+listen: 127.0.0.1:-1
+jetstream {
+  `+syncLine+`
+  store_dir: "`+t.TempDir()+`"
+}
+max_payload: `+payload+`
+`)
+}
+
+func TestStartServerRefusesInvalidConfig(t *testing.T) {
+	cases := map[string]string{
+		"missing file":    filepath.Join(t.TempDir(), "absent.conf"),
+		"periodic sync":   storeConf(t, "sync_interval: 1s", "2MB"),
+		"small payload":   storeConf(t, "sync_interval: always", "1MB"),
+		"jetstream unset": writeConf(t, "listen: 127.0.0.1:-1\nmax_payload: 2MB\n"),
+	}
+	for name, path := range cases {
+		t.Run(name, func(t *testing.T) {
+			ns, _, err := startServer(&Config{ConfigFile: path})
+			require.Error(t, err)
+			assert.Nil(t, ns)
+		})
+	}
+}
+
+func TestStartServerAcceptsFullSizeMessage(t *testing.T) {
+	ns, stop, err := startServer(&Config{ConfigFile: storeConf(t, "sync_interval: always", "2MB")})
+	require.NoError(t, err)
+	defer ns.Shutdown()
+	defer stop()
+
+	nc, err := nats.Connect(ns.ClientURL())
+	require.NoError(t, err)
+	defer nc.Close()
+	js, err := nc.JetStream()
+	require.NoError(t, err)
+	_, err = js.AddStream(&nats.StreamConfig{Name: "Q", Subjects: []string{"q.>"}})
+	require.NoError(t, err)
+
+	msg := nats.NewMsg("q.one")
+	msg.Data = make([]byte, 1<<20)
+	for i := range 10 {
+		msg.Header.Set(fmt.Sprintf("attr%d", i), "v")
+	}
+	ack, err := js.PublishMsg(msg)
+	require.NoError(t, err)
+	assert.Equal(t, uint64(1), ack.Sequence)
+}
+
+func TestApplyMemoryLimit(t *testing.T) {
+	prev := debug.SetMemoryLimit(-1)
+	defer debug.SetMemoryLimit(prev)
+
+	t.Setenv("GOMEMLIMIT", "1GiB")
+	debug.SetMemoryLimit(12345)
+	applyMemoryLimit()
+	assert.Equal(t, int64(12345), debug.SetMemoryLimit(-1), "an operator-set limit is left alone")
+
+	t.Setenv("GOMEMLIMIT", "")
+	applyMemoryLimit()
+	assert.Equal(t, int64(MemoryLimitBytes), debug.SetMemoryLimit(-1))
+}
+
+func TestStartFailsOnInvalidConfigAfterWritingPidFile(t *testing.T) {
+	dir := t.TempDir()
+	svc, err := New(&Config{ConfigFile: storeConf(t, "sync_interval: 1s", "2MB"), DataDir: dir})
+	require.NoError(t, err)
+
+	pid, err := svc.Start()
+	require.Error(t, err)
+	assert.Zero(t, pid)
+	assert.FileExists(t, filepath.Join(dir, "sqs-store.pid"))
+}
@@ -13,6 +13,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/qmpcollector"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/spinifex"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/spinifexui"
+	"github.com/mulgadc/spinifex/spinifex/runtime/roles/sqsstore"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/viperblockd"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/vpcd"
 )
@@ -25,6 +26,7 @@ type Service interface {

 var (
 	_ Service = (*nats.Service)(nil)
+	_ Service = (*sqsstore.Service)(nil)
 	_ Service = (*northstar.Service)(nil)
 	_ Service = (*predastore.Service)(nil)
 	_ Service = (*viperblockd.Service)(nil)
@@ -43,6 +45,9 @@ func New(btype string, config any) (Service, error) {
 	case "nats":
 		return nats.New(config)

+	case "sqs-store":
+		return sqsstore.New(config)
+
 	case "northstar":
 		return northstar.New(config)

@@ -8,6 +8,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/nats"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/predastore"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/spinifexui"
+	"github.com/mulgadc/spinifex/spinifex/runtime/roles/sqsstore"
 	"github.com/mulgadc/spinifex/spinifex/runtime/roles/vpcd"
 	"github.com/stretchr/testify/assert"
 )
@@ -44,8 +45,12 @@ func TestNew(t *testing.T) {
 		assert.NotNil(t, svc)
 	}

+	svc, err := New("sqs-store", &sqsstore.Config{ConfigFile: "/etc/spinifex/sqs-store/sqs-store.conf"})
+	assert.NoError(t, err)
+	assert.NotNil(t, svc)
+
 	// Test unknown service type
-	svc, err := New("unknownservice", nil)
+	svc, err = New("unknownservice", nil)
 	assert.Error(t, err)
 	assert.Nil(t, svc)
 }
Author	SHA1	Message	Date
Josh Siviour	e7a3be3b4e	feat: add a dedicated SQS message store JetStream server (#1299 ) * feat: add a dedicated SQS message store JetStream server separate from control-plane NATS * test: cover the SQS store server start, validation and memory limit	2026-10-27 15:05:27 +10:00
Julian Sommer	f92d73d6af	fix: send ALB health checks as HTTP/1.1 with Host and ELB-HealthChecker User-Agent (#1296 ) HAProxy's short 'option httpchk GET <path>' form sends HTTP/1.0 with no Host header, which Envoy rejects with 426 (and 400 for HTTP/1.1 without Host), so Envoy-fronted targets never became healthy. Match the AWS request shape: GET <path> HTTP/1.1, Host set to the target IP (with the port only when it is not the scheme default), and User-Agent ELB-HealthChecker/2.0.	2026-10-27 15:41:44 +11:00
Julian Sommer	c02cfb806b	fix: default an omitted container essential flag to true in RegisterTaskDefinition (#1295 )	2026-10-27 15:41:33 +11:00
Julian Sommer	e1916c99f8	fix: make MemoryObjectStore report NoSuchKey on delete like Predastore (#1297 ) MemoryObjectStore.DeleteObject silently succeeded for an absent key while Predastore answers NoSuchKey, so handlers that treated a missing key as fatal passed unit tests and failed in production. The fake now returns NoSuchKeyError, and delete paths that must be idempotent (DeleteSnapshot's sweep and document delete, DeleteVolume, DeregisterImage, the admin image removal sweep, and ECR upload and GC cleanup) treat it as already deleted.	2026-10-27 15:40:03 +11:00
Julian Sommer	8b589c99c7	fix: reject CreateTags resource ids with an unrecognised prefix as InvalidID (#1298 )	2026-10-27 15:39:51 +11:00
Josh Siviour	5216b18f88	feat: render AWS errors in each registration's declared protocol (REST-JSON, JSON 1.0 query-compatible) (#1294 ) * feat: let a registration select REST-JSON or JSON 1.0 query-compatible error envelopes * feat: add a per-registration internal error code and decode envelopes through the AWS SDK v2 * feat: let a JSON 1.0 registration namespace the __type of its errors * feat: match captured SQS error bytes on the JSON 1.0 envelope	2026-10-27 13:11:43 +10:00