feat: activate Route 53 hosted zones once their parent delegates them to the cluster

A zone stays PENDING_NAMESERVERS until its parent's NS set is exactly the
cluster's name servers, read from an active enclosing zone, the platform zone,
or a non-recursive walk from the roots. The first account to verify a name wins.
Adds the activation extension (GET/POST /hostedzone/{Id}/activation), the
spx admin dns zone and delegate commands, and an e2e that delegates a cluster
subdomain and checks every node serves it.

.github/scripts/suite-sets.sh

@@ -28,6 +28,9 @@
 # that runs identically on either, and because a quota left switched on would
 # cap every suite sharing the environment — so it must be proved to restore
 # itself wherever it runs, not only on one node.
+# route53 belongs to both topologies because every node serves every hosted
+# zone from one shared bucket: on one node it proves the zone loads at all, on
+# three that each node's sync reaches the same set.
 # storagefault runs last in both lists and belongs to both topologies, for
 # different reasons. On one node it is RS(1,0), so freezing the only predastore
 # is a total outage and the guest-survival assertion is unambiguous. On three it
@@ -37,7 +40,7 @@
 # received. It is last because it SIGSTOPs a cluster-wide
 # service: anything sharing the environment would see the same outage and report
 # a failure that says nothing about itself.
-E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota storagefault"
+E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota route53 storagefault"
 
 # Suites runnable against a multi-node environment.
 #
@@ -57,7 +60,7 @@ E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota
 # replicated rather than sitting on one node, so it runs before the pair that
 # take a node away — a cluster whose buckets are already wrong should report
 # that, not report a recovery failing.
-E2E_SUITES_MULTI="multinode lb cert quota kvquorum lbrecovery instancerecovery storagefault"
+E2E_SUITES_MULTI="multinode lb cert quota route53 kvquorum lbrecovery instancerecovery storagefault"
 
 # grep -xE alternation form, for narrowing a requested set to the eligible one.
 E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')"

contracts/admission/v1/README.md

@@ -0,0 +1,42 @@
+# Admission contract, version 1
+
+This package owns the cross-process contract for the admission domain's
+quota override. It is not an in-process service interface or the owner of
+quota state.
+
+## Seal account quota override
+
+`SealAccountQuotaOverrideSubject` (`admission.v1.SealAccountQuotaOverride`) is
+a request/reply route on which a gateway, in the `awsgw-quota-seal` queue
+group, irreversibly seals one account's quota override record.
+`domains/admission/quota` owns it: `awsgw` serves it.
+`accountteardown` is the consumer: it calls this subject to fence every
+override writer at the quota owner before the account record is removed.
+
+The request names the account in its body (`account_id`) and may carry an
+opaque `correlation_ref` that is echoed for logs.
+`awsgw` ignores `X-Account-ID` for this subject: the account sealed is the one
+named in the body.
+A seal is refused with `IncorrectState` unless IAM already holds the account
+as `TERMINATING`; a precondition read failure, or no termination checker
+wired, answers `InternalError`.
+Both refusals write nothing and never open the override bucket.
+
+A reply is sent only on success: `disposition` is always `sealed` and
+`revision` is the stream leader's accepted write for that seal.
+A failure is the standard awserrors error payload, not this reply shape.
+Sealing is idempotent: a seal on an already-sealed record re-writes the same
+value and returns a fresh, higher revision.
+Any current gateway may answer; the leader-accepted revision is the evidence
+whichever one does, and the override bucket is opened at seal time even on a
+gateway whose quotas were disabled at startup.
+
+### Compatibility boundary
+
+The subject name carries the contract major; `account_quota_seal_test.go`
+preserves the version-1 JSON shape.
+Producer and consumer are in the coordinated Spinifex release, so the window
+is N and N-1.
+A gateway that predates the subject has no subscription, so the request gets
+no responder.
+An incompatible change to the subject or fields needs a new major.

contracts/admission/v1/account_quota_seal.go

@@ -0,0 +1,31 @@
+// Package admissionv1 holds the admission domain's cross-process contract:
+// the subjects, request/reply shapes and wire vocabulary the quota owner and
+// its consumers share.
+package admissionv1
+
+// SealAccountQuotaOverrideSubject is the request/reply subject on which a
+// gateway seals one account's quota override record. Sealing is irreversible
+// and refused unless IAM already holds the account as TERMINATING.
+const SealAccountQuotaOverrideSubject = "admission.v1.SealAccountQuotaOverride"
+
+// AccountQuotaOverrideDispositionSealed is the only disposition a successful
+// reply carries: a reply is sent only on success, so there is no other value.
+const AccountQuotaOverrideDispositionSealed = "sealed"
+
+// SealAccountQuotaOverrideRequest asks for one account's quota override to be
+// sealed. CorrelationRef is an opaque caller reference echoed for logs; it
+// grants nothing and is never a fence or authority to write.
+type SealAccountQuotaOverrideRequest struct {
+	AccountID      string `json:"account_id"`
+	CorrelationRef string `json:"correlation_ref,omitempty"`
+}
+
+// SealAccountQuotaOverrideReply confirms accountID's quota override record
+// was sealed at Revision, the stream leader's accepted write for that seal.
+// A failure is the standard awserrors error payload, not this type.
+type SealAccountQuotaOverrideReply struct {
+	AccountID      string `json:"account_id"`
+	CorrelationRef string `json:"correlation_ref,omitempty"`
+	Disposition    string `json:"disposition"`
+	Revision       uint64 `json:"revision"`
+}

contracts/admission/v1/account_quota_seal_test.go

@@ -0,0 +1,43 @@
+package admissionv1
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestSealAccountQuotaOverrideSubject(t *testing.T) {
+	require.Equal(t, "admission.v1.SealAccountQuotaOverride", SealAccountQuotaOverrideSubject)
+}
+
+func TestSealAccountQuotaOverrideRequestJSONContract(t *testing.T) {
+	data, err := json.Marshal(SealAccountQuotaOverrideRequest{AccountID: "111122223333", CorrelationRef: "req-1"})
+	require.NoError(t, err)
+	require.JSONEq(t, `{"account_id":"111122223333","correlation_ref":"req-1"}`, string(data))
+
+	data, err = json.Marshal(SealAccountQuotaOverrideRequest{AccountID: "111122223333"})
+	require.NoError(t, err)
+	require.JSONEq(t, `{"account_id":"111122223333"}`, string(data))
+}
+
+func TestSealAccountQuotaOverrideReplyJSONContract(t *testing.T) {
+	reply := SealAccountQuotaOverrideReply{
+		AccountID:      "111122223333",
+		CorrelationRef: "req-1",
+		Disposition:    AccountQuotaOverrideDispositionSealed,
+		Revision:       42,
+	}
+	data, err := json.Marshal(reply)
+	require.NoError(t, err)
+	require.JSONEq(t, `{
+		"account_id": "111122223333",
+		"correlation_ref": "req-1",
+		"disposition": "sealed",
+		"revision": 42
+	}`, string(data))
+
+	data, err = json.Marshal(SealAccountQuotaOverrideReply{AccountID: "111122223333", Disposition: AccountQuotaOverrideDispositionSealed})
+	require.NoError(t, err)
+	require.JSONEq(t, `{"account_id":"111122223333","disposition":"sealed","revision":0}`, string(data))
+}

contracts/ec2/v1/README.md

@@ -60,3 +60,42 @@ The subject name carries the contract major; `platform_inventory_test.go` preser
 Producer and consumer are in the coordinated Spinifex release, so the window is N and N-1.
 A daemon that predates the subject has no subscription, so the request gets no responder; the consumer treats no responder, a timeout, an error reply, an undecodable reply, a reply for another account, and a missing, repeated or unknown-status kind as `scan-incomplete`, never clean.
 An incompatible change to the subject, fields or vocabulary needs a new major.
+
+## Seal account settings
+
+`SealAccountSettingsSubject` (`ec2.v1.SealAccountSettings`) is a request/reply route on which a daemon in the `spinifex-workers` queue group irreversibly seals one account's EC2 settings record.
+EC2 owns it: `domains/ec2/account` serves it through the daemon.
+`accountteardown`'s StageAccount seal reaper is the consumer: it calls this subject only after the seal-capability probe below reports every configured node compatible, and completes only once it holds a successful sealed reply.
+
+The request names the account in its body (`account_id`) and may carry an opaque `correlation_ref` that is echoed for logs.
+The daemon ignores `X-Account-ID` for this subject: the account sealed is the one named in the body.
+A seal is refused with `IncorrectState` unless IAM already holds the account as `TERMINATING`; a precondition read failure answers `InternalError`.
+Both refusals write nothing.
+
+A reply is sent only on success: `disposition` is always `sealed` and `revision` is the stream leader's accepted write for that seal.
+A failure is the standard awserrors error payload, not this reply shape.
+Sealing is idempotent: a seal on an already-sealed record re-writes the same value and returns a fresh, higher revision.
+
+### Compatibility boundary
+
+The subject name carries the contract major; `account_settings_seal_test.go` preserves the version-1 JSON shape.
+Producer and consumer are in the coordinated Spinifex release, so the window is N and N-1.
+A daemon that predates the subject has no subscription, so the request gets no responder.
+An incompatible change to the subject or fields needs a new major.
+
+## Seal-account-settings capability
+
+`AccountSettingsSealCapabilitySubject(node)` (`ec2.v1.AccountSettingsSealCapability.<node>`) is a per-node, read-only request/reply route on which `accountteardown` asks one configured daemon whether it supports the account-settings seal, immediately before every seal attempt.
+EC2 owns it: the daemon serves it with no queue group, so a request addressed to one node reaches only subscribers configured as that node, and two daemons sharing a node ID both answer.
+`accountteardown` is the consumer: its StageAccount seal reaper probes every configured node and sends no seal request unless every one answers compatible.
+
+`AccountSettingsSealCapabilityVersion` is a claim about the whole binary's writers, not about the responder: version 1 means every EC2 account-settings writer in the binary is revision-guarded and refuses a sealed record, and the binary serves the seal subject.
+A daemon binary without this responder — predating the capability, or carrying only the seal subject without it — has no subscriber, so the probe for that node reads as incomplete (`no-reply`), the same as a stopped or partitioned node.
+The reply proves nothing beyond the moment it was sent: a daemon downgraded or replaced after a successful probe is outside the guarantee.
+
+### Compatibility boundary
+
+The subject name carries the contract major; `account_settings_seal_capability_test.go` preserves the version-1 JSON shape and the subject builder.
+Producer and consumer are in the coordinated Spinifex release, so the window is N and N-1.
+A daemon that predates the capability has no subscription, so the probe gets no responder for that node.
+An incompatible change to the subject or fields needs a new major.

contracts/ec2/v1/account_settings_seal.go

@@ -0,0 +1,28 @@
+package ec2v1
+
+// SealAccountSettingsSubject is the request/reply subject on which a daemon
+// seals one account's EC2 settings record. Sealing is irreversible and refused
+// unless IAM already holds the account as TERMINATING.
+const SealAccountSettingsSubject = "ec2.v1.SealAccountSettings"
+
+// AccountSettingsDispositionSealed is the only disposition a successful reply
+// carries: a reply is sent only on success, so there is no other value.
+const AccountSettingsDispositionSealed = "sealed"
+
+// SealAccountSettingsRequest asks for one account's EC2 settings to be
+// sealed. CorrelationRef is an opaque caller reference echoed for logs; it
+// grants nothing and is never a fence or authority to write.
+type SealAccountSettingsRequest struct {
+	AccountID      string `json:"account_id"`
+	CorrelationRef string `json:"correlation_ref,omitempty"`
+}
+
+// SealAccountSettingsReply confirms accountID's settings record was sealed at
+// Revision, the stream leader's accepted write for that seal. A failure is
+// the standard awserrors error payload, not this type.
+type SealAccountSettingsReply struct {
+	AccountID      string `json:"account_id"`
+	CorrelationRef string `json:"correlation_ref,omitempty"`
+	Disposition    string `json:"disposition"`
+	Revision       uint64 `json:"revision"`
+}

contracts/ec2/v1/account_settings_seal_capability.go

@@ -0,0 +1,39 @@
+package ec2v1
+
+import "strings"
+
+// AccountSettingsSealCapabilityVersion is the version this binary reports.
+// Version 1: every EC2 account-settings writer in the binary is
+// revision-guarded and refuses a sealed record, and the seal subject is served.
+const AccountSettingsSealCapabilityVersion = 1
+
+// AccountSettingsSealCapabilitySubject is one daemon's read-only
+// seal-capability route, addressed per node so a probe reaches exactly the
+// node it asks about.
+func AccountSettingsSealCapabilitySubject(node string) string {
+	return "ec2.v1.AccountSettingsSealCapability." + node
+}
+
+// ValidSealCapabilityNode reports whether node is exactly one NATS subject
+// token. Responder and probe both check it before building a subject, so
+// neither can address a wildcard or another node's route.
+func ValidSealCapabilityNode(node string) bool {
+	return node != "" && !strings.ContainsAny(node, ".*>\t\n\r ")
+}
+
+// AccountSettingsSealCapabilityRequest asks node whether it supports the
+// account-settings seal. CorrelationRef is an opaque caller reference echoed
+// for logs; it grants nothing and is never a fence or authority to write.
+type AccountSettingsSealCapabilityRequest struct {
+	Node           string `json:"node"`
+	CorrelationRef string `json:"correlation_ref,omitempty"`
+}
+
+// AccountSettingsSealCapabilityReply reports the responding node's capability
+// version. It carries no account data and proves nothing beyond the moment it
+// was sent.
+type AccountSettingsSealCapabilityReply struct {
+	Node              string `json:"node"`
+	CapabilityVersion int    `json:"capability_version"`
+	CorrelationRef    string `json:"correlation_ref,omitempty"`
+}

contracts/ec2/v1/account_settings_seal_capability_test.go

@@ -0,0 +1,43 @@
+package ec2v1
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestAccountSettingsSealCapabilitySubject(t *testing.T) {
+	require.Equal(t, "ec2.v1.AccountSettingsSealCapability.node-1", AccountSettingsSealCapabilitySubject("node-1"))
+	require.Equal(t, "ec2.v1.AccountSettingsSealCapability.node-2", AccountSettingsSealCapabilitySubject("node-2"))
+}
+
+func TestAccountSettingsSealCapabilityRequestJSONContract(t *testing.T) {
+	data, err := json.Marshal(AccountSettingsSealCapabilityRequest{Node: "node-1", CorrelationRef: "req-1"})
+	require.NoError(t, err)
+	require.JSONEq(t, `{"node":"node-1","correlation_ref":"req-1"}`, string(data))
+
+	data, err = json.Marshal(AccountSettingsSealCapabilityRequest{Node: "node-1"})
+	require.NoError(t, err)
+	require.JSONEq(t, `{"node":"node-1"}`, string(data))
+}
+
+func TestAccountSettingsSealCapabilityReplyJSONContract(t *testing.T) {
+	reply := AccountSettingsSealCapabilityReply{Node: "node-1", CapabilityVersion: 1, CorrelationRef: "req-1"}
+	data, err := json.Marshal(reply)
+	require.NoError(t, err)
+	require.JSONEq(t, `{"node":"node-1","capability_version":1,"correlation_ref":"req-1"}`, string(data))
+
+	data, err = json.Marshal(AccountSettingsSealCapabilityReply{Node: "node-1", CapabilityVersion: 1})
+	require.NoError(t, err)
+	require.JSONEq(t, `{"node":"node-1","capability_version":1}`, string(data))
+}
+
+func TestValidSealCapabilityNode(t *testing.T) {
+	for _, node := range []string{"node-1", "node_2", "n3"} {
+		require.True(t, ValidSealCapabilityNode(node), node)
+	}
+	for _, node := range []string{"", "a.b", "*", ">", "node-*", "node >", "node\t1", "node\n1"} {
+		require.False(t, ValidSealCapabilityNode(node), "%q", node)
+	}
+}

contracts/ec2/v1/account_settings_seal_test.go

@@ -0,0 +1,43 @@
+package ec2v1
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestSealAccountSettingsSubject(t *testing.T) {
+	require.Equal(t, "ec2.v1.SealAccountSettings", SealAccountSettingsSubject)
+}
+
+func TestSealAccountSettingsRequestJSONContract(t *testing.T) {
+	data, err := json.Marshal(SealAccountSettingsRequest{AccountID: "111122223333", CorrelationRef: "req-1"})
+	require.NoError(t, err)
+	require.JSONEq(t, `{"account_id":"111122223333","correlation_ref":"req-1"}`, string(data))
+
+	data, err = json.Marshal(SealAccountSettingsRequest{AccountID: "111122223333"})
+	require.NoError(t, err)
+	require.JSONEq(t, `{"account_id":"111122223333"}`, string(data))
+}
+
+func TestSealAccountSettingsReplyJSONContract(t *testing.T) {
+	reply := SealAccountSettingsReply{
+		AccountID:      "111122223333",
+		CorrelationRef: "req-1",
+		Disposition:    AccountSettingsDispositionSealed,
+		Revision:       42,
+	}
+	data, err := json.Marshal(reply)
+	require.NoError(t, err)
+	require.JSONEq(t, `{
+		"account_id": "111122223333",
+		"correlation_ref": "req-1",
+		"disposition": "sealed",
+		"revision": 42
+	}`, string(data))
+
+	data, err = json.Marshal(SealAccountSettingsReply{AccountID: "111122223333", Disposition: AccountSettingsDispositionSealed})
+	require.NoError(t, err)
+	require.JSONEq(t, `{"account_id":"111122223333","disposition":"sealed","revision":0}`, string(data))
+}

docs/COMMANDS.md

@@ -110,6 +110,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 |---------|-------|-------------|
 | `spx admin kv digest` | `--store-dir` (default `<spinifex-dir>/nats/jetstream`), `--work-dir` (holds the temporary copy, default the system temp dir), `--stream` (repeatable filter), `--seqs` (also print a hash per sequence), `--json` (print one JSON document, the input `spx admin kv compare` reads) | Read-only per-replica consistency check. Copies this node's `$G` streams tree (without consumer state) to a temporary directory, opens the copy in a private in-process NATS server with no listener, and prints one line per stream with first/last sequence, message count and a SHA-256 over every stored message (sequence, subject, timestamp, headers, data). Run it on every node with `sudo` and diff the output with the `#` lines removed: replicas of one stream must match. A direct get over NATS is not used because any replica may answer it. Writes landing during the copy can change a replica's tail, so gate API writes or re-run before calling a tail difference divergence. Prefer `--json --seqs` on every node followed by `spx admin kv compare`, which tells a diverged replica from one that is merely behind. |
 | `spx admin kv compare <digest.json>...` | — (`-` reads any number of digests from stdin) | Reads the `spx admin kv digest --json --seqs` output of every node and checks that each stream's replicas hold the same content. A replica that is merely behind is not divergence; these are: the same sequence holding different content on two replicas, a message missing from one replica that nothing on it explains (no later write to the same subject, no expiry under `max_age` or a message TTL, no stream limit dropping its head), a stream held by fewer or more nodes than its replica count, replicas disagreeing on that count, or no sequence range common to every replica. A stream with a `max_age` whose replicas were copied at different times and share no sequence range is reported as `skipped` (volatile) rather than diverged, since a short-lived bucket such as a leader lease turns over between copies; any sequence two of its replicas share must still match, and each message a replica lacks must be superseded, expired, or written after that replica was copied. Sequences past the slowest replica's last are not compared. Exits 0 when every stream is consistent, 1 on divergence, 2 on bad input. Digests are taken from live stores seconds apart, so re-run before trusting a failure that a write landing between two copies could explain. `install-node.sh` gates cluster formation on it. |
+| `spx admin dns nameservers` | `--check` (compare against the parent zone's delegation), `--timeout` (per DNS query, default 5s) | Prints the `nsN` nameservers this cluster publishes for its base domain, with the node and address behind each. At most four are published however many nodes run northstar; `[northstar] nameserver_count` (1–4) and `nameserver_nodes` (ordered pin) in `spinifex.toml` set the count and order, and every northstar node keeps serving every zone either way. `--check` asks a server of the parent zone (a `.net` server for `spx3.net`) for the delegation without recursion and reports each name as `ok`, `not delegated`, `no glue`, `glue mismatch` or `not published here`, exiting non-zero unless all are `ok`. |
 | `spx admin cluster shutdown` | `--force` (shutdown even if nodes don't respond), `--timeout` (max wait per phase, default 120s), `--dry-run` (print phase plan without executing) | Performs coordinated, phased shutdown of entire cluster. Phases execute in order: GATE (stop API/UI) → DRAIN (stop VMs) → STORAGE (stop viperblock) → PERSIST (stop predastore) → INFRA (stop NATS/daemon). Each phase waits for all nodes to ACK before proceeding. Uses JetStream state tracking. |
 | `spx admin cluster drain-dhcp` | `--timeout` (reply-collection window, default 30s) | Asks each vpcd to DHCPRELEASE every external-pool DHCP lease it currently holds, returning them to the upstream DHCP server. Run on teardown before stopping services — an env reset otherwise strands held leases upstream until their TTL expires, eventually exhausting the upstream scope. Best-effort: warns and exits 0 if the cluster is already down. |
 | `spx admin node drain --local` | `--local` (drain the local node only, required), `--timeout` (max wait per phase, default 120s), `--unless-restarting` (drain on a real host shutdown or a plain target stop, and skip only when a restart of the stack is already queued; unset, it always drains) | Runs the GATE and DRAIN phases against the local node only: powers down its guests via QMP and unmounts their volumes (flushing the viperblock WAL) while every service is still up. STORAGE/PERSIST/INFRA are left to systemd's ordered teardown. Wired as the `ExecStop` of `spinifex-shutdown.service`, so `systemctl stop spinifex.target` and host reboot/poweroff drain guests before any storage service stops. |

docs/PACKAGE_BOUNDARY_MIGRATION.md

@@ -446,7 +446,7 @@ These residuals were recorded by individual slices and must be resolved once the
 - `providers/ebs/viperblock` → `runtime/compute/nbd` and `runtime/host/process` (dependency, moved unchanged from `services/viperblockd`): the adapter formats NBD endpoints and manages nbdkit processes through runtime packages. ADR-0001 does not list providers → runtime as forbidden and the layering test does not enforce it (same category as `providers/ebs/conformance` → `runtime/compute/nbd`), but a provider should not depend on node runtime. Clears when NBD endpoint naming has a provider-facing home (a candidate for `contracts/ebs/v1`) and process supervision is injected by the role. The adapter also imports `providers/objectstore`, a provider-to-provider edge that is allowed.
 - `gateway/ec2.go` (registration): the EC2 dispatch map stays a `GatewayConfig` method in `gateway` and imports every `domains/ec2/awsapi/*` package; `gateway/ec2_idempotency.go` also imports `domains/ec2/awsapi/idem` (`KVBucket`, `TTL`, `TokenAndParams`) for the dispatch-time idempotency wrapper. ADR-0001 makes the domain own its action registration; moving the map waits on the generic ingress registration seam and must not be done as a package move.
 - `gateway/requestaudit.go`: the outermost request-audit middleware records AWS fields and is updated from four places in the gateway's authentication flow through the gateway's own context key. Moving it would export about seven names used only by that flow, so it stays in `gateway` until the generic ingress registration seam defines how ingress middleware receives authentication results.
-- `accountteardown` (dependency): its EC2 reapers call `domains/ec2/awsapi/*` action adapters directly, as they did under `gateway/ec2`. The ADR's `accountteardown` current-to-target map row replaces this with domain-owned cleanup capabilities before the package moves. It also holds a read-only consumer of the EC2 platform-inventory contract (`contracts/ec2/v1`, `ec2.v1.PlatformInventory`), served by `domains/ec2/accountinventory`; only the explicit dry run (`Engine.DryRun`) calls it, to log a shadow comparison, and the key-pair, placement-group and launch-template reapers still use the `awsapi` adapters. A domain-owned cleanup capability for the proven-safe subset of those three kinds (key pairs and empty placement groups no instance, spot request or live launch-template version names, and launch-template residue; never live launch templates) exists as `domains/ec2/accountcleanup` (a Go interface with injected owners and reference readers, no NATS subject); nothing calls it yet.
+- `accountteardown` (dependency): its EC2 reapers call `domains/ec2/awsapi/*` action adapters directly, as they did under `gateway/ec2`. The ADR's `accountteardown` current-to-target map row replaces this with domain-owned cleanup capabilities before the package moves. It also holds a read-only consumer of the EC2 platform-inventory contract (`contracts/ec2/v1`, `ec2.v1.PlatformInventory`), served by `domains/ec2/accountinventory`; only the explicit dry run (`Engine.DryRun`) calls it, to log a shadow comparison, and the key-pair, placement-group and launch-template reapers still use the `awsapi` adapters. A domain-owned cleanup capability for the proven-safe subset of those three kinds (key pairs and empty placement groups no instance, spot request or live launch-template version names, and launch-template residue; never live launch templates) exists as `domains/ec2/accountcleanup` (a Go interface with injected owners and reference readers, no NATS subject); nothing calls it yet. It also holds the StageAccount settings-seal reaper, a consumer of `contracts/ec2/v1`'s `ec2.v1.SealAccountSettings` and per-node `ec2.v1.AccountSettingsSealCapability.<node>` subjects, served by `domains/ec2/account` through the daemon; the reaper reads no KV and imports no domain package, reaching both subjects only through NATS and the configured node set `NewClusterEngine` already receives.
 - `domains/admission/quota` (dependency, moved from `handlers/quota` unchanged): `live.go` imports `gateway/elbv2` and `handlers/rds` for its live ELBv2 and RDS counters; `records.go` imports `runtime/compute/vm` for the instance-record vCPU sweep; `live.go`, `vcpu.go` and `storage.go` call `domains/ec2/awsapi/{vpc,eip,volume}` describe adapters to count VPCs, subnets, addresses and volumes, as they did under `gateway/ec2`. The ADR's `handlers/quota` current-to-target map row replaces these with reader capabilities once their final locations exist.
 - `domains/ec2/awsapi/instance` (dependency): imports `handlers/iam` (`IAMService` in `DescribeInstances.go`, `RunInstances.go` and `IamInstanceProfileAssociation.go`; `InstanceProfile` in the latter two), `runtime/compute/cache` (`NodeState`, `NodeUnknown`, `NodeLive`) and `runtime/compute/vm` (`VM`), both in `status_synthesis.go`. The IAM edge needs an IAM capability; the runtime edges are domain-to-runtime and need the instance record and node liveness exposed through an EC2-owned projection.
 - `domains/ec2/awsapi/spotinstance` (dependency): imports `handlers/iam.IAMService` and `domains/admission/quota.Service` in `RequestSpotInstances.go`.

docs/service-interfaces.yaml

@@ -31,6 +31,7 @@ services:
       - "ec2.*"
       - "elbv2.*"
       - "acm.*"
+      - "route53.*"
       - "iam.*"
       - "rds.*"
       - "ecs.*"
@@ -39,6 +40,7 @@ services:
       - "ec2.*"
       - "elbv2.*"
       - "acm.*"
+      - "route53.*"
       - "iam.*"
       - "rds.*"
       - "ecs.*"
@@ -76,6 +78,7 @@ services:
       - spinifex/domains/ec2/vpc
       - spinifex/handlers/elbv2
       - spinifex/domains/acm
+      - spinifex/domains/route53
       - spinifex/handlers/iam
       - spinifex/handlers/ecs
       - spinifex/handlers/rds
@@ -154,6 +157,10 @@ services:
       - "acm.DescribeCertificate"
       - "acm.ListCertificates"
       - "acm.DeleteCertificate"
+      # Route 53 hosted zones (domains/route53; subscribed in daemon.go).
+      - "route53.*"
+      # Operator-only zone activation and delegation, sent by spx admin dns.
+      - "route53.admin.*"
       - "elbv2.AddListenerCertificates"
       - "elbv2.RemoveListenerCertificates"
       - "elbv2.DescribeListenerCertificates"
@@ -466,6 +473,12 @@ suites:
     # Exercises the full daemon stack under network failure injection;
     # any service touching the wire path is in scope.
 
+  e2e-route53:
+    path: tests/e2e/route53
+    covers: [awsgw, spinifex_daemon, predastore]
+    # Hosted zones end to end: delegation check, publish to the northstar zone
+    # store in predastore, and every node answering before a change is INSYNC.
+
   # Suites below are bash-only today. Entries activate as the Go ports
   # land (Bead 2 fixtures unblock new-shape ports for multinode etc.).
   #

docs/testing/e2e-coverage.md

@@ -6,8 +6,8 @@ Suites live in `tests/e2e/<name>/` and build to `tests/e2e/_bin/<name>.test`. Ev
 
 | Set | Suites |
 | --- | --- |
-| `E2E_SUITES_SINGLE` | `single iam cert eks ecs storagegrowth partialblock rds quota storagefault` |
-| `E2E_SUITES_MULTI` | `multinode lb cert quota kvquorum lbrecovery instancerecovery storagefault` |
+| `E2E_SUITES_SINGLE` | `single iam cert eks ecs storagegrowth partialblock rds quota route53 storagefault` |
+| `E2E_SUITES_MULTI` | `multinode lb cert quota route53 kvquorum lbrecovery instancerecovery storagefault` |
 | `E2E_SUITES_NIGHTLY_SINGLE` | `single cert iam` |
 | `E2E_SUITES_NIGHTLY_MULTI` | `multinode cert lb` |
 
@@ -120,6 +120,7 @@ The create/describe/connect path against a live cluster: a DB VM that actually b
 ### Other
 
 - **`quota`** — every enforced dimension refuses with the same `quotaExceeded` code; a launch is charged for every instance it asks for; one instance wider than the cap is refused.
+- **`route53`** — hosted zones as northstar serves them, on every node. `TestNorthstarServesZonesBeyondFirstListPage` writes 1,100 zone objects (more than one `ListObjectsV2` page), requires the last one to resolve on every node and the platform base zone to survive the sync, then requires the deleted zones to stop resolving.
 - **`gpu`** — device visible in the guest, driver access, launch.
 - **`bedrock`** — `ListFoundationModels`/`GetFoundationModel` against a running cluster. Wire-format parity is unit-tested, not here.
 - **`ddil`** — Denied, Disrupted, Intermittent, Limited network scenarios.

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/config v1.33.9
 	github.com/aws/aws-sdk-go-v2/credentials v1.20.9
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3
 	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/charmbracelet/bubbles v1.0.0
 	github.com/charmbracelet/bubbletea v1.3.10
@@ -24,7 +25,7 @@ require (
 	github.com/miekg/dns v1.1.73
 	github.com/minio/highwayhash v1.0.4
 	github.com/mulgadc/bluebottle v1.22.0
-	github.com/mulgadc/northstar v1.22.0
+	github.com/mulgadc/northstar v1.22.1-0.20261023224547-8c5555ef1d1d
 	github.com/mulgadc/predastore v1.22.0
 	github.com/mulgadc/viperblock v1.22.0
 	github.com/nats-io/nats-server/v2 v2.15.1
@@ -67,10 +68,9 @@ require (
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 // indirect
 	github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect

go.sum

@@ -40,14 +40,14 @@ github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 h1:e6YW5uME30TkrElR+4/xcY7NB6CCj2dcv7vtmGXhfhQ=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21/go.mod h1:obpHAMP693+f3/A8PkLhIqfK1C+NWWn8zXzqD3OH520=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 h1:/TYsZXdA8UTa+WCtCYSAJIr1vwl0+eho6TUgJGwFFO8=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5/go.mod h1:qPqp1Uwd/BqdhPufv6oem9j5J7HNsgc2V22dUiDPn+s=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 h1:CwvFlKw0uUScfgOMflXSRZa+ayXgOf6wk7lhyMemQYU=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7/go.mod h1:4SbD8BB+p+QEnG0aX6DoWwrtk9SufyYXRsUv0nve8LE=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 h1:LHHdobjo5le1sjJdolitV7uVvYg3wkaeM1Ju2fDgeZQ=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6/go.mod h1:CBGB0evAiaMD3Lk4wPO7LQCcCZLuhzL5Qf/wfEUHQ5w=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 h1:pPiWfgeNxqluKEph7hvU88kuGKBPOWzO+Dk9t2zqqNs=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4/go.mod h1:YlwGoIUDG/3kBQbdNOVs/xKZ9J01G8e/6D1mRBj9uTk=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 h1:VMAdYqr4Jn/8ATs9BHC5riwrs0d6m1Z2ohFriSwZwm0=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0/go.mod h1:9APRWGLFITKD+xzWSIyT9V7QV4bNlEuIieWlzXgGFlI=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 h1:mYViDr2azcgxOpRrObZOl631bvavqILoE/L+Kxgbwys=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6/go.mod h1:OXVdAo9jNhggB+hJRGOyy7WolGTKB6Rg7IIulDeSzPM=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3 h1:NyeuyXkZL476uoAowNActm4XJO1QPKf7aassgXHpiqA=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3/go.mod h1:GfoBeKMDJecgRUsGf/0RGdx0NJAh81ObTd922h8pQys=
 github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A=
 github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M=
 github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk=
@@ -295,8 +295,12 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc
 github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
 github.com/mulgadc/bluebottle v1.22.0 h1:pGZ5/Ly3CYem62pPu2zKBRpvMpAG0LHMIMw9Yt/lkIg=
 github.com/mulgadc/bluebottle v1.22.0/go.mod h1:DwT91bJyUsrpg1gCJmcCOTG+tgw5J124LloHYvGAQu0=
-github.com/mulgadc/northstar v1.22.0 h1:fTndlk47hK76fcNr2MqsyeKhBNVEDt8+BC6Z30ohokA=
-github.com/mulgadc/northstar v1.22.0/go.mod h1:yLH8waqfHZx177Ae6dGQGTDt/02rPE9alnUKyxDNhVs=
+github.com/mulgadc/northstar v1.22.1-0.20261023220040-11ff69dd571a h1:juPJ0jubDYy6Sdk14BGnYB4QF7N0PhAmV522FdBA2yI=
+github.com/mulgadc/northstar v1.22.1-0.20261023220040-11ff69dd571a/go.mod h1:Halhinpgo5rzUhYemGN8uYTWu1pJsL5e/7l1eEKc7nM=
+github.com/mulgadc/northstar v1.22.1-0.20261023224401-31cf7e48e4db h1:OrXatiBIUEv6InRzh0zHIPnfp+CrIxf2+rX4T3fo7cI=
+github.com/mulgadc/northstar v1.22.1-0.20261023224401-31cf7e48e4db/go.mod h1:Halhinpgo5rzUhYemGN8uYTWu1pJsL5e/7l1eEKc7nM=
+github.com/mulgadc/northstar v1.22.1-0.20261023224547-8c5555ef1d1d h1:FTch26kEKKv+c4X3L0cQKMXwBJ4D1SCFlV4Mfgh7nEU=
+github.com/mulgadc/northstar v1.22.1-0.20261023224547-8c5555ef1d1d/go.mod h1:Halhinpgo5rzUhYemGN8uYTWu1pJsL5e/7l1eEKc7nM=
 github.com/mulgadc/predastore v1.22.0 h1:mUQJMTxMiRDVvemMP+34BP/+r1U0iCptc9B++u4NJ8o=
 github.com/mulgadc/predastore v1.22.0/go.mod h1:/cMIoKtnQG/DWyeX/0matEQrG2/abY60tNHWcBKS9Kc=
 github.com/mulgadc/viperblock v1.22.0 h1:8vS2AtpIDkIhGNOazQPicKXV2y/8PAy8BNrVWIOCP6I=

spinifex/accountteardown/cluster.go

@@ -70,6 +70,11 @@ func NewClusterEngine(
 	reapers = append(reapers, S3Reapers(buckets)...)
 	reapers = append(reapers, ACMReapers(nc)...)
 	reapers = append(reapers, IAMReapers(svc)...)
+
+	// Before the account counters and reservations: the seal must still be
+	// able to find the account IAM holds as TERMINATING, which the account
+	// record reaper does not remove.
+	reapers = append(reapers, EC2AccountSettingsReapers(nc, nodeIDs)...)
 	reapers = append(reapers, AccountReapers(svc, names, usage)...)
 
 	engine := NewEngine(NewIAMAccounts(svc), reapers...)

spinifex/accountteardown/cluster_test.go

@@ -41,6 +41,27 @@ func TestNewClusterEngineCoversEveryStage(t *testing.T) {
 	}
 }
 
+// The EC2 settings seal reaper must be registered in StageAccount on every
+// surface that builds a cluster engine, or teardown would finalise accounts
+// without ever sealing their settings.
+func TestNewClusterEngineRegistersTheEC2SettingsSealReaper(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+
+	masterKey, err := handlers_iam.GenerateMasterKey()
+	require.NoError(t, err)
+	svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey)
+	require.NoError(t, err)
+
+	engine, err := NewClusterEngine(t.Context(), nc, 1, []string{"node-1"}, svc, newFakeBuckets())
+	require.NoError(t, err)
+
+	var kinds []string
+	for _, reaper := range engine.reapersFor(StageAccount) {
+		kinds = append(kinds, reaper.Kind())
+	}
+	assert.Contains(t, kinds, "ec2-account-settings")
+}
+
 // A teardown must refuse the accounts no credential may delete before it reads
 // anything, and refuse a name confirmation that does not match the record.
 func TestClusterEnginePrechecks(t *testing.T) {

spinifex/accountteardown/ec2_account_settings_seal.go

@@ -0,0 +1,125 @@
+package accountteardown
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"sync"
+	"time"
+
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/nats-io/nats.go"
+)
+
+// ec2AccountSettingsSealTimeout bounds one seal request; an earlier caller
+// deadline wins.
+const ec2AccountSettingsSealTimeout = 30 * time.Second
+
+// ec2AccountSettingsSealer is EC2's seal of one account's settings record.
+type ec2AccountSettingsSealer interface {
+	SealAccountSettings(ctx context.Context, accountID string) (revision uint64, err error)
+}
+
+// ec2AccountSettingsSealClient reaches the EC2 settings-seal contract over
+// NATS. Only a reply for this account, disposition sealed and a non-zero
+// revision is a seal; anything else, including no responder, is an error.
+type ec2AccountSettingsSealClient struct {
+	nc      *nats.Conn
+	timeout time.Duration
+}
+
+var _ ec2AccountSettingsSealer = (*ec2AccountSettingsSealClient)(nil)
+
+func newEC2AccountSettingsSealClient(nc *nats.Conn) *ec2AccountSettingsSealClient {
+	return &ec2AccountSettingsSealClient{nc: nc, timeout: ec2AccountSettingsSealTimeout}
+}
+
+func (c *ec2AccountSettingsSealClient) SealAccountSettings(ctx context.Context, accountID string) (uint64, error) {
+	timeout := c.timeout
+	if deadline, ok := ctx.Deadline(); ok {
+		timeout = min(timeout, time.Until(deadline))
+	}
+	if ctx.Err() != nil || timeout <= 0 {
+		return 0, fmt.Errorf("seal account settings: %w", context.DeadlineExceeded)
+	}
+
+	reply, err := natsmsg.NATSRequest[ec2v1.SealAccountSettingsReply](ctx, c.nc, ec2v1.SealAccountSettingsSubject,
+		&ec2v1.SealAccountSettingsRequest{AccountID: accountID}, timeout, accountID)
+	if err != nil {
+		return 0, fmt.Errorf("seal account settings: %w", err)
+	}
+	if reply.AccountID != accountID {
+		return 0, fmt.Errorf("seal account settings: reply named account %q, not %q", reply.AccountID, accountID)
+	}
+	if reply.Disposition != ec2v1.AccountSettingsDispositionSealed {
+		return 0, fmt.Errorf("seal account settings: unexpected disposition %q", reply.Disposition)
+	}
+	if reply.Revision == 0 {
+		return 0, errors.New("seal account settings: reply carried no revision")
+	}
+	return reply.Revision, nil
+}
+
+// accountSettingsSealReaper seals EC2's settings record in StageAccount. List
+// does no I/O and stays pending until this instance has sealed, so DryRun never
+// seals; state is per engine, so every fresh teardown run seals again.
+type accountSettingsSealReaper struct {
+	sealer     ec2AccountSettingsSealer
+	capability sealCapabilityChecker
+
+	mu     sync.Mutex
+	sealed map[string]uint64 // account ID -> leader revision, this run only
+}
+
+func (r *accountSettingsSealReaper) Kind() string { return "ec2-account-settings" }
+func (r *accountSettingsSealReaper) Stage() Stage { return StageAccount }
+
+func (r *accountSettingsSealReaper) alreadySealed(accountID string) bool {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	_, ok := r.sealed[accountID]
+	return ok
+}
+
+func (r *accountSettingsSealReaper) List(_ context.Context, accountID string) ([]Resource, error) {
+	if r.alreadySealed(accountID) {
+		return nil, nil
+	}
+	return []Resource{{Kind: r.Kind(), ID: accountID, Detail: "seal"}}, nil
+}
+
+// Delete probes every configured daemon immediately before each seal attempt,
+// never reusing an earlier probe. Incomplete capability sends no seal and is
+// retryable; force is ignored, since nothing may force past the probe.
+func (r *accountSettingsSealReaper) Delete(ctx context.Context, accountID string, _ Resource, _ bool) error {
+	report := r.capability.CheckSealCapability(ctx, accountID)
+	if !report.Complete {
+		return errSealCapabilityIncomplete(report)
+	}
+
+	revision, err := r.sealer.SealAccountSettings(ctx, accountID)
+	if err != nil {
+		return err
+	}
+
+	r.mu.Lock()
+	if r.sealed == nil {
+		r.sealed = map[string]uint64{}
+	}
+	r.sealed[accountID] = revision
+	r.mu.Unlock()
+	slog.InfoContext(ctx, "Account teardown sealed EC2 account settings", "accountID", accountID, "revision", revision)
+	return nil
+}
+
+// EC2AccountSettingsReapers returns the StageAccount reaper that seals EC2's
+// settings record. nodeIDs is the configured node set the capability probe
+// treats as "every relevant writer"; an empty set is unknown, never complete.
+func EC2AccountSettingsReapers(nc *nats.Conn, nodeIDs []string) []Reaper {
+	return []Reaper{&accountSettingsSealReaper{
+		sealer:     newEC2AccountSettingsSealClient(nc),
+		capability: newSealCapabilityClient(nc, nodeIDs),
+	}}
+}

spinifex/accountteardown/ec2_account_settings_seal_capability.go

@@ -0,0 +1,190 @@
+package accountteardown
+
+import (
+	"context"
+	"encoding/json"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/nats-io/nats.go"
+)
+
+// sealCapabilityWait bounds the whole probe; shortened by the caller's
+// deadline. Every configured node is probed in parallel against this one
+// wait, so N nodes cost one wait, not N.
+const sealCapabilityWait = 3 * time.Second
+
+// sealCapabilityGrace is how long a node's probe keeps listening after its
+// one expected responder has answered, long enough for a genuine second
+// subscriber on the same node to be caught as a duplicate.
+const sealCapabilityGrace = 150 * time.Millisecond
+
+// Seal capability reasons, one per failing node. Each names a category, never
+// wire error text.
+const (
+	sealCapabilityReasonNoReply       = "no-reply"
+	sealCapabilityReasonDuplicate     = "duplicate"
+	sealCapabilityReasonWrongIdentity = "wrong-identity"
+	sealCapabilityReasonErrorReply    = "error-reply"
+	sealCapabilityReasonMalformed     = "malformed"
+	sealCapabilityReasonOldVersion    = "old-version"
+	sealCapabilityReasonTransport     = "transport"
+	sealCapabilityReasonInvalidNodeID = "invalid-node-id"
+	sealCapabilityReasonSetUnknown    = "configured-node-set-unknown"
+)
+
+// sealCapabilityChecker reports whether every configured daemon supports the
+// seal.
+type sealCapabilityChecker interface {
+	CheckSealCapability(ctx context.Context, correlationRef string) sealCapabilityReport
+}
+
+// sealCapabilityReport is one probe's result, one entry per configured node.
+type sealCapabilityReport struct {
+	Complete bool
+	Nodes    []nodeSealCapability // one per configured node, sorted by node
+}
+
+// nodeSealCapability is one configured node's compatibility. Reason is ""
+// when the node is compatible.
+type nodeSealCapability struct {
+	Node    string
+	Version int    // 0 unless a valid reply arrived
+	Reason  string // "" when compatible
+}
+
+// sealCapabilityClient probes every configured node's seal-capability subject
+// in parallel. "Configured" is the same node set NewClusterEngine already
+// receives; there is no separate liveness or membership source.
+type sealCapabilityClient struct {
+	nc      *nats.Conn
+	nodeIDs []string
+	wait    time.Duration // 3 s, shortened by the caller's deadline
+	grace   time.Duration // 150 ms ResponderGrace
+}
+
+var _ sealCapabilityChecker = (*sealCapabilityClient)(nil)
+
+func newSealCapabilityClient(nc *nats.Conn, nodeIDs []string) *sealCapabilityClient {
+	return &sealCapabilityClient{nc: nc, nodeIDs: nodeIDs, wait: sealCapabilityWait, grace: sealCapabilityGrace}
+}
+
+// CheckSealCapability probes every configured node in parallel, bounded by
+// one shared deadline, run immediately before each seal attempt. A cached or
+// earlier result is never reused.
+func (c *sealCapabilityClient) CheckSealCapability(ctx context.Context, correlationRef string) sealCapabilityReport {
+	if len(c.nodeIDs) == 0 {
+		return sealCapabilityReport{Complete: false}
+	}
+
+	wait := c.wait
+	if deadline, ok := ctx.Deadline(); ok {
+		wait = min(wait, time.Until(deadline))
+	}
+
+	nodes := append([]string(nil), c.nodeIDs...)
+	sort.Strings(nodes)
+
+	results := make([]nodeSealCapability, len(nodes))
+	if ctx.Err() != nil || wait <= 0 {
+		for i, node := range nodes {
+			results[i] = nodeSealCapability{Node: node, Reason: sealCapabilityReasonNoReply}
+		}
+		return sealCapabilityReport{Complete: false, Nodes: results}
+	}
+
+	var wg sync.WaitGroup
+	for i, node := range nodes {
+		wg.Add(1)
+		go func(i int, node string) {
+			defer wg.Done()
+			results[i] = c.checkNode(ctx, node, correlationRef, wait)
+		}(i, node)
+	}
+	wg.Wait()
+
+	complete := true
+	for _, r := range results {
+		if r.Reason != "" {
+			complete = false
+			break
+		}
+	}
+	return sealCapabilityReport{Complete: complete, Nodes: results}
+}
+
+// checkNode probes one node and classifies the result. A node is compatible
+// only when exactly one identified, decodable, matching, version >= 1 frame
+// arrived, with no duplicate, conflict or cap hit.
+func (c *sealCapabilityClient) checkNode(ctx context.Context, node, correlationRef string, wait time.Duration) nodeSealCapability {
+	if !ec2v1.ValidSealCapabilityNode(node) {
+		return nodeSealCapability{Node: node, Reason: sealCapabilityReasonInvalidNodeID}
+	}
+
+	data, err := json.Marshal(ec2v1.AccountSettingsSealCapabilityRequest{Node: node, CorrelationRef: correlationRef})
+	if err != nil {
+		return nodeSealCapability{Node: node, Reason: sealCapabilityReasonMalformed}
+	}
+
+	frames, sum, err := natsmsg.Gather(ctx, c.nc, ec2v1.AccountSettingsSealCapabilitySubject(node), data, natsmsg.GatherOpts{
+		Timeout:            wait,
+		ExpectedResponders: 1,
+		Mode:               natsmsg.CollectUntilDeadline,
+		ResponderGrace:     c.grace,
+	})
+	if err != nil {
+		return nodeSealCapability{Node: node, Reason: sealCapabilityReasonTransport}
+	}
+
+	switch {
+	case sum.CapHit, len(sum.ConflictNodes) > 0, sum.DuplicateFrames > 0:
+		return nodeSealCapability{Node: node, Reason: sealCapabilityReasonDuplicate}
+	case sum.Unidentified > 0:
+		return nodeSealCapability{Node: node, Reason: sealCapabilityReasonWrongIdentity}
+	case len(sum.ErrorResponders) > 0:
+		return nodeSealCapability{Node: node, Reason: sealCapabilityReasonErrorReply}
+	case sum.Received == 0, len(frames) == 0:
+		return nodeSealCapability{Node: node, Reason: sealCapabilityReasonNoReply}
+	}
+
+	frame := frames[0]
+	if frame.NodeID != node {
+		return nodeSealCapability{Node: node, Reason: sealCapabilityReasonWrongIdentity}
+	}
+
+	var reply ec2v1.AccountSettingsSealCapabilityReply
+	if json.Unmarshal(frame.Data, &reply) != nil {
+		return nodeSealCapability{Node: node, Reason: sealCapabilityReasonMalformed}
+	}
+	if reply.Node != node {
+		return nodeSealCapability{Node: node, Reason: sealCapabilityReasonWrongIdentity}
+	}
+	if reply.CapabilityVersion < 1 {
+		return nodeSealCapability{Node: node, Version: reply.CapabilityVersion, Reason: sealCapabilityReasonOldVersion}
+	}
+	return nodeSealCapability{Node: node, Version: reply.CapabilityVersion}
+}
+
+// errSealCapabilityIncomplete builds the retryable refusal from report. It
+// names node IDs and reason categories only, never wire error text.
+func errSealCapabilityIncomplete(report sealCapabilityReport) error {
+	detail := sealCapabilityReasonSetUnknown
+	var parts []string
+	for _, n := range report.Nodes {
+		if n.Reason != "" {
+			parts = append(parts, n.Node+"="+n.Reason)
+		}
+	}
+	if len(parts) > 0 {
+		detail = strings.Join(parts, ", ")
+	}
+	return awserrors.WithRetryAfter(
+		awserrors.Errorf(awserrors.ErrorServiceUnavailable, "EC2 account-settings seal capability incomplete: %s", detail),
+		5*time.Second,
+	)
+}

spinifex/accountteardown/ec2_account_settings_seal_capability_test.go

@@ -0,0 +1,311 @@
+package accountteardown
+
+//test:in-package — the probe client, its report types and reason constants
+// are unexported, and what they guarantee (incomplete never seals, complete
+// does) is only observable from inside the package.
+
+import (
+	"encoding/json"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// testCapabilityClient builds a probe client with test-sized timeouts so the
+// incomplete cases below do not each pay the production 3 s wait.
+func testCapabilityClient(nc *nats.Conn, nodeIDs []string) *sealCapabilityClient {
+	c := newSealCapabilityClient(nc, nodeIDs)
+	c.wait = 200 * time.Millisecond
+	c.grace = 30 * time.Millisecond
+	return c
+}
+
+// serveCapability subscribes reply on subject with no queue group, which is
+// how the real responder is addressed: per node, so two subscribers on the
+// same node's subject both answer.
+func serveCapability(t *testing.T, nc *nats.Conn, subject string, reply nats.MsgHandler) *nats.Subscription {
+	t.Helper()
+	sub, err := nc.Subscribe(subject, reply)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	require.NoError(t, nc.Flush())
+	return sub
+}
+
+// capabilityReply answers with a well-formed reply naming replyNode (which
+// may deliberately differ from the subject's node) and headerNode as the
+// X-Node-ID ("" omits the header).
+func capabilityReply(replyNode, headerNode string, version int) nats.MsgHandler {
+	return func(msg *nats.Msg) {
+		data, _ := json.Marshal(ec2v1.AccountSettingsSealCapabilityReply{Node: replyNode, CapabilityVersion: version})
+		reply := nats.NewMsg(msg.Reply)
+		reply.Data = data
+		if headerNode != "" {
+			reply.Header.Set(natsmsg.NodeIDHeader, headerNode)
+		}
+		_ = msg.RespondMsg(reply)
+	}
+}
+
+func nodeResult(t *testing.T, report sealCapabilityReport, node string) nodeSealCapability {
+	t.Helper()
+	for _, n := range report.Nodes {
+		if n.Node == node {
+			return n
+		}
+	}
+	t.Fatalf("no result for node %q in %+v", node, report)
+	return nodeSealCapability{}
+}
+
+// Each sub-case names exactly one way a node fails to prove capability
+// version 1, and that failure alone makes the whole probe incomplete.
+func TestSealCapabilityIncomplete(t *testing.T) {
+	t.Run("missing", func(t *testing.T) {
+		_, nc, _ := testutil.StartTestJetStream(t)
+		report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
+		assert.False(t, report.Complete)
+		assert.Equal(t, sealCapabilityReasonNoReply, nodeResult(t, report, "node-1").Reason)
+	})
+
+	t.Run("duplicate", func(t *testing.T) {
+		_, nc, _ := testutil.StartTestJetStream(t)
+		serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-1", "node-1", 1))
+		serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-1", "node-1", 1))
+		report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
+		assert.False(t, report.Complete)
+		assert.Equal(t, sealCapabilityReasonDuplicate, nodeResult(t, report, "node-1").Reason)
+	})
+
+	t.Run("unreachable timeout", func(t *testing.T) {
+		_, nc, _ := testutil.StartTestJetStream(t)
+		serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), func(msg *nats.Msg) {
+			time.Sleep(time.Second)
+			_ = msg.Respond(nil)
+		})
+		client := testCapabilityClient(nc, []string{"node-1"})
+		report := client.CheckSealCapability(testCtx(t), "req")
+		assert.False(t, report.Complete)
+		assert.Equal(t, sealCapabilityReasonNoReply, nodeResult(t, report, "node-1").Reason)
+	})
+
+	t.Run("transport closed connection", func(t *testing.T) {
+		_, nc, _ := testutil.StartTestJetStream(t)
+		nc.Close()
+		report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
+		assert.False(t, report.Complete)
+		assert.Equal(t, sealCapabilityReasonTransport, nodeResult(t, report, "node-1").Reason)
+	})
+
+	t.Run("malformed body", func(t *testing.T) {
+		_, nc, _ := testutil.StartTestJetStream(t)
+		serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), func(msg *nats.Msg) {
+			reply := nats.NewMsg(msg.Reply)
+			reply.Data = []byte("not json")
+			reply.Header.Set(natsmsg.NodeIDHeader, "node-1")
+			_ = msg.RespondMsg(reply)
+		})
+		report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
+		assert.False(t, report.Complete)
+		assert.Equal(t, sealCapabilityReasonMalformed, nodeResult(t, report, "node-1").Reason)
+	})
+
+	t.Run("error reply", func(t *testing.T) {
+		_, nc, _ := testutil.StartTestJetStream(t)
+		serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), func(msg *nats.Msg) {
+			reply := nats.NewMsg(msg.Reply)
+			reply.Data = awserrors.GenerateErrorPayload(awserrors.ErrorServerInternal)
+			reply.Header.Set(natsmsg.NodeIDHeader, "node-1")
+			_ = msg.RespondMsg(reply)
+		})
+		report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
+		assert.False(t, report.Complete)
+		assert.Equal(t, sealCapabilityReasonErrorReply, nodeResult(t, report, "node-1").Reason)
+	})
+
+	t.Run("wrong identity no header", func(t *testing.T) {
+		_, nc, _ := testutil.StartTestJetStream(t)
+		serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-1", "", 1))
+		report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
+		assert.False(t, report.Complete)
+		assert.Equal(t, sealCapabilityReasonWrongIdentity, nodeResult(t, report, "node-1").Reason)
+	})
+
+	t.Run("wrong identity another node header", func(t *testing.T) {
+		_, nc, _ := testutil.StartTestJetStream(t)
+		serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-1", "node-9", 1))
+		report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
+		assert.False(t, report.Complete)
+		assert.Equal(t, sealCapabilityReasonWrongIdentity, nodeResult(t, report, "node-1").Reason)
+	})
+
+	t.Run("wrong identity body node differs", func(t *testing.T) {
+		_, nc, _ := testutil.StartTestJetStream(t)
+		serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-9", "node-1", 1))
+		report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
+		assert.False(t, report.Complete)
+		assert.Equal(t, sealCapabilityReasonWrongIdentity, nodeResult(t, report, "node-1").Reason)
+	})
+
+	t.Run("lower version", func(t *testing.T) {
+		_, nc, _ := testutil.StartTestJetStream(t)
+		serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-1", "node-1", 0))
+		report := testCapabilityClient(nc, []string{"node-1"}).CheckSealCapability(testCtx(t), "req")
+		assert.False(t, report.Complete)
+		assert.Equal(t, sealCapabilityReasonOldVersion, nodeResult(t, report, "node-1").Reason)
+	})
+
+	t.Run("configured set empty", func(t *testing.T) {
+		_, nc, _ := testutil.StartTestJetStream(t)
+		report := testCapabilityClient(nc, nil).CheckSealCapability(testCtx(t), "req")
+		assert.False(t, report.Complete)
+		assert.Empty(t, report.Nodes)
+	})
+
+	t.Run("invalid node id", func(t *testing.T) {
+		_, nc, _ := testutil.StartTestJetStream(t)
+		report := testCapabilityClient(nc, []string{"bad.node"}).CheckSealCapability(testCtx(t), "req")
+		assert.False(t, report.Complete)
+		assert.Equal(t, sealCapabilityReasonInvalidNodeID, nodeResult(t, report, "bad.node").Reason)
+	})
+}
+
+// Every configured node answering version 1 or later makes the probe
+// complete, and the reaper sends exactly one seal request.
+func TestSealCapabilityCompleteAllowsSeal(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-1", "node-1", 1))
+	serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-2"), capabilityReply("node-2", "node-2", 2))
+
+	client := testCapabilityClient(nc, []string{"node-1", "node-2"})
+	report := client.CheckSealCapability(testCtx(t), "req")
+	require.True(t, report.Complete)
+	assert.Equal(t, 1, nodeResult(t, report, "node-1").Version)
+	assert.Equal(t, 2, nodeResult(t, report, "node-2").Version)
+
+	sealer := &fakeSealer{}
+	reaper := &accountSettingsSealReaper{sealer: sealer, capability: client}
+	require.NoError(t, reaper.Delete(testCtx(t), testAccountID, Resource{Kind: reaper.Kind(), ID: testAccountID}, false))
+	assert.Equal(t, []string{testAccountID}, sealer.calls)
+}
+
+// An incomplete probe keeps the account TERMINATING through the full engine,
+// names the silent node and its reason, sends no seal request, and leaves the
+// settings record untouched (the fake sealer records no call).
+func TestSealCapabilityIncompleteKeepsAccountTerminating(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-1", "node-1", 1))
+	// node-2 has no subscriber.
+
+	sealer := &fakeSealer{}
+	reaper := &accountSettingsSealReaper{sealer: sealer, capability: testCapabilityClient(nc, []string{"node-1", "node-2"})}
+	accounts := newFakeAccounts()
+	engine := quietEngine(accounts, reaper)
+
+	result, err := engine.Teardown(testCtx(t), request())
+	require.ErrorIs(t, err, ErrResourcesStuck)
+	assert.False(t, result.AccountDeleted)
+	assert.Equal(t, AccountStatusTerminating, accounts.account.Status)
+	assert.Empty(t, sealer.calls)
+
+	var reason string
+	for _, stage := range result.Stages {
+		for _, stuck := range stage.Stuck {
+			if stuck.Resource.Kind == "ec2-account-settings" {
+				reason = stuck.Reason
+			}
+		}
+	}
+	assert.Contains(t, reason, "node-2="+sealCapabilityReasonNoReply)
+}
+
+// The silent node starting to answer during the drain makes the very next
+// Delete re-probe and succeed, which proves the result is retryable and the
+// probe is never cached across attempts.
+func TestSealCapabilityRetriesThenSeals(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), capabilityReply("node-1", "node-1", 1))
+
+	var node2Up atomic.Bool
+	sub, err := nc.Subscribe(ec2v1.AccountSettingsSealCapabilitySubject("node-2"), func(msg *nats.Msg) {
+		if !node2Up.Load() {
+			return
+		}
+		capabilityReply("node-2", "node-2", 1)(msg)
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	require.NoError(t, nc.Flush())
+
+	sealer := &fakeSealer{}
+	reaper := &accountSettingsSealReaper{sealer: sealer, capability: testCapabilityClient(nc, []string{"node-1", "node-2"})}
+	accounts := newFakeAccounts()
+	engine := quietEngine(accounts, reaper)
+	engine.Timeouts = Timeouts{StageDrain: time.Second, DrainPoll: 5 * time.Millisecond}
+
+	go func() {
+		time.Sleep(80 * time.Millisecond)
+		node2Up.Store(true)
+	}()
+
+	result, err := engine.Teardown(testCtx(t), request())
+	require.NoError(t, err)
+	assert.True(t, result.AccountDeleted)
+	assert.Equal(t, []string{testAccountID}, sealer.calls)
+}
+
+// Two Delete calls on one reaper probe twice: the probe runs in Delete
+// itself, never cached from a previous attempt.
+func TestSealCapabilityProbedBeforeEverySeal(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	var probes int
+	serveCapability(t, nc, ec2v1.AccountSettingsSealCapabilitySubject("node-1"), func(msg *nats.Msg) {
+		probes++
+		capabilityReply("node-1", "node-1", 1)(msg)
+	})
+
+	sealer := &fakeSealer{}
+	client := testCapabilityClient(nc, []string{"node-1"})
+	reaper := &accountSettingsSealReaper{sealer: sealer, capability: client}
+
+	require.NoError(t, reaper.Delete(testCtx(t), testAccountID, Resource{Kind: reaper.Kind(), ID: testAccountID}, false))
+	// alreadySealed now makes List empty, but Delete itself does not consult
+	// it, so a second direct call still probes and seals again.
+	require.NoError(t, reaper.Delete(testCtx(t), testAccountID, Resource{Kind: reaper.Kind(), ID: testAccountID}, false))
+	assert.Equal(t, 2, probes)
+	assert.Len(t, sealer.calls, 2)
+}
+
+// Five silent nodes still return within one deadline, not five: every node
+// is probed in parallel against the same wait.
+func TestSealCapabilityBoundedByOneDeadline(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	client := testCapabilityClient(nc, []string{"node-1", "node-2", "node-3", "node-4", "node-5"})
+
+	started := time.Now()
+	report := client.CheckSealCapability(testCtx(t), "req")
+	elapsed := time.Since(started)
+
+	assert.False(t, report.Complete)
+	assert.Less(t, elapsed, client.wait+time.Second, "five silent nodes must not cost five deadlines")
+}
+
+// The stuck reason is logged and stored as one line naming each failing node;
+// the retry hint and the ServiceUnavailable code travel with it.
+func TestSealCapabilityIncompleteErrorIsOneLine(t *testing.T) {
+	err := errSealCapabilityIncomplete(sealCapabilityReport{Nodes: []nodeSealCapability{{Node: "node-1", Reason: sealCapabilityReasonDuplicate}}})
+
+	assert.Equal(t, "EC2 account-settings seal capability incomplete: node-1=duplicate: ServiceUnavailable", err.Error())
+	d, ok := awserrors.ResolveRetryAfter(err)
+	assert.True(t, ok)
+	assert.Equal(t, 5*time.Second, d)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorServiceUnavailable))
+}

spinifex/accountteardown/ec2_account_settings_seal_test.go

@@ -0,0 +1,273 @@
+package accountteardown
+
+//test:in-package — the reaper, sealer and capability checker are unexported;
+// no seal without a complete probe, completion only after a sealed reply and
+// idempotent fresh runs are only observable from inside the package.
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	ec2account "github.com/mulgadc/spinifex/spinifex/domains/ec2/account"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// fakeSealer is a recording ec2AccountSettingsSealer. Each call beyond
+// failFor succeeds with a fresh, strictly increasing revision, unless err is
+// set, in which case every call after failFor fails with it.
+type fakeSealer struct {
+	mu       sync.Mutex
+	calls    []string
+	failFor  int
+	err      error
+	revision uint64
+}
+
+func (f *fakeSealer) SealAccountSettings(_ context.Context, accountID string) (uint64, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.calls = append(f.calls, accountID)
+	if len(f.calls) <= f.failFor {
+		return 0, errors.New("account settings owner temporarily unavailable")
+	}
+	if f.err != nil {
+		return 0, f.err
+	}
+	f.revision++
+	return f.revision, nil
+}
+
+// fakeSealCapability is a recording sealCapabilityChecker returning a fixed report.
+type fakeSealCapability struct {
+	calls  int
+	report sealCapabilityReport
+}
+
+func (f *fakeSealCapability) CheckSealCapability(context.Context, string) sealCapabilityReport {
+	f.calls++
+	return f.report
+}
+
+// completeReport builds a sealCapabilityReport where every named node answers
+// compatible at version 1.
+func completeReport(nodeIDs ...string) sealCapabilityReport {
+	var nodes []nodeSealCapability
+	for _, node := range nodeIDs {
+		nodes = append(nodes, nodeSealCapability{Node: node, Version: 1})
+	}
+	return sealCapabilityReport{Complete: true, Nodes: nodes}
+}
+
+// alwaysTerminatingChecker stands in for IAM in the end-to-end test: the
+// account under test is always TERMINATING.
+type alwaysTerminatingChecker struct{}
+
+func (alwaysTerminatingChecker) AccountTerminating(context.Context, string) (bool, error) {
+	return true, nil
+}
+
+// No subscriber on either subject makes the capability probe incomplete, so
+// no seal request is ever sent and the account stays TERMINATING with one
+// stuck ec2-account-settings item.
+func TestSealReaperNoResponderKeepsAccountTerminating(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+
+	capability := newSealCapabilityClient(nc, []string{"node-1"})
+	capability.wait = 50 * time.Millisecond
+	capability.grace = 10 * time.Millisecond
+	sealer := newEC2AccountSettingsSealClient(nc)
+	sealer.timeout = 50 * time.Millisecond
+
+	reaper := &accountSettingsSealReaper{sealer: sealer, capability: capability}
+	accounts := newFakeAccounts()
+	engine := quietEngine(accounts, reaper)
+
+	result, err := engine.Teardown(testCtx(t), request())
+	require.ErrorIs(t, err, ErrResourcesStuck)
+	assert.False(t, result.AccountDeleted)
+	assert.Equal(t, AccountStatusTerminating, accounts.account.Status)
+
+	var stuck []Stuck
+	for _, stage := range result.Stages {
+		stuck = append(stuck, stage.Stuck...)
+	}
+	require.Len(t, stuck, 1)
+	assert.Equal(t, "ec2-account-settings", stuck[0].Resource.Kind)
+}
+
+// Whatever the owner refuses with, the account stays TERMINATING: an owner
+// refusal, a timeout, and each reply shape the client rejects (wrong account,
+// wrong disposition, no revision).
+func TestSealReaperOwnerFailureKeepsAccountTerminating(t *testing.T) {
+	cases := []struct {
+		name string
+		err  error
+	}{
+		{"incorrect-state", errors.New(awserrors.ErrorIncorrectState)},
+		{"internal-error", errors.New(awserrors.ErrorInternalError)},
+		{"timeout", context.DeadlineExceeded},
+		{"wrong-account", errors.New(`seal account settings: reply named account "999999999999", not "000000000042"`)},
+		{"wrong-disposition", errors.New(`seal account settings: unexpected disposition "pending"`)},
+		{"zero-revision", errors.New("seal account settings: reply carried no revision")},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			sealer := &fakeSealer{err: tc.err}
+			reaper := &accountSettingsSealReaper{sealer: sealer, capability: &fakeSealCapability{report: completeReport("node-1")}}
+			accounts := newFakeAccounts()
+			engine := quietEngine(accounts, reaper)
+
+			result, err := engine.Teardown(testCtx(t), request())
+			require.ErrorIs(t, err, ErrResourcesStuck)
+			assert.False(t, result.AccountDeleted)
+			assert.Equal(t, AccountStatusTerminating, accounts.account.Status)
+			assert.NotEmpty(t, sealer.calls)
+		})
+	}
+}
+
+// The reaper retries through the engine's own drain mechanics: it fails
+// twice, then succeeds, and only the success lets the account be deleted.
+// List is non-empty before the success and empty only after it.
+func TestSealReaperCompletesOnlyAfterSealed(t *testing.T) {
+	sealer := &fakeSealer{failFor: 2}
+	reaper := &accountSettingsSealReaper{sealer: sealer, capability: &fakeSealCapability{report: completeReport("node-1")}}
+	accounts := newFakeAccounts()
+	engine := quietEngine(accounts, reaper)
+
+	before, err := reaper.List(testCtx(t), testAccountID)
+	require.NoError(t, err)
+	require.Len(t, before, 1)
+
+	result, err := engine.Teardown(testCtx(t), request())
+	require.NoError(t, err)
+	assert.True(t, result.AccountDeleted)
+	assert.GreaterOrEqual(t, len(sealer.calls), 3)
+
+	after, err := reaper.List(testCtx(t), testAccountID)
+	require.NoError(t, err)
+	assert.Empty(t, after)
+}
+
+// Each teardown builds a fresh engine and a fresh reaper, so a retried or
+// taken-over run seals again: two engines against the same owner both
+// succeed, each with a higher revision than the last.
+func TestSealReaperFreshRunSealsAgain(t *testing.T) {
+	sealer := &fakeSealer{}
+	capability := &fakeSealCapability{report: completeReport("node-1")}
+
+	firstEngine := quietEngine(newFakeAccounts(), &accountSettingsSealReaper{sealer: sealer, capability: capability})
+	firstResult, err := firstEngine.Teardown(testCtx(t), request())
+	require.NoError(t, err)
+	assert.True(t, firstResult.AccountDeleted)
+
+	secondEngine := quietEngine(newFakeAccounts(), &accountSettingsSealReaper{sealer: sealer, capability: capability})
+	secondResult, err := secondEngine.Teardown(testCtx(t), request())
+	require.NoError(t, err)
+	assert.True(t, secondResult.AccountDeleted)
+
+	require.Len(t, sealer.calls, 2)
+	assert.Equal(t, uint64(2), sealer.revision)
+}
+
+// Inventory and the explicit dry run, and a Teardown request marked dry run,
+// all list the pending seal and send no seal request: only a real teardown
+// may seal.
+func TestSealReaperInventoryDoesNotSeal(t *testing.T) {
+	sealer := &fakeSealer{}
+	capability := &fakeSealCapability{report: completeReport("node-1")}
+	reaper := &accountSettingsSealReaper{sealer: sealer, capability: capability}
+	accounts := newFakeAccounts()
+	engine := quietEngine(accounts, reaper)
+
+	inventory, err := engine.Inventory(testCtx(t), testAccountID)
+	require.NoError(t, err)
+	assert.Empty(t, sealer.calls)
+	assert.Zero(t, capability.calls)
+	found := false
+	for _, stage := range inventory.Stages {
+		for _, resource := range stage.Deleted {
+			if resource.Kind == "ec2-account-settings" {
+				found = true
+			}
+		}
+	}
+	assert.True(t, found, "inventory must still list the pending seal")
+
+	dryRun := request()
+	dryRun.DryRun = true
+	_, err = engine.Teardown(testCtx(t), dryRun)
+	require.NoError(t, err)
+	assert.Empty(t, sealer.calls)
+	assert.Zero(t, capability.calls)
+}
+
+// A real settings service subscribed on the real seal subject, behind a real
+// engine: teardown seals the record, replacing the characterisation test that
+// showed it used to survive.
+func TestTeardownSealsAccountSettings(t *testing.T) {
+	_, nc, js := testutil.StartTestJetStream(t)
+	ctx := testCtx(t)
+
+	settings, err := ec2account.NewAccountSettingsServiceImplWithNATS(ctx, nil, nc)
+	require.NoError(t, err)
+	settings.SetAccountTerminationChecker(alwaysTerminatingChecker{})
+	_, err = settings.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
+	require.NoError(t, err)
+
+	sealSub, err := nc.QueueSubscribe(ec2v1.SealAccountSettingsSubject, "spinifex-workers", func(msg *nats.Msg) {
+		natsmsg.ServeNATSRequestCtx(msg, func(ctx context.Context, req *ec2v1.SealAccountSettingsRequest) (*ec2v1.SealAccountSettingsReply, error) {
+			seal, err := settings.SealAccount(ctx, req.AccountID)
+			if err != nil {
+				return nil, err
+			}
+			return &ec2v1.SealAccountSettingsReply{
+				AccountID: req.AccountID, Disposition: ec2v1.AccountSettingsDispositionSealed, Revision: seal.Revision,
+			}, nil
+		})
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sealSub.Unsubscribe() })
+
+	capSub, err := nc.Subscribe(ec2v1.AccountSettingsSealCapabilitySubject("node-1"), func(msg *nats.Msg) {
+		reply, err := json.Marshal(ec2v1.AccountSettingsSealCapabilityReply{Node: "node-1", CapabilityVersion: 1})
+		require.NoError(t, err)
+		out := nats.NewMsg("")
+		out.Data = reply
+		out.Header.Set(natsmsg.NodeIDHeader, "node-1")
+		require.NoError(t, msg.RespondMsg(out))
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = capSub.Unsubscribe() })
+	require.NoError(t, nc.Flush())
+
+	accounts := newFakeAccounts()
+	engine := quietEngine(accounts, EC2AccountSettingsReapers(nc, []string{"node-1"})...)
+	// The real capability probe holds CollectUntilDeadline's full ResponderGrace
+	// (150 ms) even when the one expected node answers instantly, so the quiet
+	// engine's default 200 ms StageDrain leaves no room for the sweep after it.
+	engine.Timeouts = Timeouts{StageDrain: 2 * time.Second, DrainPoll: 10 * time.Millisecond}
+
+	result, err := engine.Teardown(ctx, request())
+	require.NoError(t, err)
+	assert.True(t, result.AccountDeleted)
+
+	bucket, err := js.KeyValue(ctx, ec2account.KVBucketAccountSettings)
+	require.NoError(t, err)
+	entry, err := bucket.Get(ctx, testAccountID)
+	require.NoError(t, err)
+	var rec ec2account.AccountSettingsRecord
+	require.NoError(t, json.Unmarshal(entry.Value(), &rec))
+	assert.Equal(t, ec2account.AccountSettingsStateSealed, rec.State)
+}

spinifex/accountteardown/reapers_quota_override_current_behaviour_test.go

@@ -0,0 +1,61 @@
+package accountteardown
+
+//test:in-package — drives the real final-stage reapers against the real quota service to pin that teardown leaves the override bucket alone.
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/admission/quota"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// No reaper in this package references the quota override bucket or calls an
+// override delete. Running the real engine and the real StageAccount reapers
+// against an account with a raised vCPU override, with usage wired, shows the
+// consequence: the override outlives the account record it belongs to, while
+// the usage counter the reapers do own is removed.
+func TestCurrentBehaviour_AccountTermination_QuotaOverridesSurviveTeardown(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	ctx := testCtx(t)
+
+	usage, err := kvutil.GetOrCreateBucket(ctx, js, quota.KVBucketAccountUsage, 1)
+	require.NoError(t, err)
+	_, err = usage.Put(ctx, testAccountID, []byte(`{"vcpus":4}`))
+	require.NoError(t, err)
+
+	overridesKV, err := kvutil.GetOrCreateBucket(ctx, js, quota.KVBucketAccountQuota, 1)
+	require.NoError(t, err)
+	svc := quota.New(quota.Limits{Enabled: true, VCPUs: 16}, usage)
+	svc.SetOverrides(overridesKV)
+	unlimited := quota.Unlimited
+	require.NoError(t, svc.PutAccountQuota(ctx, testAccountID, quota.Overrides{VCPUs: &unlimited}, "operator"))
+
+	accounts := newFakeAccounts()
+	iam := &accountIAM{account: &handlers_iam.Account{AccountID: testAccountID, AccountName: testAccountName}}
+	engine := quietEngine(accounts, AccountReapers(iam, nil, usage)...)
+
+	result, err := engine.Teardown(ctx, request())
+	require.NoError(t, err)
+	assert.True(t, result.AccountDeleted)
+	assert.True(t, accounts.deleted)
+	assert.Zero(t, result.StuckCount())
+
+	// The reaper this package does own removed the usage counter, which shows
+	// which store teardown actually clears.
+	_, err = usage.Get(ctx, testAccountID)
+	assert.ErrorIs(t, err, jetstream.ErrKeyNotFound, "teardown's quota-usage reaper must remove the vCPU counter")
+
+	_, err = overridesKV.Get(ctx, testAccountID)
+	require.NoError(t, err, "the override key must still be present after the account is torn down")
+
+	over, effective, err := svc.GetAccountQuota(ctx, testAccountID)
+	require.NoError(t, err)
+	require.NotNil(t, over.VCPUs)
+	assert.Equal(t, quota.Unlimited, *over.VCPUs)
+	assert.Equal(t, quota.Unlimited, effective.VCPUs)
+}

spinifex/accountteardown/settings_kv_boundary_test.go

@@ -0,0 +1,89 @@
+package accountteardown
+
+//test:in-package — pins ADR-0003:S4 and ADR-0001 Context item 2 for the EC2
+// settings seal: the legacy engine must reach the owner only through the
+// versioned contract and NATS, never by reading the settings bucket itself.
+
+import (
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"io/fs"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+// forbiddenSettingsImports are imports that would let this package reach the
+// EC2 settings owner or the configured node set by a path other than the
+// versioned contract and NewClusterEngine's parameters.
+var forbiddenSettingsImports = []string{
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/account",
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config",
+}
+
+// forbiddenSettingsIdentifiers would let this package name the settings
+// bucket or its key directly instead of reaching it only through
+// contracts/ec2/v1 and NATS.
+var forbiddenSettingsIdentifiers = []string{"spinifex-ec2-account-settings", "KVBucketAccountSettings"}
+
+// Teardown reaches EC2 settings only over contracts/ec2/v1: no import of the
+// settings owner or bootstrap/config (nodes arrive via NewClusterEngine), and
+// no literal naming the settings bucket or its constant.
+func TestAccountTeardownHasNoSettingsKVAccess(t *testing.T) {
+	_, file, _, ok := runtime.Caller(0)
+	if !ok {
+		t.Fatal("cannot locate test source")
+	}
+	root := filepath.Dir(file)
+
+	fset := token.NewFileSet()
+	scanned := 0
+	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+		if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
+			return err
+		}
+		f, perr := parser.ParseFile(fset, path, nil, parser.ParseComments)
+		if perr != nil {
+			return perr
+		}
+		scanned++
+
+		for _, imp := range f.Imports {
+			p, _ := strconv.Unquote(imp.Path.Value)
+			for _, bad := range forbiddenSettingsImports {
+				if p == bad {
+					t.Errorf("%s: imports %s; the legacy engine must reach the EC2 settings owner only through contracts/ec2/v1 and the configured node set NewClusterEngine already receives",
+						fset.Position(imp.Pos()), p)
+				}
+			}
+		}
+
+		ast.Inspect(f, func(n ast.Node) bool {
+			lit, ok := n.(*ast.BasicLit)
+			if !ok || lit.Kind != token.STRING {
+				return true
+			}
+			value, uerr := strconv.Unquote(lit.Value)
+			if uerr != nil {
+				return true
+			}
+			for _, bad := range forbiddenSettingsIdentifiers {
+				if value == bad {
+					t.Errorf("%s: names %q; the settings bucket and its key are the owner's own, never this package's",
+						fset.Position(lit.Pos()), value)
+				}
+			}
+			return true
+		})
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if scanned == 0 {
+		t.Fatalf("scanned no Go files under %s; the invariant would pass vacuously", root)
+	}
+}

spinifex/bootstrap/config/config.go

@@ -18,13 +18,29 @@ import (
 // ClusterConfig is the root of spinifex.toml as returned by LoadConfig: the
 // cluster-wide settings plus a [nodes.<name>] Config for every node.
 type ClusterConfig struct {
-	Epoch     uint64            `mapstructure:"epoch"`     // bump when leader commits changes
-	Node      string            `mapstructure:"node"`      // my node name
-	Version   string            `mapstructure:"version"`   // spinifex version
-	Network   NetworkConfig     `mapstructure:"network"`   // cluster-wide external network settings
-	Bootstrap BootstrapConfig   `mapstructure:"bootstrap"` // default VPC IDs for OVN reconciliation
-	AWS       AWSConfig         `mapstructure:"aws"`       // cluster-wide AWS-parity settings (region, endpoint suffix)
-	Nodes     map[string]Config `mapstructure:"nodes"`     // full config for every node
+	Epoch     uint64                 `mapstructure:"epoch"`     // bump when leader commits changes
+	Node      string                 `mapstructure:"node"`      // my node name
+	Version   string                 `mapstructure:"version"`   // spinifex version
+	Network   NetworkConfig          `mapstructure:"network"`   // cluster-wide external network settings
+	Bootstrap BootstrapConfig        `mapstructure:"bootstrap"` // default VPC IDs for OVN reconciliation
+	AWS       AWSConfig              `mapstructure:"aws"`       // cluster-wide AWS-parity settings (region, endpoint suffix)
+	Northstar NorthstarClusterConfig `mapstructure:"northstar"` // cluster-wide authoritative DNS settings
+	Nodes     map[string]Config      `mapstructure:"nodes"`     // full config for every node
+}
+
+// MaxNameservers caps the nsN names a cluster publishes for its zones. Every
+// northstar node still serves every zone; the cap bounds only the delegation a
+// registrar has to carry.
+const MaxNameservers = 4
+
+// NorthstarClusterConfig is the cluster-wide [northstar] block.
+type NorthstarClusterConfig struct {
+	// NameserverCount is how many nsN names to publish, 1 to MaxNameservers.
+	// Zero means one per northstar node, up to MaxNameservers.
+	NameserverCount int `mapstructure:"nameserver_count"`
+	// NameserverNodes pins ns1, ns2, ... to these nodes, in order. Unpinned
+	// slots are filled from the remaining northstar nodes in name order.
+	NameserverNodes []string `mapstructure:"nameserver_nodes"`
 }
 
 // Defaults for cluster-wide AWS-parity settings.
@@ -654,6 +670,32 @@ func LoadConfig(configPath string) (*ClusterConfig, error) {
 	return &config, nil
 }
 
+// validateNorthstarCluster checks the [northstar] nameserver settings against
+// the cap and the configured nodes.
+func validateNorthstarCluster(cc *ClusterConfig) error {
+	ns := cc.Northstar
+	if ns.NameserverCount < 0 || ns.NameserverCount > MaxNameservers {
+		return fmt.Errorf("config: [northstar] nameserver_count=%d out of range; use 1-%d, or remove the key for one per northstar node up to %d", ns.NameserverCount, MaxNameservers, MaxNameservers)
+	}
+	if len(ns.NameserverNodes) > MaxNameservers {
+		return fmt.Errorf("config: [northstar] nameserver_nodes lists %d nodes; at most %d are published", len(ns.NameserverNodes), MaxNameservers)
+	}
+	if ns.NameserverCount > 0 && len(ns.NameserverNodes) > ns.NameserverCount {
+		return fmt.Errorf("config: [northstar] nameserver_nodes lists %d nodes but nameserver_count is %d", len(ns.NameserverNodes), ns.NameserverCount)
+	}
+	seen := make(map[string]bool, len(ns.NameserverNodes))
+	for _, name := range ns.NameserverNodes {
+		if _, ok := cc.Nodes[name]; !ok {
+			return fmt.Errorf("config: [northstar] nameserver_nodes names %q, which is not a configured node", name)
+		}
+		if seen[name] {
+			return fmt.Errorf("config: [northstar] nameserver_nodes names %q twice", name)
+		}
+		seen[name] = true
+	}
+	return nil
+}
+
 // validateClusterConfig rejects legacy DHCP config keys and validates external pool ranges.
 func validateClusterConfig(cc *ClusterConfig) error {
 	if viper.IsSet("network.external_dhcp") {
@@ -684,6 +726,10 @@ func validateClusterConfig(cc *ClusterConfig) error {
 		}
 	}
 
+	if err := validateNorthstarCluster(cc); err != nil {
+		return err
+	}
+
 	if len(cc.Network.NATExemptCIDRs) > 0 && cc.Network.ExternalMode != "nat" {
 		return fmt.Errorf("config: [network] nat_exempt_cidrs requires external_mode = \"nat\"")
 	}

spinifex/bootstrap/config/config_test.go

@@ -1426,3 +1426,43 @@ exoscale_instance_id = "0f1e2d3c-0000-4000-8000-000000000001"
 		})
 	}
 }
+
+func TestLoadConfig_NorthstarNameservers(t *testing.T) {
+	load := func(t *testing.T, block string) (*ClusterConfig, error) {
+		t.Helper()
+		resetViper(t)
+		path := filepath.Join(t.TempDir(), "spinifex.toml")
+		body := "node = \"n1\"\n\n" + block + "\n[nodes.n1]\nregion = \"us-east-1\"\n\n[nodes.n2]\nregion = \"us-east-1\"\n"
+		require.NoError(t, os.WriteFile(path, []byte(body), 0600))
+		return LoadConfig(path)
+	}
+
+	t.Run("absent block means automatic", func(t *testing.T) {
+		cfg, err := load(t, "")
+		require.NoError(t, err)
+		assert.Zero(t, cfg.Northstar.NameserverCount)
+		assert.Empty(t, cfg.Northstar.NameserverNodes)
+	})
+
+	t.Run("count and pin load", func(t *testing.T) {
+		cfg, err := load(t, "[northstar]\nnameserver_count = 2\nnameserver_nodes = [\"n2\", \"n1\"]\n")
+		require.NoError(t, err)
+		assert.Equal(t, 2, cfg.Northstar.NameserverCount)
+		assert.Equal(t, []string{"n2", "n1"}, cfg.Northstar.NameserverNodes)
+	})
+
+	for name, tc := range map[string]struct{ block, want string }{
+		"count above the cap":       {"[northstar]\nnameserver_count = 5\n", "nameserver_count=5"},
+		"negative count":            {"[northstar]\nnameserver_count = -1\n", "nameserver_count=-1"},
+		"pin longer than count":     {"[northstar]\nnameserver_count = 1\nnameserver_nodes = [\"n1\", \"n2\"]\n", "nameserver_count is 1"},
+		"pin names an unknown node": {"[northstar]\nnameserver_nodes = [\"n9\"]\n", `"n9"`},
+		"pin repeats a node":        {"[northstar]\nnameserver_nodes = [\"n1\", \"n1\"]\n", "twice"},
+		"pin above the cap":         {"[northstar]\nnameserver_nodes = [\"n1\", \"n2\", \"n3\", \"n4\", \"n5\"]\n", "at most 4"},
+	} {
+		t.Run(name, func(t *testing.T) {
+			_, err := load(t, tc.block)
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), tc.want)
+		})
+	}
+}

spinifex/daemon/daemon.go

@@ -71,6 +71,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/domains/ochre"
 	ochrevector "github.com/mulgadc/spinifex/spinifex/domains/ochre/vector"
 	rdsengine "github.com/mulgadc/spinifex/spinifex/domains/rds/engine"
+	route53domain "github.com/mulgadc/spinifex/spinifex/domains/route53"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
@@ -175,6 +176,8 @@ type Daemon struct {
 	bedrockReaper         *ochre.Reaper
 	acmService            *acmdomain.ACMServiceImpl
 	acmRenewalWorker      *acmdomain.Worker
+	route53Service        *route53domain.Service
+	route53Verifier       *route53domain.Verifier
 	ochreVectorService    ochrevector.VectorService
 	ochreAppliance        *ochrevector.Appliance
 	ochreBackupService    *ochrevector.BackupService
@@ -1026,6 +1029,7 @@ func (d *Daemon) subscribeAll() error {
 		{"ec2.GetSerialConsoleAccessStatus", handleNATSRequest(d.node, d.accountService.GetSerialConsoleAccessStatus), "spinifex-workers"},
 		{"ec2.EnableSerialConsoleAccess", handleNATSRequest(d.node, d.accountService.EnableSerialConsoleAccess), "spinifex-workers"},
 		{"ec2.DisableSerialConsoleAccess", handleNATSRequest(d.node, d.accountService.DisableSerialConsoleAccess), "spinifex-workers"},
+		{ec2v1.SealAccountSettingsSubject, handleNATSRequest(d.node, d.sealAccountSettings), "spinifex-workers"},
 		{clusterv1.NodeHealthSubject(d.node), d.handleHealthCheck, ""},
 		{clusterv1.NodesDiscoverSubject, d.handleNodeDiscover, ""},
 		{clusterv1.NodeStatusSubject, d.handleNodeStatus, ""},
@@ -1231,6 +1235,31 @@ func (d *Daemon) subscribeAll() error {
 		)
 	}
 
+	// Route 53 is global: any daemon serves any account's zones from shared KV.
+	if d.route53Service != nil {
+		subs = append(subs,
+			natsSub{"route53.CreateHostedZone", handleNATSRequest(d.node, d.route53Service.CreateHostedZone), "spinifex-workers"},
+			natsSub{"route53.GetHostedZone", handleNATSRequest(d.node, d.route53Service.GetHostedZone), "spinifex-workers"},
+			natsSub{"route53.GetHostedZoneCount", handleNATSRequest(d.node, d.route53Service.GetHostedZoneCount), "spinifex-workers"},
+			natsSub{"route53.ListHostedZones", handleNATSRequest(d.node, d.route53Service.ListHostedZones), "spinifex-workers"},
+			natsSub{"route53.ListHostedZonesByName", handleNATSRequest(d.node, d.route53Service.ListHostedZonesByName), "spinifex-workers"},
+			natsSub{"route53.UpdateHostedZoneComment", handleNATSRequest(d.node, d.route53Service.UpdateHostedZoneComment), "spinifex-workers"},
+			natsSub{"route53.DeleteHostedZone", handleNATSRequest(d.node, d.route53Service.DeleteHostedZone), "spinifex-workers"},
+			natsSub{"route53.ChangeResourceRecordSets", handleNATSRequest(d.node, d.route53Service.ChangeResourceRecordSets), "spinifex-workers"},
+			natsSub{"route53.ListResourceRecordSets", handleNATSRequest(d.node, d.route53Service.ListResourceRecordSets), "spinifex-workers"},
+			natsSub{"route53.GetChange", handleNATSRequest(d.node, d.route53Service.GetChange), "spinifex-workers"},
+			natsSub{"route53.ChangeTagsForResource", handleNATSRequest(d.node, d.route53Service.ChangeTagsForResource), "spinifex-workers"},
+			natsSub{"route53.ListTagsForResource", handleNATSRequest(d.node, d.route53Service.ListTagsForResource), "spinifex-workers"},
+			natsSub{"route53.ListTagsForResources", handleNATSRequest(d.node, d.route53Service.ListTagsForResources), "spinifex-workers"},
+			natsSub{"route53.GetHostedZoneActivation", handleNATSRequest(d.node, d.route53Service.GetHostedZoneActivation), "spinifex-workers"},
+			natsSub{"route53.VerifyHostedZoneDelegation", handleNATSRequest(d.node, d.route53Service.VerifyHostedZoneDelegation), "spinifex-workers"},
+			natsSub{"route53.admin.FindZones", handleNATSRequest(d.node, d.route53Service.AdminFindZones), "spinifex-workers"},
+			natsSub{"route53.admin.Verify", handleNATSRequest(d.node, d.route53Service.AdminVerify), "spinifex-workers"},
+			natsSub{"route53.admin.Override", handleNATSRequest(d.node, d.route53Service.AdminOverride), "spinifex-workers"},
+			natsSub{"route53.admin.Delegate", handleNATSRequest(d.node, d.route53Delegate), "spinifex-workers"},
+		)
+	}
+
 	// Ochre vector store tenant surface is deliberately NOT registered here:
 	// its subjects only exist once the platform appliance has finished
 	// launching (startOchreVector, run in the background after this method
@@ -1288,6 +1317,15 @@ func (d *Daemon) subscribeAll() error {
 		natsSub{viperblocklegacyv1.VolumeFencedSubject(d.node), d.handleVolumeFenced, ""},
 	)
 
+	// Per-node, no queue group: a second daemon claiming this node ID also
+	// answers, so the teardown probe sees a duplicate and refuses to seal. An
+	// invalid node ID serves nothing, so this daemon reads as incompatible.
+	if ec2v1.ValidSealCapabilityNode(d.node) {
+		subs = append(subs, natsSub{ec2v1.AccountSettingsSealCapabilitySubject(d.node), handleNATSRequest(d.node, d.accountSettingsSealCapability), ""})
+	} else {
+		slog.Error("Not serving account-settings seal capability: node ID is not a single NATS subject token", "node", d.node)
+	}
+
 	return d.registerNatsSubs(subs)
 }
 
@@ -1992,6 +2030,12 @@ func (d *Daemon) startCluster() error {
 	if err != nil {
 		return fmt.Errorf("failed to initialize account settings service: %w", err)
 	}
+	accountStatusJS, err := jetstream.New(d.natsConn)
+	if err != nil {
+		return fmt.Errorf("jetstream handle for account status reader: %w", err)
+	}
+	var _ ec2account.AccountTerminationChecker = (*handlers_iam.AccountStatusReader)(nil)
+	d.accountService.SetAccountTerminationChecker(handlers_iam.NewAccountStatusReader(accountStatusJS))
 
 	// ELBv2 resolves listener certificate private keys through its own ACM
 	// Store over the same JetStream bucket the ACM service writes, so it needs
@@ -2157,6 +2201,14 @@ func (d *Daemon) startCluster() error {
 		return dns.HostsZone(d.config, domain)
 	}
 
+	route53JS, err := jetstream.New(d.natsConn)
+	if err != nil {
+		return fmt.Errorf("failed to initialize Route 53 service: %w", err)
+	}
+	d.route53Service = d.newRoute53Service(route53JS)
+	d.route53Verifier = d.newRoute53Verifier()
+	d.route53Service.Verifier = d.route53Verifier
+
 	// The tenant private CA is optional, unlike the master key above: a
 	// deployment issuing only public (PROVIDER_API/MANUAL_TXT) certificates has
 	// no reason to have one, so its absence must not block daemon startup.
@@ -2299,6 +2351,10 @@ func (d *Daemon) startCluster() error {
 		go d.dnsReconciler.Run(d.ctx)
 		slog.Info("Started DNS reconcile backstop", "interval_ms", otelsetup.Millis(dns.DefaultReconcileInterval))
 	}
+	if d.route53Service != nil {
+		d.startRoute53Realizer()
+		go d.route53Verifier.Run(d.ctx)
+	}
 
 	// Initialize per-instance-type NATS subscriptions for capacity-aware routing.
 	d.resourceMgr.initSubscriptions(d.natsConn, d.handleEC2RunInstances, d.handleSystemLaunchInstance, d.node)

spinifex/daemon/daemon_handlers_account_settings_seal.go

@@ -0,0 +1,23 @@
+package daemon
+
+import (
+	"context"
+
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+)
+
+// sealAccountSettings serves the irreversible EC2 settings seal. The account
+// comes from the request body, not X-Account-ID: the sealed account is part
+// of the versioned contract, so the header is ignored.
+func (d *Daemon) sealAccountSettings(ctx context.Context, req *ec2v1.SealAccountSettingsRequest, _ string) (*ec2v1.SealAccountSettingsReply, error) {
+	seal, err := d.accountService.SealAccount(ctx, req.AccountID)
+	if err != nil {
+		return nil, err
+	}
+	return &ec2v1.SealAccountSettingsReply{
+		AccountID:      req.AccountID,
+		CorrelationRef: req.CorrelationRef,
+		Disposition:    ec2v1.AccountSettingsDispositionSealed,
+		Revision:       seal.Revision,
+	}, nil
+}

spinifex/daemon/daemon_handlers_account_settings_seal_capability.go

@@ -0,0 +1,23 @@
+package daemon
+
+import (
+	"context"
+
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// accountSettingsSealCapability serves this daemon's read-only seal capability.
+// It reads no KV and takes no account; a request naming another node is a
+// caller mistake, since the subject is already addressed to this node.
+func (d *Daemon) accountSettingsSealCapability(_ context.Context, req *ec2v1.AccountSettingsSealCapabilityRequest, _ string) (*ec2v1.AccountSettingsSealCapabilityReply, error) {
+	if req.Node != d.node {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue,
+			"seal capability request for node %q answered by node %q", req.Node, d.node)
+	}
+	return &ec2v1.AccountSettingsSealCapabilityReply{
+		Node:              d.node,
+		CapabilityVersion: ec2v1.AccountSettingsSealCapabilityVersion,
+		CorrelationRef:    req.CorrelationRef,
+	}, nil
+}

spinifex/daemon/daemon_handlers_account_settings_seal_capability_test.go

@@ -0,0 +1,115 @@
+package daemon
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// newCapabilityTestDaemon builds a daemon serving its real subscription table
+// with node as its configured node ID.
+func newCapabilityTestDaemon(t *testing.T, node string) (*nats.Conn, *nats.Conn) {
+	t.Helper()
+	ns, nc, _ := testutil.StartTestJetStream(t)
+
+	d := newSubscribeTestDaemon(t, nc, "", "")
+	d.node = node
+	require.NoError(t, d.subscribeAll())
+	require.NoError(t, nc.Flush())
+
+	caller, err := nats.Connect(ns.ClientURL())
+	require.NoError(t, err)
+	t.Cleanup(caller.Close)
+	return nc, caller
+}
+
+func capabilityRequest(t *testing.T, nc *nats.Conn, node string) *nats.Msg {
+	t.Helper()
+	data, err := json.Marshal(ec2v1.AccountSettingsSealCapabilityRequest{Node: node, CorrelationRef: "req-1"})
+	require.NoError(t, err)
+	reply, err := nc.Request(ec2v1.AccountSettingsSealCapabilitySubject(node), data, 5*time.Second)
+	require.NoError(t, err)
+	return reply
+}
+
+// The responder answers its own node with version 1, the configured node and
+// no KV access (no settings service is wired into the test daemon at all).
+func TestAccountSettingsSealCapabilityAnswersOwnNode(t *testing.T) {
+	_, caller := newCapabilityTestDaemon(t, "node-1")
+
+	raw := capabilityRequest(t, caller, "node-1")
+	assert.Equal(t, "node-1", raw.Header.Get("X-Node-ID"))
+
+	var reply ec2v1.AccountSettingsSealCapabilityReply
+	require.NoError(t, json.Unmarshal(raw.Data, &reply))
+	assert.Equal(t, "node-1", reply.Node)
+	assert.Equal(t, ec2v1.AccountSettingsSealCapabilityVersion, reply.CapabilityVersion)
+	assert.Equal(t, "req-1", reply.CorrelationRef)
+}
+
+// A request naming a node other than this daemon's own is refused, not
+// answered as if it were this node.
+func TestAccountSettingsSealCapabilityRefusesOtherNode(t *testing.T) {
+	_, caller := newCapabilityTestDaemon(t, "node-1")
+
+	data, err := json.Marshal(ec2v1.AccountSettingsSealCapabilityRequest{Node: "node-2"})
+	require.NoError(t, err)
+	raw, err := caller.Request(ec2v1.AccountSettingsSealCapabilitySubject("node-1"), data, 5*time.Second)
+	require.NoError(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidParameterValue, replyErrCode(t, raw.Data))
+}
+
+// The subject is per node with no queue group, so two daemons configured
+// with the same node ID both answer a request addressed to that node.
+func TestAccountSettingsSealCapabilityHasNoQueueGroup(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+
+	first := newSubscribeTestDaemon(t, nc, "", "")
+	first.node = "node-1"
+	require.NoError(t, first.subscribeAll())
+
+	second := newSubscribeTestDaemon(t, nc, "", "")
+	second.node = "node-1"
+	require.NoError(t, second.subscribeAll())
+	require.NoError(t, nc.Flush())
+
+	sub, err := nc.SubscribeSync(nats.NewInbox())
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+	data, err := json.Marshal(ec2v1.AccountSettingsSealCapabilityRequest{Node: "node-1"})
+	require.NoError(t, err)
+	msg := nats.NewMsg(ec2v1.AccountSettingsSealCapabilitySubject("node-1"))
+	msg.Data = data
+	msg.Reply = sub.Subject
+	require.NoError(t, nc.PublishMsg(msg))
+	require.NoError(t, nc.Flush())
+
+	seen := 0
+	for {
+		_, err := sub.NextMsg(300 * time.Millisecond)
+		if err != nil {
+			break
+		}
+		seen++
+	}
+	assert.Equal(t, 2, seen)
+}
+
+// A wildcard node ID must not become a wildcard subscription: built unchecked,
+// "node-*" would answer probes addressed to node-1 on node-1's behalf.
+func TestAccountSettingsSealCapabilityNotServedForInvalidNodeID(t *testing.T) {
+	_, caller := newCapabilityTestDaemon(t, "node-*")
+
+	data, err := json.Marshal(ec2v1.AccountSettingsSealCapabilityRequest{Node: "node-1"})
+	require.NoError(t, err)
+	_, err = caller.Request(ec2v1.AccountSettingsSealCapabilitySubject("node-1"), data, time.Second)
+	require.ErrorIs(t, err, nats.ErrNoResponders)
+}

spinifex/daemon/daemon_handlers_account_settings_seal_test.go

@@ -0,0 +1,101 @@
+package daemon
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	ec2account "github.com/mulgadc/spinifex/spinifex/domains/ec2/account"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// sealTestChecker answers AccountTerminating from a fixed set, standing in
+// for IAM in these daemon-level tests.
+type sealTestChecker struct {
+	terminating map[string]bool
+}
+
+func (c *sealTestChecker) AccountTerminating(_ context.Context, accountID string) (bool, error) {
+	return c.terminating[accountID], nil
+}
+
+// newSealTestDaemon builds a daemon serving its real subscription table over
+// a real account settings service, wired with checker as its termination
+// precondition.
+func newSealTestDaemon(t *testing.T, checker ec2account.AccountTerminationChecker) (*Daemon, *nats.Conn) {
+	t.Helper()
+	ns, nc, _ := testutil.StartTestJetStream(t)
+
+	acctSvc, err := ec2account.NewAccountSettingsServiceImplWithNATS(t.Context(), nil, nc)
+	require.NoError(t, err)
+	acctSvc.SetAccountTerminationChecker(checker)
+
+	d := newSubscribeTestDaemon(t, nc, "", "")
+	d.accountService = acctSvc
+	require.NoError(t, d.subscribeAll())
+	require.NoError(t, nc.Flush())
+
+	caller, err := nats.Connect(ns.ClientURL())
+	require.NoError(t, err)
+	t.Cleanup(caller.Close)
+	return d, caller
+}
+
+const (
+	sealTestTerminating = "111122223333"
+	sealTestActive      = "444455556666"
+)
+
+func sealRequest(t *testing.T, nc *nats.Conn, body any, header string) []byte {
+	t.Helper()
+	data, err := json.Marshal(body)
+	require.NoError(t, err)
+	reply, err := natsRequestAs(nc, ec2v1.SealAccountSettingsSubject, data, header, 5*time.Second)
+	require.NoError(t, err)
+	return reply.Data
+}
+
+// The header is ignored: the body account is what gets sealed, even when
+// X-Account-ID names a different one.
+func TestSealAccountSettingsSubjectSealsTheBodyAccount(t *testing.T) {
+	_, nc := newSealTestDaemon(t, &sealTestChecker{terminating: map[string]bool{sealTestTerminating: true}})
+
+	raw := sealRequest(t, nc, ec2v1.SealAccountSettingsRequest{AccountID: sealTestTerminating, CorrelationRef: "req-1"}, sealTestActive)
+	require.Empty(t, replyErrCode(t, raw), string(raw))
+
+	var reply ec2v1.SealAccountSettingsReply
+	require.NoError(t, json.Unmarshal(raw, &reply))
+	assert.Equal(t, sealTestTerminating, reply.AccountID)
+	assert.Equal(t, "req-1", reply.CorrelationRef)
+	assert.Equal(t, ec2v1.AccountSettingsDispositionSealed, reply.Disposition)
+	assert.NotZero(t, reply.Revision)
+}
+
+// Only an account IAM holds as TERMINATING may be sealed; ACTIVE is refused
+// before any write.
+func TestSealAccountSettingsSubjectRefusesAnActiveAccount(t *testing.T) {
+	_, nc := newSealTestDaemon(t, &sealTestChecker{terminating: map[string]bool{sealTestTerminating: true}})
+
+	raw := sealRequest(t, nc, ec2v1.SealAccountSettingsRequest{AccountID: sealTestActive}, "")
+	assert.Equal(t, awserrors.ErrorIncorrectState, replyErrCode(t, raw))
+}
+
+// A request without a valid account is refused, not sealed.
+func TestSealAccountSettingsSubjectRefusesAMissingAccount(t *testing.T) {
+	_, nc := newSealTestDaemon(t, &sealTestChecker{terminating: map[string]bool{sealTestTerminating: true}})
+
+	assert.Equal(t, awserrors.ErrorMissingParameter, replyErrCode(t, sealRequest(t, nc, ec2v1.SealAccountSettingsRequest{}, "")))
+}
+
+// A daemon that does not serve the subject leaves the request with no responder.
+func TestSealAccountSettingsSubjectHasNoResponderWithoutTheDaemon(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	_, err := natsRequestAs(nc, ec2v1.SealAccountSettingsSubject, []byte(`{"account_id":"111122223333"}`), "", time.Second)
+	require.ErrorIs(t, err, nats.ErrNoResponders)
+}

spinifex/daemon/route53.go

@@ -0,0 +1,92 @@
+package daemon
+
+import (
+	"context"
+	"log/slog"
+
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/mulgadc/spinifex/spinifex/domains/dns"
+	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
+	route53domain "github.com/mulgadc/spinifex/spinifex/domains/route53"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+var _ route53domain.Publisher = (*dns.ZonePublisher)(nil)
+
+// newRoute53Service builds the Route 53 control plane. The nameservers and the
+// reserved names are read from the cluster's DNS settings on every call, so a
+// node joining or a domain change needs no restart.
+func (d *Daemon) newRoute53Service(js jetstream.JetStream) *route53domain.Service {
+	svc := route53domain.NewService(route53domain.NewStore(js))
+	svc.Nameservers = func() []string {
+		base := dns.ResolveBaseDomain(d.config)
+		if base == "" || d.clusterConfig == nil {
+			return nil
+		}
+		published := dns.PublishedNameservers(d.clusterConfig)
+		hosts := make([]string, 0, len(published))
+		for _, ns := range published {
+			hosts = append(hosts, ns.Host+"."+base)
+		}
+		return hosts
+	}
+	svc.Reserved = func(name string) bool {
+		return dns.ReservedZoneName(name, dns.ResolveBaseDomain(d.config), dns.ResolveInternalDomain(d.config))
+	}
+	return svc
+}
+
+// startRoute53Realizer publishes ACTIVE hosted zones to northstar. Every node
+// runs it, and a per-pass election picks the one that writes. Without a zone
+// store the zones stay unpublished and their changes stay PENDING.
+func (d *Daemon) startRoute53Realizer() {
+	pub, ok := dns.NewZonePublisher(d.config, func() *config.ClusterConfig { return d.clusterConfig }, d.natsConn)
+	if !ok {
+		slog.Info("Route 53 realizer not started: no northstar zone store is configured")
+		return
+	}
+	r := route53domain.NewRealizer(d.route53Service.Store(), pub)
+	r.Elect = func(ctx context.Context) (func(), bool) {
+		return reconcilelock.AcquireLeader(ctx, d.natsConn, route53domain.KVBucketRoute53Realize, d.node)
+	}
+	r.Reserved = d.route53Service.Reserved
+	go r.Run(d.ctx)
+	slog.Info("Started Route 53 realizer")
+}
+
+// newRoute53Verifier checks each zone's delegation: from this cluster's own
+// records when the parent is a zone it serves, and from the parent's public
+// servers otherwise.
+func (d *Daemon) newRoute53Verifier() *route53domain.Verifier {
+	v := route53domain.NewVerifier(d.route53Service.Store())
+	v.Nameservers = d.route53Service.Nameservers
+	if d.dnsWriter.Enabled() {
+		v.Platform = func(_ context.Context, name string) (route53domain.Delegation, bool, error) {
+			servers, zone, ok, err := d.dnsWriter.PlatformDelegation(name)
+			return route53domain.Delegation{Source: "platform zone " + zone, Servers: servers}, ok, err
+		}
+	}
+	lookup := &dns.ParentLookup{}
+	v.Public = func(ctx context.Context, name string) (route53domain.Delegation, error) {
+		servers, parent, err := lookup.Delegation(ctx, name)
+		return route53domain.Delegation{Source: "the " + parent + " name servers", Servers: servers}, err
+	}
+	v.Elect = func(ctx context.Context) (func(), bool) {
+		return reconcilelock.AcquireLeader(ctx, d.natsConn, route53domain.KVBucketRoute53Verify, d.node)
+	}
+	return v
+}
+
+// route53Delegate writes or withdraws platform-zone delegations for the
+// operator's spx admin dns delegate.
+func (d *Daemon) route53Delegate(ctx context.Context, in *route53domain.AdminDelegateInput, _ string) (*route53domain.AdminDelegateOutput, error) {
+	if len(in.Names) == 0 {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "no names to delegate")
+	}
+	res, err := d.dnsWriter.Delegate(ctx, d.natsConn, d.node, in.Names, in.Remove)
+	if err != nil {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "%v", err)
+	}
+	return &route53domain.AdminDelegateOutput{Applied: res.Applied}, nil
+}

spinifex/daemon/route53_test.go

@@ -0,0 +1,50 @@
+//test:in-package — newRoute53Service is an unexported Daemon method and reads
+//the unexported config and clusterConfig fields.
+
+package daemon
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/route53"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRoute53ServiceDelegatesToThePublishedNameservers(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	ns := config.NorthstarConfig{ConfigPath: "/etc/spinifex/northstar.toml", DefaultDomain: "spx3.net"}
+	nodes := map[string]config.Config{
+		"node1": {AdvertiseIP: "10.0.0.1", Northstar: ns},
+		"node2": {AdvertiseIP: "10.0.0.2", Northstar: ns},
+	}
+	local := nodes["node1"]
+	d := &Daemon{config: &local, clusterConfig: &config.ClusterConfig{Node: "node1", Nodes: nodes}}
+	svc := d.newRoute53Service(js)
+
+	out, err := svc.CreateHostedZone(context.Background(), &route53.CreateHostedZoneInput{
+		Name: aws.String("example.com"), CallerReference: aws.String("ref-1"),
+	}, "000000000001")
+	require.NoError(t, err)
+	assert.Equal(t, []string{"ns1.spx3.net", "ns2.spx3.net"}, aws.StringValueSlice(out.DelegationSet.NameServers))
+
+	_, err = svc.CreateHostedZone(context.Background(), &route53.CreateHostedZoneInput{
+		Name: aws.String("lb.elb.spx3.net"), CallerReference: aws.String("ref-2"),
+	}, "000000000001")
+	assert.ErrorContains(t, err, "InvalidDomainName")
+}
+
+func TestRoute53ServiceRefusesZonesWithoutABaseDomain(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	local := config.Config{AdvertiseIP: "10.0.0.1"}
+	d := &Daemon{config: &local, clusterConfig: &config.ClusterConfig{Node: "node1", Nodes: map[string]config.Config{"node1": local}}}
+
+	_, err := d.newRoute53Service(js).CreateHostedZone(context.Background(), &route53.CreateHostedZoneInput{
+		Name: aws.String("example.com"), CallerReference: aws.String("ref-1"),
+	}, "000000000001")
+	assert.ErrorContains(t, err, "no nameservers")
+}

spinifex/domains/admission/quota/live.go

@@ -39,7 +39,11 @@ func (s *Service) EnforceVPCs(ctx context.Context, natsConn *nats.Conn, accountI
 	if err != nil {
 		return err
 	}
-	return exceeds(len(out.Vpcs), want, s.limitsFor(ctx, accountID).VPCs)
+	limits, err := s.limitsFor(ctx, accountID)
+	if err != nil {
+		return err
+	}
+	return exceeds(len(out.Vpcs), want, limits.VPCs)
 }
 
 // EnforceSubnets gates CreateSubnet on the account's live DescribeSubnets count.
@@ -52,7 +56,11 @@ func (s *Service) EnforceSubnets(ctx context.Context, natsConn *nats.Conn, accou
 	if err != nil {
 		return err
 	}
-	return exceeds(len(out.Subnets), want, s.limitsFor(ctx, accountID).Subnets)
+	limits, err := s.limitsFor(ctx, accountID)
+	if err != nil {
+		return err
+	}
+	return exceeds(len(out.Subnets), want, limits.Subnets)
 }
 
 // EnforceEIPs gates AllocateAddress on the account's live DescribeAddresses count.
@@ -64,7 +72,11 @@ func (s *Service) EnforceEIPs(ctx context.Context, natsConn *nats.Conn, accountI
 	if err != nil {
 		return err
 	}
-	return exceeds(len(out.Addresses), want, s.limitsFor(ctx, accountID).EIPs)
+	limits, err := s.limitsFor(ctx, accountID)
+	if err != nil {
+		return err
+	}
+	return exceeds(len(out.Addresses), want, limits.EIPs)
 }
 
 // EnforceLoadBalancers gates CreateLoadBalancer on the account's live count.
@@ -77,7 +89,11 @@ func (s *Service) EnforceLoadBalancers(ctx context.Context, natsConn *nats.Conn,
 	if err != nil {
 		return err
 	}
-	return exceeds(len(out.LoadBalancers), want, s.limitsFor(ctx, accountID).LoadBalancers)
+	limits, err := s.limitsFor(ctx, accountID)
+	if err != nil {
+		return err
+	}
+	return exceeds(len(out.LoadBalancers), want, limits.LoadBalancers)
 }
 
 // EnforceRDSInstances gates CreateDBInstance on the account's live count. A DB
@@ -91,5 +107,9 @@ func (s *Service) EnforceRDSInstances(ctx context.Context, natsConn *nats.Conn,
 	if err != nil {
 		return err
 	}
-	return exceeds(len(out.DBInstances), want, s.limitsFor(ctx, accountID).RDSInstances)
+	limits, err := s.limitsFor(ctx, accountID)
+	if err != nil {
+		return err
+	}
+	return exceeds(len(out.DBInstances), want, limits.RDSInstances)
 }

spinifex/domains/admission/quota/override.go

@@ -8,12 +8,16 @@ import (
 	"log/slog"
 	"time"
 
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
 // Overrides raises or lowers individual dimensions for one account. Every field
 // is a pointer so an explicit 0 (a limit that denies everything) is distinct
-// from an absent field, which inherits the configured value.
+// from an absent field, which inherits the configured value. This is also the
+// admin wire type: an admin request body can carry only these fields, so it can
+// never set or clear a stored record's state.
 type Overrides struct {
 	VCPUs         *int `json:"vcpus,omitempty"`
 	VPCs          *int `json:"vpcs,omitempty"`
@@ -29,11 +33,62 @@ type Overrides struct {
 	UpdatedBy string `json:"updated_by,omitempty"`
 }
 
-// Empty reports whether no dimension is overridden, in which case the record
-// carries nothing but provenance and the account resolves to the config.
-func (o Overrides) Empty() bool {
-	return o.VCPUs == nil && o.VPCs == nil && o.Subnets == nil && o.EIPs == nil &&
-		o.VolumesGiB == nil && o.Volumes == nil && o.RDSInstances == nil && o.LoadBalancers == nil
+// OverrideState is a stored override record's lifecycle state. "" is active;
+// a record written before the state field existed has none and stays active.
+type OverrideState string
+
+// OverrideStateSealed marks a record no writer may change or recreate.
+const OverrideStateSealed OverrideState = "sealed"
+
+// overrideRecord is the stored form: the admin-settable fields inline, plus a
+// state no admin request can read or write.
+type overrideRecord struct {
+	Overrides
+
+	State OverrideState `json:"state,omitempty"`
+}
+
+// errOverridesSealed marks a read or write that found a sealed record, so a
+// caller maps it to Blocked without logging it as a fault.
+var errOverridesSealed = errors.New(awserrors.ErrorBlocked)
+
+// errOverridesContended marks a write whose CAS retries were exhausted under
+// concurrent writers, bounding the retry loop instead of spinning.
+var errOverridesContended = errors.New("quota override: CAS retries exhausted")
+
+// overrideMaxAttempts bounds a write's retries against a revision that keeps
+// moving under concurrent writers, so it fails closed instead of spinning.
+const overrideMaxAttempts = 5
+
+// checkActive rejects a sealed or unknown-state record; "" is active and
+// passes through. A caller must never read or mutate a record that failed
+// this as if it were active.
+func checkActive(rec overrideRecord) error {
+	switch rec.State {
+	case "":
+		return nil
+	case OverrideStateSealed:
+		return errOverridesSealed
+	default:
+		return fmt.Errorf("quota override: unknown state %q", rec.State)
+	}
+}
+
+// mapOverrideError turns a recordFor, PutAccountQuota or clear failure into
+// the AWS error a caller returns: sealed is an expected, unlogged Blocked;
+// everything else is InternalError, logged with the account ID.
+func mapOverrideError(ctx context.Context, accountID string, err error) error {
+	switch {
+	case err == nil:
+		return nil
+	case errors.Is(err, errOverridesSealed):
+		return errors.New(awserrors.ErrorBlocked)
+	case errors.Is(err, errOverridesContended):
+		slog.ErrorContext(ctx, "Quota override CAS retries exhausted under contention", "accountID", accountID)
+	default:
+		slog.ErrorContext(ctx, "Quota override operation failed", "accountID", accountID, "err", err)
+	}
+	return errors.New(awserrors.ErrorInternalError)
 }
 
 // apply overlays the set fields onto base, leaving Enabled and the Bedrock
@@ -60,72 +115,128 @@ func (o Overrides) apply(base Limits) Limits {
 	return out
 }
 
-// limitsFor resolves the effective limits for accountID: its overrides where
-// set, the configured limits everywhere else. A read failure falls back to the
-// configured limits rather than to no limit, so a KV outage cannot lift a cap.
-func (s *Service) limitsFor(ctx context.Context, accountID string) Limits {
-	over, err := s.overridesFor(ctx, accountID)
-	if err != nil {
-		slog.WarnContext(ctx, "quota: override read failed, using configured limits",
-			"account", accountID, "err", err)
-		return s.limits
-	}
-	return over.apply(s.limits)
+// Empty reports whether no dimension is overridden, in which case the record
+// carries nothing but provenance and the account resolves to the config.
+func (o Overrides) Empty() bool {
+	return o.VCPUs == nil && o.VPCs == nil && o.Subnets == nil && o.EIPs == nil &&
+		o.VolumesGiB == nil && o.Volumes == nil && o.RDSInstances == nil && o.LoadBalancers == nil
 }
 
-// overridesFor reads accountID's override record. An absent key is the normal
-// case for nearly every account and yields the zero value, not an error; only a
-// genuine read or decode failure is returned.
-func (s *Service) overridesFor(ctx context.Context, accountID string) (Overrides, error) {
+// recordFor reads and decodes accountID's stored override record. An absent
+// key is the active zero value, not an error; only a genuine read or decode
+// failure is returned.
+func (s *Service) recordFor(ctx context.Context, accountID string) (overrideRecord, error) {
 	if s == nil || s.overrides == nil {
-		return Overrides{}, nil
+		return overrideRecord{}, nil
 	}
 	entry, err := s.overrides.Get(ctx, accountID)
 	if errors.Is(err, jetstream.ErrKeyNotFound) {
-		return Overrides{}, nil
+		return overrideRecord{}, nil
 	}
 	if err != nil {
-		return Overrides{}, fmt.Errorf("read quota override for %s: %w", accountID, err)
+		return overrideRecord{}, fmt.Errorf("read quota override for %s: %w", accountID, err)
 	}
-	var over Overrides
-	if err := json.Unmarshal(entry.Value(), &over); err != nil {
-		return Overrides{}, fmt.Errorf("decode quota override for %s: %w", accountID, err)
+	var rec overrideRecord
+	if err := json.Unmarshal(entry.Value(), &rec); err != nil {
+		return overrideRecord{}, fmt.Errorf("decode quota override for %s: %w", accountID, err)
 	}
-	return over, nil
+	return rec, nil
 }
 
-// GetAccountQuota returns accountID's stored overrides and the limits they
-// resolve to. A read failure is surfaced here rather than masked, because an
-// operator inspecting a quota must not be shown the fallback as if it were real.
+// limitsFor resolves accountID's effective limits: overrides where set, the
+// config elsewhere. A read or decode failure falls back to the config so an
+// outage cannot lift a cap; a sealed or unknown-state record refuses instead.
+func (s *Service) limitsFor(ctx context.Context, accountID string) (Limits, error) {
+	rec, err := s.recordFor(ctx, accountID)
+	if err != nil {
+		slog.WarnContext(ctx, "quota: override read failed, using configured limits",
+			"account", accountID, "err", err)
+		return s.limits, nil
+	}
+	if err := checkActive(rec); err != nil {
+		return Limits{}, mapOverrideError(ctx, accountID, err)
+	}
+	return rec.apply(s.limits), nil
+}
+
+// GetAccountQuota returns accountID's stored overrides and resolved limits.
+// Read failures surface rather than show the fallback as real; a sealed or
+// unknown-state record returns Blocked or InternalError.
 func (s *Service) GetAccountQuota(ctx context.Context, accountID string) (Overrides, Limits, error) {
-	over, err := s.overridesFor(ctx, accountID)
+	rec, err := s.recordFor(ctx, accountID)
 	if err != nil {
 		return Overrides{}, Limits{}, err
 	}
-	return over, over.apply(s.limits), nil
+	if err := checkActive(rec); err != nil {
+		return Overrides{}, Limits{}, mapOverrideError(ctx, accountID, err)
+	}
+	return rec.Overrides, rec.apply(s.limits), nil
 }
 
-// PutAccountQuota stores accountID's overrides, stamping provenance. An empty
-// set deletes the record so the account resolves to the config, which keeps
-// "has this account been touched?" a single existence check.
+// PutAccountQuota stores or clears accountID's overrides, stamping provenance
+// on a store. Both branches are revision-guarded and refuse a sealed record,
+// so a seal landing between a read and a write fails the CAS.
 func (s *Service) PutAccountQuota(ctx context.Context, accountID string, over Overrides, updatedBy string) error {
 	if s == nil || s.overrides == nil {
 		return errors.New("quota: override bucket not configured")
 	}
 	if over.Empty() {
-		if err := s.overrides.Delete(ctx, accountID); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) {
-			return fmt.Errorf("clear quota override for %s: %w", accountID, err)
-		}
-		return nil
+		return s.clearAccountQuota(ctx, accountID)
 	}
+
 	over.UpdatedAt = time.Now().UTC().Format(time.RFC3339)
 	over.UpdatedBy = updatedBy
-	data, err := json.Marshal(over)
-	if err != nil {
-		return fmt.Errorf("encode quota override for %s: %w", accountID, err)
-	}
-	if _, err := s.overrides.Put(ctx, accountID, data); err != nil {
-		return fmt.Errorf("store quota override for %s: %w", accountID, err)
-	}
-	return nil
+
+	_, err := kvutil.Update(ctx, s.overrides, accountID, kvutil.CASConfig{
+		CreateIfAbsent: true,
+		Attempts:       overrideMaxAttempts,
+		Exhausted:      func(string, int) error { return errOverridesContended },
+	}, func(rec *overrideRecord) (bool, error) {
+		if err := checkActive(*rec); err != nil {
+			return false, err
+		}
+		rec.Overrides = over
+		// Every success is a leader-accepted write over an active value: a
+		// seal landing between the read and the write fails this CAS.
+		return true, nil
+	})
+	return mapOverrideError(ctx, accountID, err)
+}
+
+// clearAccountQuota deletes an active override at the revision it read. An
+// observed-absent key writes nothing, so a stale read cannot unseal; a sealed
+// record refuses rather than being erased and later recreated active.
+func (s *Service) clearAccountQuota(ctx context.Context, accountID string) error {
+	for range overrideMaxAttempts {
+		entry, err := s.overrides.Get(ctx, accountID)
+		if errors.Is(err, jetstream.ErrKeyNotFound) {
+			return nil
+		}
+		if err != nil {
+			return mapOverrideError(ctx, accountID, fmt.Errorf("read quota override for clear: %w", err))
+		}
+
+		var rec overrideRecord
+		if err := json.Unmarshal(entry.Value(), &rec); err != nil {
+			return mapOverrideError(ctx, accountID, fmt.Errorf("decode quota override for clear: %w", err))
+		}
+		if err := checkActive(rec); err != nil {
+			return mapOverrideError(ctx, accountID, err)
+		}
+
+		err = s.overrides.Delete(ctx, accountID, jetstream.LastRevision(entry.Revision()))
+		switch {
+		case err == nil:
+			return nil
+		case errors.Is(err, jetstream.ErrKeyNotFound):
+			return nil
+		case isRevisionConflict(err):
+			// A concurrent write landed between the read and the guarded
+			// delete; retry against whatever revision now holds.
+			continue
+		default:
+			return mapOverrideError(ctx, accountID, fmt.Errorf("clear quota override: %w", err))
+		}
+	}
+	return mapOverrideError(ctx, accountID, errOverridesContended)
 }

spinifex/domains/admission/quota/override_kv_test.go

@@ -31,7 +31,8 @@ func newOverrideService(t *testing.T) (*Service, jetstream.KeyValue) {
 func TestLimitsForAbsentKeyInherits(t *testing.T) {
 	svc, _ := newOverrideService(t)
 
-	got := svc.limitsFor(t.Context(), "123456789012")
+	got, err := svc.limitsFor(t.Context(), "123456789012")
+	require.NoError(t, err)
 	require.Equal(t, baseLimits(), got)
 }
 
@@ -44,10 +45,14 @@ func TestLimitsForIsPerAccount(t *testing.T) {
 	require.NoError(t, svc.PutAccountQuota(t.Context(), upgraded,
 		Overrides{VCPUs: ptr(32), RDSInstances: ptr(8)}, "operator"))
 
-	require.Equal(t, 32, svc.limitsFor(t.Context(), upgraded).VCPUs)
-	require.Equal(t, 8, svc.limitsFor(t.Context(), upgraded).RDSInstances)
-	require.Equal(t, baseLimits().VCPUs, svc.limitsFor(t.Context(), sandbox).VCPUs)
-	require.Equal(t, baseLimits().RDSInstances, svc.limitsFor(t.Context(), sandbox).RDSInstances)
+	upgradedLimits, err := svc.limitsFor(t.Context(), upgraded)
+	require.NoError(t, err)
+	sandboxLimits, err := svc.limitsFor(t.Context(), sandbox)
+	require.NoError(t, err)
+	require.Equal(t, 32, upgradedLimits.VCPUs)
+	require.Equal(t, 8, upgradedLimits.RDSInstances)
+	require.Equal(t, baseLimits().VCPUs, sandboxLimits.VCPUs)
+	require.Equal(t, baseLimits().RDSInstances, sandboxLimits.RDSInstances)
 }
 
 // A stored override must round-trip through KV with its provenance, and an
@@ -79,7 +84,9 @@ func TestPutAccountQuotaEmptyClears(t *testing.T) {
 
 	_, err := kv.Get(t.Context(), account)
 	require.ErrorIs(t, err, jetstream.ErrKeyNotFound)
-	require.Equal(t, baseLimits(), svc.limitsFor(t.Context(), account))
+	limits, err := svc.limitsFor(t.Context(), account)
+	require.NoError(t, err)
+	require.Equal(t, baseLimits(), limits)
 }
 
 // Clearing an account that never had a record must succeed rather than fail on
@@ -97,8 +104,10 @@ func TestPutAccountQuotaExplicitZero(t *testing.T) {
 
 	require.NoError(t, svc.PutAccountQuota(t.Context(), account, Overrides{VPCs: ptr(0)}, "operator"))
 
-	require.Equal(t, 0, svc.limitsFor(t.Context(), account).VPCs)
-	require.Error(t, exceeds(0, 1, svc.limitsFor(t.Context(), account).VPCs))
+	limits, err := svc.limitsFor(t.Context(), account)
+	require.NoError(t, err)
+	require.Equal(t, 0, limits.VPCs)
+	require.Error(t, exceeds(0, 1, limits.VPCs))
 }
 
 // A corrupt record is a read failure, not an absent one: it must fall back to
@@ -110,7 +119,9 @@ func TestLimitsForCorruptRecordFailsClosed(t *testing.T) {
 	_, err := kv.Put(t.Context(), account, []byte("{not json"))
 	require.NoError(t, err)
 
-	require.Equal(t, baseLimits(), svc.limitsFor(t.Context(), account),
+	limits, err := svc.limitsFor(t.Context(), account)
+	require.NoError(t, err, "a decode failure must fall back, not error, from limitsFor")
+	require.Equal(t, baseLimits(), limits,
 		"a corrupt override must fall back to the configured limits")
 
 	_, _, err = svc.GetAccountQuota(t.Context(), account)

spinifex/domains/admission/quota/override_test.go

@@ -126,7 +126,11 @@ func TestOverridesEmpty(t *testing.T) {
 // fall through to the configured limits rather than panicking.
 func TestLimitsForWithoutBucket(t *testing.T) {
 	svc := New(baseLimits(), nil)
-	if got := svc.limitsFor(t.Context(), "123456789012"); got != baseLimits() {
+	got, err := svc.limitsFor(t.Context(), "123456789012")
+	if err != nil {
+		t.Fatalf("limitsFor error = %v, want nil", err)
+	}
+	if got != baseLimits() {
 		t.Fatalf("limitsFor = %+v, want the configured limits", got)
 	}
 }

spinifex/domains/admission/quota/override_writes_guarded_test.go

@@ -0,0 +1,75 @@
+package quota
+
+//test:in-package — pins what AccountQuotaOverrideSealCapabilityVersion 1 will
+// assert: nothing in this package writes the override bucket except through
+// PutAccountQuota's CAS loop, clearAccountQuota's guarded delete or
+// SealAccountOverrides' CAS loop.
+
+import (
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"path/filepath"
+	"runtime"
+	"testing"
+)
+
+// overrideWriteFiles are the only files in this package that touch the
+// override bucket. The package also holds the vCPU usage and Bedrock usage
+// buckets, which write through their own stores and are out of scope here;
+// scanning the whole package directory, as the EC2 settings equivalent does,
+// would false-flag those unrelated writes.
+var overrideWriteFiles = []string{"override.go", "seal.go"}
+
+func quotaPackageDir(t *testing.T) string {
+	t.Helper()
+	_, file, _, ok := runtime.Caller(0)
+	if !ok {
+		t.Fatal("cannot locate test source")
+	}
+	return filepath.Dir(file)
+}
+
+// No file owning the override bucket may call Put, PutString, or Delete
+// without a revision option: every write must go through PutAccountQuota's
+// CAS loop, clearAccountQuota's guarded delete, or SealAccountOverrides' CAS
+// loop.
+func TestOverrideWritesAreRevisionGuarded(t *testing.T) {
+	root := quotaPackageDir(t)
+	fset := token.NewFileSet()
+	scanned := 0
+
+	for _, name := range overrideWriteFiles {
+		path := filepath.Join(root, name)
+		f, err := parser.ParseFile(fset, path, nil, 0)
+		if err != nil {
+			t.Fatalf("parse %s: %v", path, err)
+		}
+		scanned++
+
+		ast.Inspect(f, func(n ast.Node) bool {
+			call, ok := n.(*ast.CallExpr)
+			if !ok {
+				return true
+			}
+			sel, ok := call.Fun.(*ast.SelectorExpr)
+			if !ok {
+				return true
+			}
+			switch sel.Sel.Name {
+			case "Put", "PutString":
+				t.Errorf("%s: unconditional %s call bypasses the revision-guarded override write; use PutAccountQuota or SealAccountOverrides instead",
+					fset.Position(call.Pos()), sel.Sel.Name)
+			case "Delete":
+				if len(call.Args) < 3 {
+					t.Errorf("%s: Delete call with no revision option bypasses the guarded clear; pin it to the read revision",
+						fset.Position(call.Pos()))
+				}
+			}
+			return true
+		})
+	}
+	if scanned == 0 {
+		t.Fatalf("scanned no override-write files under %s; the invariant would pass vacuously", root)
+	}
+}

spinifex/domains/admission/quota/quota.go

@@ -90,6 +90,14 @@ type Service struct {
 	// overrides holds the per-account limit overrides. Nil leaves every account
 	// on the configured limits, which is also what an absent key means.
 	overrides jetstream.KeyValue
+
+	// js opens the override bucket for SealAccountOverrides when quotas were
+	// disabled at startup; nil makes such a seal refuse rather than create it.
+	js jetstream.KeyValueManager
+
+	// terminationChecker is the seal precondition; SealAccountOverrides
+	// refuses until it is wired.
+	terminationChecker AccountTerminationChecker
 }
 
 // New constructs a quota Service from the configured limits and the gateway-owned
@@ -125,6 +133,16 @@ func (s *Service) SetOverrides(kv jetstream.KeyValue) {
 	s.overrides = kv
 }
 
+// SetJetStream wires the client SealAccountOverrides uses to get-or-create
+// the override bucket when this gateway disabled quotas. Called once at
+// wiring time, regardless of whether quotas are enabled here.
+func (s *Service) SetJetStream(js jetstream.KeyValueManager) {
+	if s == nil {
+		return
+	}
+	s.js = js
+}
+
 // Exempt returns true for the global/system account and whenever quotas are
 // disabled, so those callers bypass every quota check. A nil Service (quotas
 // never built) is treated as disabled so callers need no nil guard.

spinifex/domains/admission/quota/seal.go

@@ -0,0 +1,163 @@
+package quota
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"log/slog"
+
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// sealMaxAttempts bounds SealAccountOverrides' retry loop against a revision
+// that keeps moving under concurrent writes, so it fails closed instead of
+// spinning.
+const sealMaxAttempts = 5
+
+// sealedRecord is the fixed value a seal writes. No dimension or provenance
+// survives, so no tenant commercial setting outlives the seal.
+var sealedRecord = &overrideRecord{State: OverrideStateSealed}
+
+// OverrideSeal is SealAccountOverrides' result: the revision at which the
+// stream leader accepted the sealed value.
+type OverrideSeal struct {
+	Revision uint64
+}
+
+// AccountTerminationChecker reports whether IAM holds accountID as
+// TERMINATING. A missing record is (false, nil); only a failed read is an
+// error.
+type AccountTerminationChecker interface {
+	AccountTerminating(ctx context.Context, accountID string) (bool, error)
+}
+
+// SetAccountTerminationChecker wires the seal precondition; until it is set,
+// SealAccountOverrides refuses.
+func (s *Service) SetAccountTerminationChecker(c AccountTerminationChecker) {
+	if s == nil {
+		return
+	}
+	s.terminationChecker = c
+}
+
+// checkTerminating runs before any override-KV call, so a refusal never
+// writes or opens the bucket. Not TERMINATING (incl. missing) is
+// IncorrectState; a read failure or unwired checker is InternalError.
+func (s *Service) checkTerminating(ctx context.Context, accountID string) error {
+	if s.terminationChecker == nil {
+		slog.ErrorContext(ctx, "Quota override seal has no termination checker wired", "accountID", accountID)
+		return errors.New(awserrors.ErrorInternalError)
+	}
+	terminating, err := s.terminationChecker.AccountTerminating(ctx, accountID)
+	if err != nil {
+		slog.ErrorContext(ctx, "Failed to read account status for seal precondition", "accountID", accountID, "err", err)
+		return errors.New(awserrors.ErrorInternalError)
+	}
+	if !terminating {
+		return awserrors.Errorf(awserrors.ErrorIncorrectState, "account %s is not TERMINATING", accountID)
+	}
+	return nil
+}
+
+// isRevisionConflict reports a lost CAS race to retry, including the bare
+// wrong-last-sequence APIError from Create's retry against a delete marker.
+// A private copy of domains/ec2/account's helper, which this package may not import.
+func isRevisionConflict(err error) bool {
+	if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
+		return true
+	}
+	if apiErr, ok := errors.AsType[*jetstream.APIError](err); ok {
+		return apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence ||
+			apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequenceConstant
+	}
+	return false
+}
+
+// sealBucket returns this gateway's override bucket, or get-or-creates it
+// when quotas are disabled here. Called only after checkTerminating accepts,
+// so a refused seal never creates the bucket.
+func (s *Service) sealBucket(ctx context.Context) (jetstream.KeyValue, error) {
+	if s.overrides != nil {
+		return s.overrides, nil
+	}
+	if s.js == nil {
+		slog.ErrorContext(ctx, "Quota override seal has no JetStream client wired to open the override bucket")
+		return nil, errors.New(awserrors.ErrorInternalError)
+	}
+	kv, err := kvutil.GetOrCreateBucket(ctx, s.js, KVBucketAccountQuota, 1)
+	if err != nil {
+		slog.ErrorContext(ctx, "Failed to open quota override bucket for seal", "err", err)
+		return nil, errors.New(awserrors.ErrorInternalError)
+	}
+	return kv, nil
+}
+
+// SealAccountOverrides seals accountID's override record so no later writer
+// can recreate it. Success is only a leader-accepted write, evidenced by the
+// revision; a missing key is created sealed, never reported from absence.
+func (s *Service) SealAccountOverrides(ctx context.Context, accountID string) (*OverrideSeal, error) {
+	if accountID == "" {
+		return nil, awserrors.Errorf(awserrors.ErrorMissingParameter, "accountID is required to seal quota overrides")
+	}
+	if !awsidentifiers.IsAccountID(accountID) {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "accountID %q is not a valid AWS account ID", accountID)
+	}
+	if accountID == awsidentifiers.GlobalAccountID {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "quota override seal must not target the global account")
+	}
+
+	if err := s.checkTerminating(ctx, accountID); err != nil {
+		return nil, err
+	}
+
+	overrides, err := s.sealBucket(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	data, err := json.Marshal(sealedRecord)
+	if err != nil {
+		slog.ErrorContext(ctx, "Failed to encode sealed quota override record", "accountID", accountID, "err", err)
+		return nil, errors.New(awserrors.ErrorInternalError)
+	}
+
+	// accountID is never resolved through any fallback here: a seal must act
+	// on exactly accountID and never fall back to the global key.
+	for range sealMaxAttempts {
+		var rev uint64
+		entry, getErr := overrides.Get(ctx, accountID)
+		switch {
+		case getErr == nil:
+			rev = entry.Revision()
+		case errors.Is(getErr, jetstream.ErrKeyNotFound):
+			rev = 0
+		default:
+			slog.ErrorContext(ctx, "Failed to read quota override for seal", "accountID", accountID, "err", getErr)
+			return nil, errors.New(awserrors.ErrorInternalError)
+		}
+
+		var newRev uint64
+		var writeErr error
+		if rev == 0 {
+			newRev, writeErr = overrides.Create(ctx, accountID, data)
+		} else {
+			newRev, writeErr = overrides.Update(ctx, accountID, data, rev)
+		}
+
+		switch {
+		case writeErr == nil:
+			return &OverrideSeal{Revision: newRev}, nil
+		case isRevisionConflict(writeErr):
+			continue
+		default:
+			slog.ErrorContext(ctx, "Failed to write sealed quota override record", "accountID", accountID, "err", writeErr)
+			return nil, errors.New(awserrors.ErrorInternalError)
+		}
+	}
+
+	slog.ErrorContext(ctx, "Quota override seal exhausted attempts against concurrent writes", "accountID", accountID, "attempts", sealMaxAttempts)
+	return nil, errors.New(awserrors.ErrorInternalError)
+}

spinifex/domains/admission/quota/seal_test.go

@@ -0,0 +1,806 @@
+package quota
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sync"
+	"testing"
+
+	"github.com/mulgadc/spinifex/internal/testkit"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// stubTerminationChecker answers AccountTerminating from a fixed set, for
+// tests that exercise the KV paths rather than IAM itself.
+type stubTerminationChecker struct {
+	terminating map[string]bool
+	err         error
+}
+
+func (c *stubTerminationChecker) AccountTerminating(_ context.Context, accountID string) (bool, error) {
+	if c.err != nil {
+		return false, c.err
+	}
+	return c.terminating[accountID], nil
+}
+
+// terminatingChecker answers true for exactly accountID.
+func terminatingChecker(accountID string) *stubTerminationChecker {
+	return &stubTerminationChecker{terminating: map[string]bool{accountID: true}}
+}
+
+// newSealService builds a Service over a real JetStream KV bucket with the
+// override bucket already attached, mirroring a quota-enabled gateway.
+func newSealService(t *testing.T) (*Service, jetstream.JetStream) {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	kv, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{
+		Bucket:  KVBucketAccountQuota,
+		History: 1,
+	})
+	require.NoError(t, err)
+	usage, err := js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{
+		Bucket:  KVBucketAccountUsage,
+		History: 1,
+	})
+	require.NoError(t, err)
+
+	svc := New(baseLimits(), usage)
+	svc.SetOverrides(kv)
+	svc.SetJetStream(js)
+	return svc, js
+}
+
+// newDisabledSealService builds a Service with only the JetStream client
+// wired, mirroring a quota-disabled gateway: the override bucket does not
+// exist until a seal creates it.
+func newDisabledSealService(t *testing.T) (*Service, jetstream.JetStream) {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	svc := New(baseLimits(), nil)
+	svc.SetJetStream(js)
+	return svc, js
+}
+
+// lastOverrideSeq reports the override stream's last sequence, so a test can
+// assert a failed attempt wrote nothing.
+func lastOverrideSeq(t *testing.T, js jetstream.JetStream) uint64 {
+	t.Helper()
+	stream, err := js.Stream(context.Background(), "KV_"+KVBucketAccountQuota)
+	require.NoError(t, err)
+	info, err := stream.Info(context.Background())
+	require.NoError(t, err)
+	return info.State.LastSeq
+}
+
+// mutationCase is one of PutAccountQuota's two mutations, called uniformly so
+// a test can table over them.
+type mutationCase struct {
+	name string
+	call func(svc *Service, ctx context.Context) error
+}
+
+func mutationTable() []mutationCase {
+	return []mutationCase{
+		{"Set", func(svc *Service, ctx context.Context) error {
+			return svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(32)}, "operator")
+		}},
+		{"Clear", func(svc *Service, ctx context.Context) error {
+			return svc.PutAccountQuota(ctx, testAccount, Overrides{}, "operator")
+		}},
+	}
+}
+
+// seedOverrideState prepares testAccount's record for a race test: present
+// (an active record), absent (never written) or tombstone (written then
+// deleted, leaving a delete marker Get hides).
+func seedOverrideState(t *testing.T, svc *Service, state string) {
+	t.Helper()
+	ctx := context.Background()
+	switch state {
+	case "present":
+		require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(8)}, "operator"))
+	case "tombstone":
+		require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(8)}, "operator"))
+		require.NoError(t, svc.overrides.Delete(ctx, testAccount))
+	case "absent":
+	default:
+		t.Fatalf("unknown seed state %q", state)
+	}
+}
+
+// sealRaceKV wraps a real KeyValue so a test can inject a deterministic race
+// between a caller's read and its guarded write (Create, Update or Delete),
+// instead of relying on goroutine timing.
+type sealRaceKV struct {
+	jetstream.KeyValue
+
+	writeCalls  int
+	beforeWrite func(attempt int) error
+}
+
+func (f *sealRaceKV) Create(ctx context.Context, key string, value []byte, opts ...jetstream.KVCreateOpt) (uint64, error) {
+	f.writeCalls++
+	if f.beforeWrite != nil {
+		if err := f.beforeWrite(f.writeCalls); err != nil {
+			return 0, err
+		}
+	}
+	return f.KeyValue.Create(ctx, key, value, opts...)
+}
+
+func (f *sealRaceKV) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) {
+	f.writeCalls++
+	if f.beforeWrite != nil {
+		if err := f.beforeWrite(f.writeCalls); err != nil {
+			return 0, err
+		}
+	}
+	return f.KeyValue.Update(ctx, key, value, revision)
+}
+
+func (f *sealRaceKV) Delete(ctx context.Context, key string, opts ...jetstream.KVDeleteOpt) error {
+	f.writeCalls++
+	if f.beforeWrite != nil {
+		if err := f.beforeWrite(f.writeCalls); err != nil {
+			return err
+		}
+	}
+	return f.KeyValue.Delete(ctx, key, opts...)
+}
+
+// staleGetKV always answers Get with a fixed snapshot taken at construction,
+// so a caller's CAS keeps retrying against a revision that can never win.
+type staleGetKV struct {
+	jetstream.KeyValue
+
+	entry jetstream.KeyValueEntry
+}
+
+func (f *staleGetKV) Get(context.Context, string) (jetstream.KeyValueEntry, error) {
+	return f.entry, nil
+}
+
+// staleNotFoundKV always answers Get as absent, regardless of what the real
+// bucket holds.
+type staleNotFoundKV struct {
+	jetstream.KeyValue
+}
+
+func (f *staleNotFoundKV) Get(context.Context, string) (jetstream.KeyValueEntry, error) {
+	return nil, jetstream.ErrKeyNotFound
+}
+
+// errGetKV always fails Get with a fixed store error.
+type errGetKV struct {
+	jetstream.KeyValue
+
+	err error
+}
+
+func (f *errGetKV) Get(context.Context, string) (jetstream.KeyValueEntry, error) {
+	return nil, f.err
+}
+
+func decodeOverride(t *testing.T, entry jetstream.KeyValueEntry) overrideRecord {
+	t.Helper()
+	var rec overrideRecord
+	require.NoError(t, json.Unmarshal(entry.Value(), &rec))
+	return rec
+}
+
+// A seal landing between a mutation's read and its guarded write is not
+// outlived: the mutation's retry sees the seal and is rejected, and nothing
+// it wrote survives. Clear on an absent or tombstoned key never reaches a
+// write at all (see TestClearOnMissingKeyWritesNothing), so only the four
+// cells with a real write are exercised here.
+func TestSealRaceMutation_SealLandsBetweenReadAndWrite(t *testing.T) {
+	cases := []struct {
+		name  string
+		state string
+		call  func(svc *Service, ctx context.Context) error
+	}{
+		{"Set_present", "present", mutationTable()[0].call},
+		{"Set_absent", "absent", mutationTable()[0].call},
+		{"Set_tombstone", "tombstone", mutationTable()[0].call},
+		{"Clear_present", "present", mutationTable()[1].call},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			svc, _ := newSealService(t)
+			ctx := context.Background()
+			seedOverrideState(t, svc, tc.state)
+
+			realKV := svc.overrides
+			sealer := New(baseLimits(), nil)
+			sealer.SetOverrides(realKV)
+			sealer.SetAccountTerminationChecker(terminatingChecker(testAccount))
+
+			var sealed *OverrideSeal
+			race := &sealRaceKV{KeyValue: realKV}
+			race.beforeWrite = func(attempt int) error {
+				if attempt != 1 {
+					return nil
+				}
+				var sealErr error
+				sealed, sealErr = sealer.SealAccountOverrides(ctx, testAccount)
+				return sealErr
+			}
+			svc.overrides = race
+
+			err := tc.call(svc, ctx)
+			require.Error(t, err)
+			assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked), "got %v", err)
+			require.NotNil(t, sealed)
+
+			entry, err := realKV.Get(ctx, testAccount)
+			require.NoError(t, err)
+			assert.Equal(t, OverrideStateSealed, decodeOverride(t, entry).State)
+			assert.Equal(t, sealed.Revision, entry.Revision(), "no write landed after the seal")
+		})
+	}
+}
+
+// A mutation landing inside a seal's read-write window does not survive it:
+// the seal retries against the mutation's write and wins, leaving the sealed
+// value with no overridden dimension.
+func TestSealRaceMutation_MutationLandsBetweenSealReadAndWrite(t *testing.T) {
+	for _, m := range mutationTable() {
+		for _, state := range []string{"present", "absent", "tombstone"} {
+			t.Run(m.name+"_"+state, func(t *testing.T) {
+				svc, _ := newSealService(t)
+				ctx := context.Background()
+				seedOverrideState(t, svc, state)
+
+				realKV := svc.overrides
+				race := &sealRaceKV{KeyValue: realKV}
+				race.beforeWrite = func(attempt int) error {
+					if attempt != 1 {
+						return nil
+					}
+					_ = m.call(svc, ctx)
+					return nil
+				}
+
+				sealer := New(baseLimits(), nil)
+				sealer.SetOverrides(race)
+				sealer.SetAccountTerminationChecker(terminatingChecker(testAccount))
+
+				seal, err := sealer.SealAccountOverrides(ctx, testAccount)
+				require.NoError(t, err)
+
+				entry, err := realKV.Get(ctx, testAccount)
+				require.NoError(t, err)
+				rec := decodeOverride(t, entry)
+				assert.Equal(t, OverrideStateSealed, rec.State)
+				assert.Nil(t, rec.VCPUs)
+				assert.Equal(t, seal.Revision, entry.Revision())
+			})
+		}
+	}
+}
+
+// Neither mutation can act on an already-sealed record.
+func TestMutationOnSealedRecordIsRejected(t *testing.T) {
+	for _, m := range mutationTable() {
+		t.Run(m.name, func(t *testing.T) {
+			svc, _ := newSealService(t)
+			ctx := context.Background()
+			svc.SetAccountTerminationChecker(terminatingChecker(testAccount))
+			_, err := svc.SealAccountOverrides(ctx, testAccount)
+			require.NoError(t, err)
+
+			before, err := svc.overrides.Get(ctx, testAccount)
+			require.NoError(t, err)
+
+			err = m.call(svc, ctx)
+			require.Error(t, err)
+			assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked), "got %v", err)
+
+			after, err := svc.overrides.Get(ctx, testAccount)
+			require.NoError(t, err)
+			assert.Equal(t, before.Revision(), after.Revision())
+		})
+	}
+}
+
+// Every override mutation races one seal on a real bucket; afterwards the
+// record is sealed and stays sealed, so no success among them could have
+// landed after the seal.
+func TestSealConcurrentMutations(t *testing.T) {
+	svc, _ := newSealService(t)
+	ctx := context.Background()
+	svc.SetAccountTerminationChecker(terminatingChecker(testAccount))
+	require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(8)}, "operator"))
+
+	mutations := mutationTable()
+	errs := make([]error, len(mutations))
+	var wg sync.WaitGroup
+	for i, m := range mutations {
+		wg.Add(1)
+		go func(i int, m mutationCase) {
+			defer wg.Done()
+			errs[i] = m.call(svc, ctx)
+		}(i, m)
+	}
+
+	var sealResult *OverrideSeal
+	var sealErr error
+	wg.Go(func() {
+		sealResult, sealErr = svc.SealAccountOverrides(ctx, testAccount)
+	})
+	wg.Wait()
+
+	require.NoError(t, sealErr)
+	require.NotNil(t, sealResult)
+	for i, err := range errs {
+		if err != nil {
+			assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked), "mutation %s: got %v", mutations[i].name, err)
+		}
+	}
+
+	entry, err := svc.overrides.Get(ctx, testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, OverrideStateSealed, decodeOverride(t, entry).State)
+	assert.Equal(t, sealResult.Revision, entry.Revision())
+
+	_, _, err = svc.GetAccountQuota(ctx, testAccount)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
+}
+
+// A Get pinned to a revision the record has already moved past exhausts the
+// mutation's retries without ever writing.
+func TestMutationStaleReadExhaustsWithoutWriting(t *testing.T) {
+	svc, js := newSealService(t)
+	ctx := context.Background()
+
+	require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(8)}, "operator"))
+	stale, err := svc.overrides.Get(ctx, testAccount)
+	require.NoError(t, err)
+
+	require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(16)}, "operator"))
+
+	before := lastOverrideSeq(t, js)
+	svc.overrides = &staleGetKV{KeyValue: svc.overrides, entry: stale}
+
+	err = svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(32)}, "operator")
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
+	assert.Equal(t, before, lastOverrideSeq(t, js))
+}
+
+// A Get that always reports absent, while a sealed record actually exists,
+// never gets to recreate it: every Create attempt conflicts.
+func TestMutationStaleAbsentDoesNotRecreate(t *testing.T) {
+	svc, js := newSealService(t)
+	ctx := context.Background()
+	svc.SetAccountTerminationChecker(terminatingChecker(testAccount))
+	_, err := svc.SealAccountOverrides(ctx, testAccount)
+	require.NoError(t, err)
+
+	before := lastOverrideSeq(t, js)
+	svc.overrides = &staleNotFoundKV{KeyValue: svc.overrides}
+
+	err = svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(8)}, "operator")
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
+	assert.Equal(t, before, lastOverrideSeq(t, js))
+}
+
+// A revision that keeps moving under sustained concurrent writes must not let
+// the seal spin forever: it gives up, bounded by its attempt budget, and its
+// own write never lands.
+func TestSealSustainedConflictFailsWithoutWriting(t *testing.T) {
+	svc, _ := newSealService(t)
+	ctx := context.Background()
+	svc.SetAccountTerminationChecker(terminatingChecker(testAccount))
+	require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(8)}, "operator"))
+
+	realKV := svc.overrides
+	race := &sealRaceKV{KeyValue: realKV}
+	race.beforeWrite = func(int) error {
+		data, merr := json.Marshal(&overrideRecord{Overrides: Overrides{VCPUs: ptr(16)}})
+		require.NoError(t, merr)
+		_, perr := realKV.Put(ctx, testAccount, data)
+		return perr
+	}
+	svc.overrides = race
+
+	_, err := svc.SealAccountOverrides(ctx, testAccount)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
+	assert.Equal(t, sealMaxAttempts, race.writeCalls)
+
+	entry, err := realKV.Get(ctx, testAccount)
+	require.NoError(t, err)
+	assert.NotEqual(t, OverrideStateSealed, decodeOverride(t, entry).State, "the seal's own write never landed")
+}
+
+// A store error on the seal's read reports failed, never written.
+func TestSealStoreErrorWritesNothing(t *testing.T) {
+	svc, js := newSealService(t)
+	ctx := context.Background()
+	svc.SetAccountTerminationChecker(terminatingChecker(testAccount))
+
+	before := lastOverrideSeq(t, js)
+	svc.overrides = &errGetKV{KeyValue: svc.overrides, err: errors.New("kv unavailable")}
+
+	_, err := svc.SealAccountOverrides(ctx, testAccount)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
+	assert.Equal(t, before, lastOverrideSeq(t, js))
+}
+
+func TestIsRevisionConflictRecognisesTombstoneRetry(t *testing.T) {
+	assert.True(t, isRevisionConflict(jetstream.ErrKeyExists))
+	assert.True(t, isRevisionConflict(jetstream.ErrKeyRevisionMismatch))
+	assert.True(t, isRevisionConflict(&jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequence}))
+	assert.True(t, isRevisionConflict(&jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequenceConstant}))
+	assert.True(t, isRevisionConflict(fmt.Errorf("wrapped: %w", &jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequence})))
+	assert.False(t, isRevisionConflict(errors.New("unrelated failure")))
+	assert.False(t, isRevisionConflict(nil))
+}
+
+// Clearing never writes when there is nothing active to clear: an absent key,
+// a tombstoned key, and a sealed key hidden behind a stale not-found Get all
+// leave the stream exactly where they found it.
+func TestClearOnMissingKeyWritesNothing(t *testing.T) {
+	t.Run("absent", func(t *testing.T) {
+		svc, js := newSealService(t)
+		ctx := context.Background()
+		before := lastOverrideSeq(t, js)
+
+		require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{}, "operator"))
+		assert.Equal(t, before, lastOverrideSeq(t, js))
+	})
+
+	t.Run("tombstone", func(t *testing.T) {
+		svc, js := newSealService(t)
+		ctx := context.Background()
+		require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(8)}, "operator"))
+		require.NoError(t, svc.overrides.Delete(ctx, testAccount))
+		before := lastOverrideSeq(t, js)
+
+		require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{}, "operator"))
+		assert.Equal(t, before, lastOverrideSeq(t, js))
+	})
+
+	t.Run("sealed_hidden_by_stale_not_found", func(t *testing.T) {
+		svc, js := newSealService(t)
+		ctx := context.Background()
+		svc.SetAccountTerminationChecker(terminatingChecker(testAccount))
+		sealed, err := svc.SealAccountOverrides(ctx, testAccount)
+		require.NoError(t, err)
+		before := lastOverrideSeq(t, js)
+
+		realKV := svc.overrides
+		svc.overrides = &staleNotFoundKV{KeyValue: realKV}
+
+		require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{}, "operator"))
+		assert.Equal(t, before, lastOverrideSeq(t, js))
+
+		entry, err := realKV.Get(ctx, testAccount)
+		require.NoError(t, err)
+		rec := decodeOverride(t, entry)
+		assert.Equal(t, OverrideStateSealed, rec.State)
+		assert.Equal(t, sealed.Revision, entry.Revision())
+	})
+}
+
+// A seal against an absent or a tombstoned key creates the sealed record.
+func TestSealOnMissingKeyCreatesSealedRecord(t *testing.T) {
+	for _, state := range []string{"absent", "tombstone"} {
+		t.Run(state, func(t *testing.T) {
+			svc, _ := newSealService(t)
+			ctx := context.Background()
+			svc.SetAccountTerminationChecker(terminatingChecker(testAccount))
+			if state == "tombstone" {
+				require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(8)}, "operator"))
+				require.NoError(t, svc.overrides.Delete(ctx, testAccount))
+			}
+			_, err := svc.overrides.Get(ctx, testAccount)
+			require.ErrorIs(t, err, jetstream.ErrKeyNotFound)
+
+			seal, err := svc.SealAccountOverrides(ctx, testAccount)
+			require.NoError(t, err)
+
+			entry, err := svc.overrides.Get(ctx, testAccount)
+			require.NoError(t, err)
+			assert.Equal(t, seal.Revision, entry.Revision())
+			assert.Equal(t, OverrideStateSealed, decodeOverride(t, entry).State)
+		})
+	}
+}
+
+// alwaysConflictThenErrorKV answers Get as absent and every Create attempt
+// with a conflict, then a store error on the final attempt: a recording
+// wrapper proving the seal tried a real write on every attempt before giving
+// up, rather than ever reporting success from an observed absence.
+type alwaysConflictThenErrorKV struct {
+	jetstream.KeyValue
+
+	calls int
+}
+
+func (f *alwaysConflictThenErrorKV) Get(context.Context, string) (jetstream.KeyValueEntry, error) {
+	return nil, jetstream.ErrKeyNotFound
+}
+
+func (f *alwaysConflictThenErrorKV) Create(context.Context, string, []byte, ...jetstream.KVCreateOpt) (uint64, error) {
+	f.calls++
+	if f.calls < sealMaxAttempts {
+		return 0, jetstream.ErrKeyExists
+	}
+	return 0, errors.New("store unavailable")
+}
+
+func TestSealNeverSucceedsWithoutAWrite(t *testing.T) {
+	svc, _ := newSealService(t)
+	ctx := context.Background()
+	svc.SetAccountTerminationChecker(terminatingChecker(testAccount))
+	fake := &alwaysConflictThenErrorKV{}
+	svc.overrides = fake
+
+	seal, err := svc.SealAccountOverrides(ctx, testAccount)
+	assert.Error(t, err)
+	assert.Nil(t, seal)
+	assert.Equal(t, sealMaxAttempts, fake.calls, "every attempt must have tried a real write before giving up")
+}
+
+// An account IAM does not hold as TERMINATING refuses the seal before any
+// override-KV write, whether the key is present or absent.
+func TestSealRefusedForActiveAccount(t *testing.T) {
+	for _, state := range []string{"present", "absent"} {
+		t.Run(state, func(t *testing.T) {
+			svc, js := newSealService(t)
+			ctx := context.Background()
+			if state == "present" {
+				require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(8)}, "operator"))
+			}
+			svc.SetAccountTerminationChecker(&stubTerminationChecker{terminating: map[string]bool{}})
+
+			before := lastOverrideSeq(t, js)
+			_, err := svc.SealAccountOverrides(ctx, testAccount)
+			assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorIncorrectState), "got %v", err)
+			assert.Equal(t, before, lastOverrideSeq(t, js))
+		})
+	}
+}
+
+func TestSealRefusedForMissingIAMRecord(t *testing.T) {
+	svc, js := newSealService(t)
+	ctx := context.Background()
+	svc.SetAccountTerminationChecker(&stubTerminationChecker{terminating: map[string]bool{}})
+
+	before := lastOverrideSeq(t, js)
+	_, err := svc.SealAccountOverrides(ctx, testAccount)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorIncorrectState), "got %v", err)
+	assert.Equal(t, before, lastOverrideSeq(t, js))
+}
+
+func TestSealRefusedWhenIAMReadFails(t *testing.T) {
+	svc, js := newSealService(t)
+	ctx := context.Background()
+	svc.SetAccountTerminationChecker(&stubTerminationChecker{err: errors.New("iam unavailable")})
+
+	before := lastOverrideSeq(t, js)
+	_, err := svc.SealAccountOverrides(ctx, testAccount)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
+	assert.Equal(t, before, lastOverrideSeq(t, js))
+}
+
+func TestSealRefusedWithNoCheckerWired(t *testing.T) {
+	svc, js := newSealService(t)
+	ctx := context.Background()
+
+	before := lastOverrideSeq(t, js)
+	_, err := svc.SealAccountOverrides(ctx, testAccount)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
+	assert.Equal(t, before, lastOverrideSeq(t, js))
+}
+
+func TestSealAcceptedForTerminatingAccount(t *testing.T) {
+	svc, _ := newSealService(t)
+	ctx := context.Background()
+	svc.SetAccountTerminationChecker(terminatingChecker(testAccount))
+
+	seal, err := svc.SealAccountOverrides(ctx, testAccount)
+	require.NoError(t, err)
+	assert.NotZero(t, seal.Revision)
+
+	entry, err := svc.overrides.Get(ctx, testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, seal.Revision, entry.Revision())
+}
+
+// A refused seal, on a gateway that disabled quotas and so holds no override
+// bucket, never creates one.
+func TestSealRefusalDoesNotCreateBucket(t *testing.T) {
+	t.Run("not_terminating", func(t *testing.T) {
+		svc, js := newDisabledSealService(t)
+		ctx := context.Background()
+		svc.SetAccountTerminationChecker(&stubTerminationChecker{terminating: map[string]bool{}})
+
+		_, err := svc.SealAccountOverrides(ctx, testAccount)
+		assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorIncorrectState), "got %v", err)
+		_, err = js.KeyValue(context.Background(), KVBucketAccountQuota)
+		assert.ErrorIs(t, err, jetstream.ErrBucketNotFound)
+	})
+
+	t.Run("iam_read_fails", func(t *testing.T) {
+		svc, js := newDisabledSealService(t)
+		ctx := context.Background()
+		svc.SetAccountTerminationChecker(&stubTerminationChecker{err: errors.New("iam unavailable")})
+
+		_, err := svc.SealAccountOverrides(ctx, testAccount)
+		assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
+		_, err = js.KeyValue(context.Background(), KVBucketAccountQuota)
+		assert.ErrorIs(t, err, jetstream.ErrBucketNotFound)
+	})
+}
+
+// A seal accepted on a gateway whose quotas were disabled at startup creates
+// the override bucket at seal time, never at startup.
+func TestSealOnQuotaDisabledGatewayCreatesBucket(t *testing.T) {
+	svc, js := newDisabledSealService(t)
+	ctx := context.Background()
+	svc.SetAccountTerminationChecker(terminatingChecker(testAccount))
+
+	seal, err := svc.SealAccountOverrides(ctx, testAccount)
+	require.NoError(t, err)
+
+	kv, err := js.KeyValue(context.Background(), KVBucketAccountQuota)
+	require.NoError(t, err)
+	entry, err := kv.Get(context.Background(), testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, seal.Revision, entry.Revision())
+	assert.Equal(t, OverrideStateSealed, decodeOverride(t, entry).State)
+}
+
+// A sealed record stays sealed after the service is rebuilt over the same
+// JetStream, every read and write refuses it, and a re-seal gets a higher
+// revision.
+func TestSealSurvivesServiceReconstruction(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	ctx := context.Background()
+
+	kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: KVBucketAccountQuota, History: 1})
+	require.NoError(t, err)
+
+	svc := New(baseLimits(), nil)
+	svc.SetOverrides(kv)
+	svc.SetJetStream(js)
+	svc.SetAccountTerminationChecker(terminatingChecker(testAccount))
+	first, err := svc.SealAccountOverrides(ctx, testAccount)
+	require.NoError(t, err)
+
+	rebuiltKV, err := js.KeyValue(ctx, KVBucketAccountQuota)
+	require.NoError(t, err)
+	rebuilt := New(baseLimits(), nil)
+	rebuilt.SetOverrides(rebuiltKV)
+	rebuilt.SetJetStream(js)
+
+	entry, err := rebuilt.overrides.Get(ctx, testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, OverrideStateSealed, decodeOverride(t, entry).State)
+
+	_, _, err = rebuilt.GetAccountQuota(ctx, testAccount)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
+	assert.True(t, awserrors.IsErrorCode(rebuilt.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(8)}, "operator"), awserrors.ErrorBlocked))
+	assert.True(t, awserrors.IsErrorCode(rebuilt.PutAccountQuota(ctx, testAccount, Overrides{}, "operator"), awserrors.ErrorBlocked))
+
+	limits, err := rebuilt.limitsFor(ctx, testAccount)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
+	assert.Zero(t, limits)
+
+	rebuilt.SetAccountTerminationChecker(terminatingChecker(testAccount))
+	second, err := rebuilt.SealAccountOverrides(ctx, testAccount)
+	require.NoError(t, err)
+	assert.Greater(t, second.Revision, first.Revision)
+}
+
+func TestSealRefusesEmptyInvalidAndGlobalAccount(t *testing.T) {
+	svc, _ := newSealService(t)
+	ctx := context.Background()
+
+	_, err := svc.SealAccountOverrides(ctx, "")
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorMissingParameter), "got %v", err)
+
+	_, err = svc.SealAccountOverrides(ctx, "not-an-account")
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidParameterValue), "got %v", err)
+
+	_, err = svc.SealAccountOverrides(ctx, awsidentifiers.GlobalAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidParameterValue), "got %v", err)
+}
+
+func TestSealLeavesOtherAccountsUntouched(t *testing.T) {
+	svc, _ := newSealService(t)
+	ctx := context.Background()
+	accountB := "222222222222"
+	require.NoError(t, svc.PutAccountQuota(ctx, accountB, Overrides{VCPUs: ptr(8)}, "operator"))
+
+	svc.SetAccountTerminationChecker(terminatingChecker(testAccount))
+	_, err := svc.SealAccountOverrides(ctx, testAccount)
+	require.NoError(t, err)
+
+	over, _, err := svc.GetAccountQuota(ctx, accountB)
+	require.NoError(t, err)
+	require.NotNil(t, over.VCPUs)
+	assert.Equal(t, 8, *over.VCPUs)
+}
+
+// A record written before the state field existed has none and is treated as
+// active by both reads and writes; a write on it preserves the old shape.
+func TestRecordWithoutStateIsActive(t *testing.T) {
+	svc, _ := newSealService(t)
+	ctx := context.Background()
+
+	data, err := json.Marshal(Overrides{VCPUs: ptr(8)})
+	require.NoError(t, err)
+	_, err = svc.overrides.Put(ctx, testAccount, data)
+	require.NoError(t, err)
+
+	over, limits, err := svc.GetAccountQuota(ctx, testAccount)
+	require.NoError(t, err)
+	require.NotNil(t, over.VCPUs)
+	assert.Equal(t, 8, limits.VCPUs)
+
+	require.NoError(t, svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(16)}, "operator"))
+	entry, err := svc.overrides.Get(ctx, testAccount)
+	require.NoError(t, err)
+	assert.NotContains(t, string(entry.Value()), `"state"`)
+}
+
+// A record whose state is neither active nor sealed fails every read and
+// write closed, rather than being treated as active.
+func TestUnknownStateFailsClosed(t *testing.T) {
+	svc, _ := newSealService(t)
+	ctx := context.Background()
+
+	_, err := svc.overrides.Put(ctx, testAccount, []byte(`{"state":"frozen"}`))
+	require.NoError(t, err)
+
+	_, _, err = svc.GetAccountQuota(ctx, testAccount)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
+
+	limits, err := svc.limitsFor(ctx, testAccount)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
+	assert.Zero(t, limits)
+
+	assert.True(t, awserrors.IsErrorCode(svc.PutAccountQuota(ctx, testAccount, Overrides{VCPUs: ptr(8)}, "operator"), awserrors.ErrorInternalError))
+	assert.True(t, awserrors.IsErrorCode(svc.PutAccountQuota(ctx, testAccount, Overrides{}, "operator"), awserrors.ErrorInternalError))
+}
+
+// A sealed record is Blocked for admission's own resolver, which every
+// caller (admin read, ELBv2 DescribeAccountLimits, every Enforce* method)
+// shares.
+func TestAdmissionOnSealedRecordIsBlocked(t *testing.T) {
+	svc, _ := newSealService(t)
+	ctx := context.Background()
+	svc.SetAccountTerminationChecker(terminatingChecker(testAccount))
+	_, err := svc.SealAccountOverrides(ctx, testAccount)
+	require.NoError(t, err)
+
+	limits, err := svc.limitsFor(ctx, testAccount)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked), "got %v", err)
+	assert.Zero(t, limits)
+
+	assert.Error(t, svc.CheckVCPU(ctx, testAccount, 1))
+	assert.True(t, awserrors.IsErrorCode(svc.CheckVCPU(ctx, testAccount, 1), awserrors.ErrorBlocked))
+}
+
+// The existing read-failure fallback in limitsFor (a decode or store error
+// falls back to the configured limits) is unchanged by the seal work: it is
+// distinct from a successfully read sealed record, which refuses instead.
+func TestAdmissionReadFailureFallbackUnchanged(t *testing.T) {
+	svc, kv := newOverrideService(t)
+
+	_, err := kv.Put(t.Context(), testAccount, []byte("{not json"))
+	require.NoError(t, err)
+
+	limits, err := svc.limitsFor(t.Context(), testAccount)
+	require.NoError(t, err, "a decode failure must still fall back, not error, from limitsFor")
+	require.Equal(t, baseLimits(), limits)
+}

spinifex/domains/admission/quota/storage.go

@@ -23,7 +23,10 @@ func (s *Service) EnforceVolumeCreate(ctx context.Context, natsConn *nats.Conn,
 	if err != nil {
 		return err
 	}
-	limits := s.limitsFor(ctx, accountID)
+	limits, err := s.limitsFor(ctx, accountID)
+	if err != nil {
+		return err
+	}
 	if err := exceeds(len(out.Volumes), 1, limits.Volumes); err != nil {
 		return err
 	}
@@ -43,7 +46,11 @@ func (s *Service) EnforceVolumeModify(ctx context.Context, natsConn *nats.Conn,
 	if err != nil {
 		return err
 	}
-	return exceeds(total-oldGiB, newGiB, s.limitsFor(ctx, accountID).VolumesGiB)
+	limits, err := s.limitsFor(ctx, accountID)
+	if err != nil {
+		return err
+	}
+	return exceeds(total-oldGiB, newGiB, limits.VolumesGiB)
 }
 
 // volumeUsage sums the account's volume sizes in GiB via a single DescribeVolumes

spinifex/domains/admission/quota/vcpu.go

@@ -37,7 +37,11 @@ func (s *Service) CheckVCPU(ctx context.Context, accountID string, want int) err
 	if err != nil {
 		return err
 	}
-	return exceeds(current, want, s.limitsFor(ctx, accountID).VCPUs)
+	limits, err := s.limitsFor(ctx, accountID)
+	if err != nil {
+		return err
+	}
+	return exceeds(current, want, limits.VCPUs)
 }
 
 // casVCPU applies next to accountID's counter under bounded JetStream CAS so

spinifex/domains/dns/delegation.go

@@ -0,0 +1,104 @@
+package dns
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	nsconfig "github.com/mulgadc/northstar/pkg/config"
+	"github.com/nats-io/nats.go"
+
+	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
+)
+
+// delegateLockPoll is how often Delegate retries the zone-writer election
+// while another node holds it.
+var delegateLockPoll = time.Second
+
+// PlatformDelegation returns the NS set the base zone publishes for name and
+// the zone it read. ok is false when name is not strictly below the base
+// domain, so the zone's parent is somewhere else.
+func (w *Writer) PlatformDelegation(name string) (servers []string, zone string, ok bool, err error) {
+	if !w.Enabled() {
+		return nil, "", false, nil
+	}
+	base := normalizeZone(w.baseDomain)
+	n := normalizeZone(name)
+	if base == "" || !strings.HasSuffix(n, "."+base) {
+		return nil, "", false, nil
+	}
+	cfg, exists, err := nsconfig.ReadZoneRaw(w.s3cfg, base)
+	if err != nil {
+		return nil, base, true, fmt.Errorf("read zone %s: %w", base, err)
+	}
+	if !exists {
+		return nil, base, true, nil
+	}
+	label := relativeLabel(n, base)
+	for _, r := range cfg.Records {
+		if r.Type == nsconfig.TypeNS && strings.EqualFold(r.Domain, label) {
+			servers = append(servers, r.Address)
+		}
+	}
+	return servers, base, true, nil
+}
+
+// Delegate writes, or with remove withdraws, an NS set in the base zone that
+// delegates each name to this cluster's nameservers. It takes the zone-writer
+// election first, so it never races the reconcile's own writes to the zone.
+func (w *Writer) Delegate(ctx context.Context, nc *nats.Conn, holder string, names []string, remove bool) (*ChangeResult, error) {
+	if !w.Enabled() {
+		return nil, errors.New("northstar is not configured on this node")
+	}
+	base := normalizeZone(w.baseDomain)
+	values := make([]string, 0, len(w.nameservers))
+	for _, ns := range w.nameservers {
+		values = append(values, ns.Host+"."+base+".")
+	}
+	batch := &ChangeBatch{}
+	for _, name := range names {
+		n := normalizeZone(name)
+		if !strings.HasSuffix(n, "."+base) {
+			return nil, fmt.Errorf("%s is not below the base domain %s", name, base)
+		}
+		if ReservedZoneName(n, base, w.internalDomain) {
+			return nil, fmt.Errorf("%s is a platform name and cannot be delegated", name)
+		}
+		if remove {
+			for _, v := range values {
+				batch.Changes = append(batch.Changes, Change{Action: ActionDelete, Zone: base, Name: n + ".", Type: "NS", Value: v})
+			}
+			continue
+		}
+		batch.Changes = append(batch.Changes, Change{Action: ActionUpsertSet, Zone: base, Name: n + ".", Type: "NS", Values: values})
+	}
+	if len(batch.Changes) == 0 {
+		return &ChangeResult{}, nil
+	}
+	release, err := w.awaitWriterElection(ctx, nc, holder)
+	if err != nil {
+		return nil, err
+	}
+	defer release()
+	return w.ApplyBatch(batch)
+}
+
+func (w *Writer) awaitWriterElection(ctx context.Context, nc *nats.Conn, holder string) (func(), error) {
+	if nc == nil {
+		return func() {}, nil
+	}
+	tick := time.NewTicker(delegateLockPoll)
+	defer tick.Stop()
+	for {
+		if release, ok := reconcilelock.AcquireLeader(ctx, nc, KVBucketDNSReconcile, holder); ok {
+			return release, nil
+		}
+		select {
+		case <-ctx.Done():
+			return nil, fmt.Errorf("another node is writing the zone: %w", ctx.Err())
+		case <-tick.C:
+		}
+	}
+}

spinifex/domains/dns/delegation_test.go

@@ -0,0 +1,82 @@
+package dns
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
+)
+
+func TestDelegateWritesAndWithdrawsPlatformDelegations(t *testing.T) {
+	w, _ := newMultiNodeTestWriter(t)
+	_, err := w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net", "B.r53e2e.spx3.net."}, false)
+	require.NoError(t, err)
+
+	for _, name := range []string{"a.r53e2e.spx3.net.", "b.r53e2e.spx3.net"} {
+		servers, zone, ok, err := w.PlatformDelegation(name)
+		require.NoError(t, err)
+		assert.True(t, ok)
+		assert.Equal(t, "spx3.net", zone)
+		assert.ElementsMatch(t, []string{"ns1.spx3.net.", "ns2.spx3.net."}, servers, name)
+	}
+	servers, _, ok, err := w.PlatformDelegation("c.r53e2e.spx3.net")
+	require.NoError(t, err)
+	assert.True(t, ok, "a name below the base domain is decided by the base zone")
+	assert.Empty(t, servers)
+
+	_, err = w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net"}, true)
+	require.NoError(t, err)
+	servers, _, _, err = w.PlatformDelegation("a.r53e2e.spx3.net")
+	require.NoError(t, err)
+	assert.Empty(t, servers)
+	servers, _, _, err = w.PlatformDelegation("b.r53e2e.spx3.net")
+	require.NoError(t, err)
+	assert.Len(t, servers, 2, "withdrawing one delegation leaves the others")
+}
+
+func TestPlatformDelegationIgnoresNamesOutsideTheBaseDomain(t *testing.T) {
+	w, _ := newMultiNodeTestWriter(t)
+	for _, name := range []string{"example.com", "spx3.net", "notspx3.net"} {
+		_, _, ok, err := w.PlatformDelegation(name)
+		require.NoError(t, err)
+		assert.False(t, ok, name)
+	}
+}
+
+func TestDelegateRefusesPlatformAndForeignNames(t *testing.T) {
+	w, _ := newMultiNodeTestWriter(t)
+	for _, name := range []string{"ns1.spx3.net", "x.elb.spx3.net", "compute.spx3.net", "example.com", "spx3.net"} {
+		_, err := w.Delegate(t.Context(), nil, "node1", []string{"ok.spx3.net", name}, false)
+		require.Error(t, err, name)
+	}
+	servers, _, _, err := w.PlatformDelegation("ok.spx3.net")
+	require.NoError(t, err)
+	assert.Empty(t, servers, "a refused batch writes nothing")
+}
+
+func TestDelegateWaitsForTheZoneWriterElection(t *testing.T) {
+	w, _ := newMultiNodeTestWriter(t)
+	_, nc, _ := testutil.StartTestJetStream(t)
+	prev := delegateLockPoll
+	delegateLockPoll = 10 * time.Millisecond
+	t.Cleanup(func() { delegateLockPoll = prev })
+
+	release, ok := reconcilelock.AcquireLeader(t.Context(), nc, KVBucketDNSReconcile, "node2")
+	require.True(t, ok)
+	ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
+	defer cancel()
+	_, err := w.Delegate(ctx, nc, "node1", []string{"a.spx3.net"}, false)
+	require.ErrorContains(t, err, "another node is writing the zone")
+
+	release()
+	_, err = w.Delegate(t.Context(), nc, "node1", []string{"a.spx3.net"}, false)
+	require.NoError(t, err)
+	servers, _, _, err := w.PlatformDelegation("a.spx3.net")
+	require.NoError(t, err)
+	assert.Len(t, servers, 2)
+}

spinifex/domains/dns/nameservers.go

@@ -10,17 +10,18 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 )
 
-// NameserverSeeds derives one nameserver (nsN → node IP) per cluster node that
-// runs northstar and is reachable by its peers, ordered deterministically so
-// every node computes the same set. Falls back to the local node when none
-// qualifies (single-node / dev).
+// NameserverSeeds derives the published nameservers (nsN → node IP) from the
+// cluster nodes that run northstar and are reachable by their peers: the
+// [northstar] pins first, then the rest in name order, capped at
+// nameserver_count or MaxNameservers. Every node computes the same set. Falls
+// back to the local node when none qualifies (single-node / dev).
 //
 // Both zone creators build from this: the northstar bootstrap seed and the
 // writer's on-demand materialisation. Both are create-if-absent and neither
 // overwrites, so they must agree — a set derived independently would be baked
 // into the zone permanently by whichever creator ran first.
 func NameserverSeeds(cluster *config.ClusterConfig) []nsconfig.NameserverSeed {
-	names := usableNorthstarNodeNames(cluster)
+	names := publishedNameserverNodes(usableNorthstarNodeNames(cluster), cluster.Northstar)
 	// A zone needs an SOA and at least one NS whatever the topology, so unlike the
 	// resolver list this falls back to the local node. On single-node dev that node
 	// is loopback, which is correct there: it is the zone's only reader.
@@ -29,17 +30,37 @@ func NameserverSeeds(cluster *config.ClusterConfig) []nsconfig.NameserverSeed {
 	}
 
 	seeds := make([]nsconfig.NameserverSeed, 0, len(names))
-	for i, name := range names {
-		seeds = append(seeds, nsconfig.NameserverSeed{
-			Host: fmt.Sprintf("ns%d", i+1),
-			IP:   nameserverIP(cluster.Nodes[name]),
-		})
+	for _, ns := range nameserverAssignments(cluster, names) {
+		seeds = append(seeds, nsconfig.NameserverSeed{Host: ns.Host, IP: ns.IP})
 	}
 	return seeds
 }
 
-// ResolverNameserverIPs returns the WAN IPs of cluster nodes running northstar,
-// in the same deterministic order as the seeded nameservers. vpcd's per-tap DNS
+// Nameserver is one published nameserver: its label under the base domain,
+// the node it is assigned to and that node's address.
+type Nameserver struct {
+	Host string
+	Node string
+	IP   string
+}
+
+// PublishedNameservers returns the nsN assignment NameserverSeeds publishes,
+// with the node behind each name. Unlike NameserverSeeds it has no local-node
+// fallback, so an empty result means no node qualifies.
+func PublishedNameservers(cluster *config.ClusterConfig) []Nameserver {
+	return nameserverAssignments(cluster, publishedNameserverNodes(usableNorthstarNodeNames(cluster), cluster.Northstar))
+}
+
+func nameserverAssignments(cluster *config.ClusterConfig, names []string) []Nameserver {
+	out := make([]Nameserver, 0, len(names))
+	for i, name := range names {
+		out = append(out, Nameserver{Host: fmt.Sprintf("ns%d", i+1), Node: name, IP: nameserverIP(cluster.Nodes[name])})
+	}
+	return out
+}
+
+// ResolverNameserverIPs returns the WAN IPs of every cluster node running
+// northstar in name order, not only the published nameservers. vpcd's per-tap DNS
 // shim uses these as forward targets (northstar's :5300 listener), so internal
 // names resolve authoritatively and external names via upstream forwarders.
 //
@@ -55,6 +76,36 @@ func ResolverNameserverIPs(cluster *config.ClusterConfig) []string {
 	return ips
 }
 
+// publishedNameserverNodes orders the usable northstar nodes for publication
+// and truncates them to the configured count. A pinned node that is not usable
+// is skipped rather than published with an address its peers cannot reach.
+func publishedNameserverNodes(usable []string, ns config.NorthstarClusterConfig) []string {
+	limit := ns.NameserverCount
+	if limit <= 0 || limit > config.MaxNameservers {
+		limit = config.MaxNameservers
+	}
+	ok := make(map[string]bool, len(usable))
+	for _, name := range usable {
+		ok[name] = true
+	}
+	ordered := make([]string, 0, len(usable))
+	for _, name := range ns.NameserverNodes {
+		if ok[name] {
+			ordered = append(ordered, name)
+			delete(ok, name)
+		}
+	}
+	for _, name := range usable {
+		if ok[name] {
+			ordered = append(ordered, name)
+		}
+	}
+	if len(ordered) > limit {
+		ordered = ordered[:limit]
+	}
+	return ordered
+}
+
 // usableNorthstarNodeNames returns the deterministically ordered nodes that run
 // northstar and can be reached by their peers.
 //

spinifex/domains/dns/nameservers_test.go

@@ -1,8 +1,10 @@
 package dns
 
 import (
+	"fmt"
 	"testing"
 
+	nsconfig "github.com/mulgadc/northstar/pkg/config"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -132,3 +134,62 @@ func TestResolverNameserverIPs(t *testing.T) {
 	}}
 	assert.Empty(t, ResolverNameserverIPs(dev))
 }
+
+func northstarNodes(n int) map[string]config.Config {
+	nodes := make(map[string]config.Config, n)
+	for i := 1; i <= n; i++ {
+		nodes[fmt.Sprintf("node%02d", i)] = config.Config{
+			AdvertiseIP: fmt.Sprintf("10.0.0.%d", i),
+			Northstar:   config.NorthstarConfig{ConfigPath: "/etc/n.toml"},
+		}
+	}
+	return nodes
+}
+
+func seedIPs(seeds []nsconfig.NameserverSeed) []string {
+	ips := make([]string, 0, len(seeds))
+	for i, s := range seeds {
+		if s.Host != fmt.Sprintf("ns%d", i+1) {
+			return []string{"misnumbered " + s.Host}
+		}
+		ips = append(ips, s.IP)
+	}
+	return ips
+}
+
+// A large cluster publishes at most four nameservers, while every northstar
+// node stays a resolver backend for the guest shim.
+func TestNameserverSeedsCappedAtFour(t *testing.T) {
+	cluster := &config.ClusterConfig{Node: "node01", Nodes: northstarNodes(12)}
+	assert.Equal(t, []string{"10.0.0.1", "10.0.0.2", "10.0.0.3", "10.0.0.4"}, seedIPs(NameserverSeeds(cluster)))
+	assert.Len(t, ResolverNameserverIPs(cluster), 12)
+}
+
+func TestNameserverSeedsHonourCountAndPins(t *testing.T) {
+	nodes := northstarNodes(6)
+	nodes["node06"] = config.Config{AdvertiseIP: "127.0.0.1", Northstar: config.NorthstarConfig{ConfigPath: "/etc/n.toml"}}
+
+	for name, tc := range map[string]struct {
+		ns   config.NorthstarClusterConfig
+		want []string
+	}{
+		"count below the node count": {config.NorthstarClusterConfig{NameserverCount: 2}, []string{"10.0.0.1", "10.0.0.2"}},
+		"pins first, rest in name order": {
+			config.NorthstarClusterConfig{NameserverNodes: []string{"node05", "node03"}},
+			[]string{"10.0.0.5", "10.0.0.3", "10.0.0.1", "10.0.0.2"},
+		},
+		"pins fill the count": {
+			config.NorthstarClusterConfig{NameserverCount: 2, NameserverNodes: []string{"node04", "node02"}},
+			[]string{"10.0.0.4", "10.0.0.2"},
+		},
+		"unusable pin is skipped": {
+			config.NorthstarClusterConfig{NameserverCount: 2, NameserverNodes: []string{"node06", "node05"}},
+			[]string{"10.0.0.5", "10.0.0.1"},
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			cluster := &config.ClusterConfig{Node: "node01", Nodes: nodes, Northstar: tc.ns}
+			assert.Equal(t, tc.want, seedIPs(NameserverSeeds(cluster)))
+		})
+	}
+}

spinifex/domains/dns/parent.go

@@ -0,0 +1,179 @@
+package dns
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"slices"
+	"strings"
+	"time"
+
+	mdns "github.com/miekg/dns"
+)
+
+// RootServers are the IANA root servers, by IPv4 address.
+var RootServers = []string{
+	"198.41.0.4", "170.247.170.2", "192.33.4.12", "199.7.91.13", "192.203.230.10",
+	"192.5.5.241", "192.112.36.4", "198.97.190.53", "192.36.148.17", "192.58.128.30",
+	"193.0.14.129", "199.7.83.42", "202.12.27.33",
+}
+
+const (
+	parentQueryTimeout = 3 * time.Second
+	maxReferrals       = 16
+)
+
+// ParentLookup reads a domain's delegation from its parent zone's own servers,
+// found by following referrals down from the roots. It never asks a recursive
+// resolver, whose cache can hold a delegation the parent has since withdrawn.
+type ParentLookup struct {
+	// Roots are the servers the walk starts from; nil means RootServers.
+	Roots []string
+	// Exchange sends one query; nil means UDP to port 53, retried over TCP
+	// when truncated.
+	Exchange func(ctx context.Context, m *mdns.Msg, server string) (*mdns.Msg, error)
+	// Resolve finds a name server's addresses when a referral has no glue.
+	Resolve func(ctx context.Context, host string) ([]string, error)
+}
+
+// Delegation returns the NS names the parent of name publishes for it and the
+// parent zone that answered. An empty set with a nil error means the parent
+// answered and does not delegate name.
+func (p *ParentLookup) Delegation(ctx context.Context, name string) (servers []string, parent string, err error) {
+	target := mdns.Fqdn(strings.ToLower(name))
+	addrs := p.Roots
+	if addrs == nil {
+		addrs = RootServers
+	}
+	zone := "."
+	for range maxReferrals {
+		resp, err := p.ask(ctx, target, addrs)
+		if err != nil {
+			return nil, zone, fmt.Errorf("ask the %s servers: %w", zone, err)
+		}
+		owner, ns := referral(resp, target)
+		if owner == "" || resp.Authoritative {
+			// The servers hold target's own zone or its parent: what they list
+			// for target is the answer, and nothing listed is no delegation.
+			return answerNS(resp, target), strings.TrimSuffix(zone, "."), nil
+		}
+		if owner == target {
+			return ns, strings.TrimSuffix(zone, "."), nil
+		}
+		if !mdns.IsSubDomain(zone, owner) || owner == zone {
+			return nil, zone, fmt.Errorf("the %s servers referred %s back to %s", zone, target, owner)
+		}
+		zone = owner
+		if addrs, err = p.addresses(ctx, resp, ns); err != nil {
+			return nil, zone, err
+		}
+	}
+	return nil, zone, fmt.Errorf("more than %d referrals for %s", maxReferrals, target)
+}
+
+// ask sends the query to each server in turn and returns the first answer
+// that is NOERROR or NXDOMAIN.
+func (p *ParentLookup) ask(ctx context.Context, target string, servers []string) (*mdns.Msg, error) {
+	m := new(mdns.Msg)
+	m.SetQuestion(target, mdns.TypeNS)
+	m.RecursionDesired = false
+	exchange := p.Exchange
+	if exchange == nil {
+		exchange = exchangeUDP
+	}
+	lastErr := errors.New("no servers")
+	for _, s := range servers {
+		resp, err := exchange(ctx, m, s)
+		switch {
+		case err != nil:
+			lastErr = err
+		case resp.Rcode != mdns.RcodeSuccess && resp.Rcode != mdns.RcodeNameError:
+			lastErr = fmt.Errorf("%s answered %s", s, mdns.RcodeToString[resp.Rcode])
+		default:
+			return resp, nil
+		}
+		if ctx.Err() != nil {
+			return nil, ctx.Err()
+		}
+	}
+	return nil, lastErr
+}
+
+// referral returns the deepest zone at or above target that the response's
+// authority section delegates, and its NS names.
+func referral(resp *mdns.Msg, target string) (string, []string) {
+	var owner string
+	var ns []string
+	for _, rr := range resp.Ns {
+		r, ok := rr.(*mdns.NS)
+		if !ok {
+			continue
+		}
+		o := strings.ToLower(r.Hdr.Name)
+		if !mdns.IsSubDomain(o, target) {
+			continue
+		}
+		if o != owner && (owner == "" || mdns.CountLabel(o) > mdns.CountLabel(owner)) {
+			owner, ns = o, nil
+		}
+		if o == owner {
+			ns = append(ns, strings.ToLower(r.Ns))
+		}
+	}
+	return owner, ns
+}
+
+func answerNS(resp *mdns.Msg, target string) []string {
+	var ns []string
+	for _, rr := range append(resp.Answer, resp.Ns...) {
+		if r, ok := rr.(*mdns.NS); ok && strings.EqualFold(r.Hdr.Name, target) {
+			ns = append(ns, strings.ToLower(r.Ns))
+		}
+	}
+	return ns
+}
+
+// addresses returns where to send the next query: the referral's glue, or a
+// lookup of the server names when the parent gave none.
+func (p *ParentLookup) addresses(ctx context.Context, resp *mdns.Msg, ns []string) ([]string, error) {
+	var addrs []string
+	for _, rr := range resp.Extra {
+		if a, ok := rr.(*mdns.A); ok && slices.ContainsFunc(ns, func(n string) bool { return strings.EqualFold(n, a.Hdr.Name) }) {
+			addrs = append(addrs, a.A.String())
+		}
+	}
+	if len(addrs) > 0 {
+		return addrs, nil
+	}
+	resolve := p.Resolve
+	if resolve == nil {
+		resolve = net.DefaultResolver.LookupHost
+	}
+	var lastErr error
+	for _, host := range ns {
+		found, err := resolve(ctx, strings.TrimSuffix(host, "."))
+		if err != nil {
+			lastErr = err
+			continue
+		}
+		for _, a := range found {
+			if ip := net.ParseIP(a); ip != nil && ip.To4() != nil {
+				addrs = append(addrs, a)
+			}
+		}
+	}
+	if len(addrs) == 0 {
+		return nil, fmt.Errorf("no address for any of %s: %w", strings.Join(ns, ", "), lastErr)
+	}
+	return addrs, nil
+}
+
+func exchangeUDP(ctx context.Context, m *mdns.Msg, server string) (*mdns.Msg, error) {
+	addr := net.JoinHostPort(server, "53")
+	resp, _, err := (&mdns.Client{Timeout: parentQueryTimeout}).ExchangeContext(ctx, m, addr)
+	if err == nil && resp.Truncated {
+		resp, _, err = (&mdns.Client{Net: "tcp", Timeout: parentQueryTimeout}).ExchangeContext(ctx, m, addr)
+	}
+	return resp, err
+}

spinifex/domains/dns/parent_test.go

@@ -0,0 +1,139 @@
+package dns_test
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	mdns "github.com/miekg/dns"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/dns"
+)
+
+// fakeTree is the DNS hierarchy as each server's address answers it.
+type fakeTree struct {
+	servers map[string]func(q mdns.Question) *mdns.Msg
+	asked   []string
+}
+
+func (f *fakeTree) exchange(_ context.Context, m *mdns.Msg, server string) (*mdns.Msg, error) {
+	f.asked = append(f.asked, server)
+	if m.RecursionDesired {
+		return nil, errors.New("recursion requested")
+	}
+	h, ok := f.servers[server]
+	if !ok {
+		return nil, errors.New("i/o timeout")
+	}
+	resp := h(m.Question[0])
+	rcode, aa := resp.Rcode, resp.Authoritative
+	resp.SetReply(m)
+	resp.Rcode, resp.Authoritative = rcode, aa
+	return resp, nil
+}
+
+func ns(owner string, hosts ...string) []mdns.RR {
+	var out []mdns.RR
+	for _, h := range hosts {
+		out = append(out, &mdns.NS{Hdr: mdns.RR_Header{Name: owner, Rrtype: mdns.TypeNS, Class: mdns.ClassINET, Ttl: 172800}, Ns: h})
+	}
+	return out
+}
+
+func glue(host, ip string) mdns.RR {
+	rr, _ := mdns.NewRR(host + " 172800 IN A " + ip)
+	return rr
+}
+
+func referralTo(owner string, extra []mdns.RR, hosts ...string) *mdns.Msg {
+	return &mdns.Msg{Ns: ns(owner, hosts...), Extra: extra}
+}
+
+func authoritative(rcode int) *mdns.Msg {
+	m := &mdns.Msg{}
+	m.Authoritative = true
+	m.Rcode = rcode
+	return m
+}
+
+func newTree() *fakeTree {
+	return &fakeTree{servers: map[string]func(mdns.Question) *mdns.Msg{
+		"10.0.0.1": func(q mdns.Question) *mdns.Msg {
+			if mdns.IsSubDomain("org.", q.Name) {
+				return referralTo("org.", nil, "a0.org-servers.test.") // no glue
+			}
+			return referralTo("com.", []mdns.RR{glue("a.gtld.test.", "10.0.1.1")}, "a.gtld.test.")
+		},
+		"10.0.1.1": func(q mdns.Question) *mdns.Msg {
+			switch {
+			case mdns.IsSubDomain("example.com.", q.Name):
+				return referralTo("example.com.", nil, "NS1.spx3.net.", "ns2.spx3.net.")
+			case mdns.IsSubDomain("deep.com.", q.Name):
+				return referralTo("deep.com.", []mdns.RR{glue("ns.deep.com.", "10.0.2.1")}, "ns.deep.com.")
+			default:
+				return authoritative(mdns.RcodeNameError)
+			}
+		},
+		"10.0.2.1": func(q mdns.Question) *mdns.Msg {
+			if q.Name == "shop.deep.com." {
+				return referralTo("shop.deep.com.", nil, "ns1.spx3.net.")
+			}
+			return authoritative(mdns.RcodeSuccess) // NODATA: not delegated
+		},
+		"10.0.3.1": func(q mdns.Question) *mdns.Msg {
+			return referralTo(q.Name, nil, "ns3.spx3.net.")
+		},
+	}}
+}
+
+func TestParentLookupFollowsReferralsToTheParent(t *testing.T) {
+	tree := newTree()
+	p := &dns.ParentLookup{
+		Roots:    []string{"10.0.0.9", "10.0.0.1"}, // the first root is down
+		Exchange: tree.exchange,
+		Resolve: func(_ context.Context, host string) ([]string, error) {
+			if host == "a0.org-servers.test" {
+				return []string{"2001:db8::1", "10.0.3.1"}, nil
+			}
+			return nil, errors.New("no such host")
+		},
+	}
+	cases := []struct {
+		name    string
+		servers []string
+		parent  string
+	}{
+		{"Example.COM", []string{"ns1.spx3.net.", "ns2.spx3.net."}, "com"},
+		{"shop.deep.com.", []string{"ns1.spx3.net."}, "deep.com"},
+		{"other.deep.com", nil, "deep.com"},
+		{"unregistered.com", nil, "com"},
+		{"example.org", []string{"ns3.spx3.net."}, "org"},
+	}
+	for _, tc := range cases {
+		servers, parent, err := p.Delegation(context.Background(), tc.name)
+		require.NoError(t, err, tc.name)
+		assert.Equal(t, tc.servers, servers, tc.name)
+		assert.Equal(t, tc.parent, parent, tc.name)
+	}
+}
+
+func TestParentLookupFailsWhenNoServerAnswers(t *testing.T) {
+	tree := newTree()
+	tree.servers["10.0.1.1"] = func(mdns.Question) *mdns.Msg { return &mdns.Msg{MsgHdr: mdns.MsgHdr{Rcode: mdns.RcodeServerFailure}} }
+	p := &dns.ParentLookup{Roots: []string{"10.0.0.1"}, Exchange: tree.exchange}
+	_, parent, err := p.Delegation(context.Background(), "example.com")
+	require.ErrorContains(t, err, "SERVFAIL")
+	assert.Equal(t, "com.", parent)
+}
+
+func TestParentLookupRefusesAReferralUpwards(t *testing.T) {
+	tree := newTree()
+	tree.servers["10.0.1.1"] = func(mdns.Question) *mdns.Msg {
+		return referralTo(".", []mdns.RR{glue("a.root.test.", "10.0.0.1")}, "a.root.test.")
+	}
+	p := &dns.ParentLookup{Roots: []string{"10.0.0.1"}, Exchange: tree.exchange}
+	_, _, err := p.Delegation(context.Background(), "example.com")
+	require.Error(t, err)
+}

spinifex/domains/dns/publisher.go

@@ -0,0 +1,78 @@
+package dns
+
+import (
+	"context"
+	"time"
+
+	nsconfig "github.com/mulgadc/northstar/pkg/config"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/nats-io/nats.go"
+)
+
+// zoneReloadTimeout bounds one wait for every northstar server to load a zone.
+const zoneReloadTimeout = 5 * time.Second
+
+// ZonePublisher writes whole zone documents to the northstar zone store and
+// asks every northstar server to load them, for callers that own a zone
+// outright rather than editing records in one.
+type ZonePublisher struct {
+	s3      *nsconfig.S3Config
+	nc      *nats.Conn
+	cluster func() *config.ClusterConfig
+}
+
+// NewZonePublisher returns a publisher, or false when northstar's S3 zone store
+// is not configured on this node. cluster is read on every reload, so a node
+// joining changes how many acknowledgements are awaited.
+func NewZonePublisher(cfg *config.Config, cluster func() *config.ClusterConfig, nc *nats.Conn) (*ZonePublisher, bool) {
+	zoneCfg, ok := zoneS3Config(cfg)
+	if !ok || nc == nil {
+		return nil, false
+	}
+	return &ZonePublisher{s3: zoneCfg.s3, nc: nc, cluster: cluster}, true
+}
+
+// PutZone writes apex's zone document.
+func (p *ZonePublisher) PutZone(_ context.Context, apex string, body []byte) error {
+	return nsconfig.WriteZoneFile(p.s3, apex, body)
+}
+
+// DeleteZone removes apex's zone document.
+func (p *ZonePublisher) DeleteZone(_ context.Context, apex string) error {
+	return nsconfig.DeleteZoneFile(p.s3, apex)
+}
+
+// Reload asks every northstar server to load apex and returns the version each
+// reports, "" where it serves nothing. A server that failed to load reports a
+// value no digest can equal. complete is whether every server answered,
+// counted by node so a repeated acknowledgement cannot stand in for another.
+func (p *ZonePublisher) Reload(ctx context.Context, apex string) ([]string, bool, error) {
+	want := NorthstarServerCount(p.cluster())
+	ctx, cancel := context.WithTimeout(ctx, zoneReloadTimeout)
+	defer cancel()
+	acks, err := RequestZoneReload(ctx, p.nc, apex, want)
+	if err != nil {
+		return nil, false, err
+	}
+	served := make([]string, 0, len(acks))
+	nodes := map[string]bool{}
+	for _, ack := range acks {
+		nodes[ack.Node] = true
+		if ack.Error != "" {
+			served = append(served, "error: "+ack.Error)
+			continue
+		}
+		served = append(served, ack.ETag)
+	}
+	return served, len(nodes) >= want, nil
+}
+
+// NorthstarServerCount is how many northstar servers the cluster runs, and so
+// how many must acknowledge a reload. A cluster with none configured still runs
+// the local one.
+func NorthstarServerCount(cluster *config.ClusterConfig) int {
+	if cluster == nil {
+		return 1
+	}
+	return max(1, len(configuredNorthstarNodeNames(cluster)))
+}

spinifex/domains/dns/publisher_test.go

@@ -0,0 +1,135 @@
+package dns_test
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/mulgadc/spinifex/spinifex/domains/dns"
+)
+
+// zoneBucket is a path-style S3 endpoint holding whole objects.
+func zoneBucket(t *testing.T) (endpoint string, get func(key string) (string, bool)) {
+	t.Helper()
+	var mu sync.Mutex
+	objects := map[string]string{}
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		key := strings.TrimPrefix(r.URL.Path, "/northstar/")
+		mu.Lock()
+		defer mu.Unlock()
+		switch r.Method {
+		case http.MethodPut:
+			body, _ := io.ReadAll(r.Body)
+			objects[key] = string(body)
+		case http.MethodDelete:
+			delete(objects, key)
+			w.WriteHeader(http.StatusNoContent)
+		default:
+			http.Error(w, "unsupported", http.StatusMethodNotAllowed)
+		}
+	}))
+	t.Cleanup(srv.Close)
+	return srv.URL, func(key string) (string, bool) {
+		mu.Lock()
+		defer mu.Unlock()
+		v, ok := objects[key]
+		return v, ok
+	}
+}
+
+func publisherConfig(t *testing.T, endpoint string) *config.Config {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "northstar.toml")
+	body := fmt.Sprintf("listen = \"0.0.0.0:5300\"\ndefault_domain = \"spx3.net\"\n[s3]\nendpoint = %q\nbucket = \"northstar\"\nregion = \"us-east-1\"\naccess_key = \"RO\"\nsecret_key = \"RO\"\n", endpoint)
+	require.NoError(t, os.WriteFile(path, []byte(body), 0o600))
+	return &config.Config{
+		Predastore: config.PredastoreConfig{AccessKey: "SYSTEM", SecretKey: "SYSTEMSECRET"},
+		Northstar:  config.NorthstarConfig{ConfigPath: path},
+	}
+}
+
+func cluster(nodes ...string) *config.ClusterConfig {
+	c := &config.ClusterConfig{Node: nodes[0], Nodes: map[string]config.Config{}}
+	for _, n := range nodes {
+		c.Nodes[n] = config.Config{Northstar: config.NorthstarConfig{ConfigPath: "/etc/spinifex/northstar.toml"}}
+	}
+	return c
+}
+
+func TestZonePublisherWritesAndDeletesZoneObjects(t *testing.T) {
+	endpoint, get := zoneBucket(t)
+	_, nc := testutil.StartTestNATS(t)
+	pub, ok := dns.NewZonePublisher(publisherConfig(t, endpoint), func() *config.ClusterConfig { return cluster("node1") }, nc)
+	require.True(t, ok)
+
+	require.NoError(t, pub.PutZone(context.Background(), "Example.com", []byte("body")))
+	body, ok := get("example.com.toml")
+	require.True(t, ok)
+	assert.Equal(t, "body", body)
+
+	require.NoError(t, pub.DeleteZone(context.Background(), "example.com"))
+	_, ok = get("example.com.toml")
+	assert.False(t, ok)
+}
+
+func TestZonePublisherNeedsTheZoneStore(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	_, ok := dns.NewZonePublisher(&config.Config{}, func() *config.ClusterConfig { return nil }, nc)
+	assert.False(t, ok)
+}
+
+// A reload is complete only when every northstar node has answered; a failed
+// node reports a version no digest matches, and a node answering twice does not
+// stand in for one that never answered.
+func TestZonePublisherReloadCountsEveryNorthstarNode(t *testing.T) {
+	endpoint, _ := zoneBucket(t)
+	nodes := cluster("node1", "node2", "node3")
+	cases := []struct {
+		name     string
+		acks     []dns.ZoneReloadAck
+		served   []string
+		complete bool
+	}{
+		{"one failed, one silent", []dns.ZoneReloadAck{{Node: "node1", ETag: "v1"}, {Node: "node2", Error: "s3 down"}}, []string{"v1", "error: s3 down"}, false},
+		{"one node twice", []dns.ZoneReloadAck{{Node: "node1", ETag: "v1"}, {Node: "node1", ETag: "v1"}}, []string{"v1", "v1"}, false},
+		{"every node", []dns.ZoneReloadAck{{Node: "node1", ETag: "v1"}, {Node: "node2", ETag: "v1"}, {Node: "node3", ETag: "v1"}}, []string{"v1", "v1", "v1"}, true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			_, nc := testutil.StartTestNATS(t)
+			pub, ok := dns.NewZonePublisher(publisherConfig(t, endpoint), func() *config.ClusterConfig { return nodes }, nc)
+			require.True(t, ok)
+			ackAs(t, nc, func(zone string) []dns.ZoneReloadAck {
+				acks := slices.Clone(tc.acks)
+				for i := range acks {
+					acks[i].Zone = zone
+				}
+				return acks
+			})
+
+			served, complete, err := pub.Reload(context.Background(), "example.com")
+			require.NoError(t, err)
+			assert.Equal(t, tc.complete, complete)
+			assert.ElementsMatch(t, tc.served, served)
+		})
+	}
+}
+
+func TestNorthstarServerCount(t *testing.T) {
+	assert.Equal(t, 1, dns.NorthstarServerCount(nil))
+	assert.Equal(t, 1, dns.NorthstarServerCount(&config.ClusterConfig{Nodes: map[string]config.Config{"node1": {}}}))
+	assert.Equal(t, 3, dns.NorthstarServerCount(cluster("node1", "node2", "node3")))
+}

spinifex/domains/dns/reload.go

@@ -0,0 +1,46 @@
+package dns
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/nats-io/nats.go"
+)
+
+// RequestZoneReload asks every northstar server to reload zone and collects
+// their acknowledgements until want have arrived or ctx ends. The acks gathered
+// so far are returned either way; the error reports only a transport failure.
+func RequestZoneReload(ctx context.Context, nc *nats.Conn, zone string, want int) ([]ZoneReloadAck, error) {
+	payload, err := json.Marshal(ZoneReload{Zone: zone})
+	if err != nil {
+		return nil, fmt.Errorf("marshal zone reload: %w", err)
+	}
+	inbox := nc.NewInbox()
+	sub, err := nc.SubscribeSync(inbox)
+	if err != nil {
+		return nil, fmt.Errorf("subscribe reload acks: %w", err)
+	}
+	defer func() { _ = sub.Unsubscribe() }()
+	if err := nc.PublishRequest(SubjectZoneReload, inbox, payload); err != nil {
+		return nil, fmt.Errorf("publish zone reload: %w", err)
+	}
+
+	var acks []ZoneReloadAck
+	for len(acks) < want {
+		msg, err := sub.NextMsgWithContext(ctx)
+		if err != nil {
+			if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
+				return acks, nil
+			}
+			return acks, fmt.Errorf("receive reload ack: %w", err)
+		}
+		var ack ZoneReloadAck
+		if json.Unmarshal(msg.Data, &ack) != nil || ack.Zone != zone {
+			continue
+		}
+		acks = append(acks, ack)
+	}
+	return acks, nil
+}

spinifex/domains/dns/reload_test.go

@@ -0,0 +1,87 @@
+package dns_test
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/dns"
+)
+
+// ackAs subscribes a fake northstar that answers every reload with the acks
+// reply returns for the requested zone.
+func ackAs(t *testing.T, nc *nats.Conn, reply func(zone string) []dns.ZoneReloadAck) {
+	t.Helper()
+	sub, err := nc.Subscribe(dns.SubjectZoneReload, func(msg *nats.Msg) {
+		var evt dns.ZoneReload
+		require.NoError(t, json.Unmarshal(msg.Data, &evt))
+		for _, ack := range reply(evt.Zone) {
+			body, err := json.Marshal(ack)
+			require.NoError(t, err)
+			require.NoError(t, msg.Respond(body))
+		}
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+}
+
+func TestRequestZoneReloadCollectsOneAckPerServer(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	for _, node := range []string{"node1", "node2", "node3"} {
+		ackAs(t, nc, func(zone string) []dns.ZoneReloadAck {
+			return []dns.ZoneReloadAck{{Zone: zone, Node: node, ETag: "v2"}}
+		})
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	acks, err := dns.RequestZoneReload(ctx, nc, "example.com", 3)
+	require.NoError(t, err)
+	require.Len(t, acks, 3)
+	nodes := []string{acks[0].Node, acks[1].Node, acks[2].Node}
+	assert.ElementsMatch(t, []string{"node1", "node2", "node3"}, nodes)
+	for _, ack := range acks {
+		assert.Equal(t, "v2", ack.ETag)
+	}
+}
+
+func TestRequestZoneReloadIgnoresAcksForOtherZones(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	ackAs(t, nc, func(zone string) []dns.ZoneReloadAck {
+		return []dns.ZoneReloadAck{{Zone: "other.com", Node: "stray"}, {Zone: zone, Node: "node1"}}
+	})
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	acks, err := dns.RequestZoneReload(ctx, nc, "example.com", 1)
+	require.NoError(t, err)
+	require.Len(t, acks, 1)
+	assert.Equal(t, "node1", acks[0].Node)
+}
+
+func TestRequestZoneReloadReturnsPartialAcksAtDeadline(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	ackAs(t, nc, func(zone string) []dns.ZoneReloadAck {
+		return []dns.ZoneReloadAck{{Zone: zone, Node: "node1", Error: "s3 unavailable"}}
+	})
+
+	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
+	defer cancel()
+	acks, err := dns.RequestZoneReload(ctx, nc, "example.com", 2)
+	require.NoError(t, err)
+	require.Len(t, acks, 1)
+	assert.Equal(t, "s3 unavailable", acks[0].Error)
+}
+
+func TestRequestZoneReloadFailsOnClosedConnection(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	nc.Close()
+	_, err := dns.RequestZoneReload(context.Background(), nc, "example.com", 1)
+	require.Error(t, err)
+}

spinifex/domains/dns/reserved.go

@@ -0,0 +1,44 @@
+package dns
+
+import (
+	"regexp"
+	"strings"
+)
+
+// managedLabels are the labels directly below the base domain under which the
+// platform writes its own records.
+var managedLabels = map[string]bool{"compute": true, "elb": true, "eks": true, "rds": true}
+
+var nameserverLabel = regexp.MustCompile(`^ns[0-9]+$`)
+
+// ReservedZoneName reports whether a tenant may not create a hosted zone named
+// name. It refuses the base and internal domains, any ancestor of them, every
+// name below the internal domain, and any name below the base domain that falls
+// under a platform-managed label or a published nameserver.
+func ReservedZoneName(name, baseDomain, internalDomain string) bool {
+	name = normalizeZone(name)
+	if name == "" {
+		return true
+	}
+	for _, zone := range []string{normalizeZone(baseDomain), normalizeZone(privateZoneOrDefault(internalDomain))} {
+		if zone == "" {
+			continue
+		}
+		if name == zone || strings.HasSuffix(zone, "."+name) {
+			return true
+		}
+	}
+	if internal := normalizeZone(privateZoneOrDefault(internalDomain)); strings.HasSuffix(name, "."+internal) {
+		return true
+	}
+	if base := normalizeZone(baseDomain); base != "" && strings.HasSuffix(name, "."+base) {
+		labels := strings.Split(strings.TrimSuffix(name, "."+base), ".")
+		top := labels[len(labels)-1]
+		return managedLabels[top] || nameserverLabel.MatchString(top)
+	}
+	return false
+}
+
+func normalizeZone(name string) string {
+	return strings.TrimSuffix(strings.ToLower(strings.TrimSpace(name)), ".")
+}

spinifex/domains/dns/reserved_test.go

@@ -0,0 +1,46 @@
+package dns_test
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/dns"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestReservedZoneName(t *testing.T) {
+	cases := []struct {
+		name     string
+		reserved bool
+	}{
+		{"spx3.net.", true},
+		{"SPX3.NET", true},
+		{"net", true},
+		{"compute.internal", true},
+		{"ip-10-0-0-1.us-west-1.compute.internal.", true},
+		{"internal", true},
+		{"compute.spx3.net", true},
+		{"ec2-1-2-3-4.us-west-1.compute.spx3.net", true},
+		{"lb.us-west-1.elb.spx3.net", true},
+		{"x.eks.spx3.net", true},
+		{"db.rds.spx3.net", true},
+		{"ns1.spx3.net", true},
+		{"a.ns4.spx3.net", true},
+		{"app.spx3.net", false},
+		{"nsx.spx3.net", false},
+		{"compute.app.spx3.net", false},
+		{"example.com", false},
+		{"compute.example.com", false},
+		{"", true},
+	}
+	for _, c := range cases {
+		assert.Equal(t, c.reserved, dns.ReservedZoneName(c.name, "spx3.net", ""), c.name)
+	}
+}
+
+func TestReservedZoneNameHonoursTheConfiguredDomains(t *testing.T) {
+	assert.True(t, dns.ReservedZoneName("cloud.example.com", "cloud.example.com", "corp.internal"))
+	assert.True(t, dns.ReservedZoneName("example.com", "cloud.example.com", "corp.internal"))
+	assert.True(t, dns.ReservedZoneName("host.corp.internal", "cloud.example.com", "corp.internal"))
+	assert.False(t, dns.ReservedZoneName("spx3.net", "cloud.example.com", "corp.internal"))
+	assert.False(t, dns.ReservedZoneName("app.spx3.net", "", ""))
+}

spinifex/domains/dns/types.go

@@ -67,3 +67,13 @@ type ChangeResult struct {
 type ZoneReload struct {
 	Zone string `json:"zone"`
 }
+
+// ZoneReloadAck is one northstar server's reply to a ZoneReload sent with a
+// reply subject. ETag is the version of the zone it now serves, "" when the
+// zone is absent; Error is set when the reload failed.
+type ZoneReloadAck struct {
+	Zone  string `json:"zone"`
+	Node  string `json:"node"`
+	ETag  string `json:"etag,omitempty"`
+	Error string `json:"error,omitempty"`
+}

spinifex/domains/dns/writer.go

@@ -19,14 +19,16 @@ import (
 // of zone TOML files in s3://northstar/ using the system predastore credentials.
 // Only the reconcile's elected node calls it, so a zone has one writer at a time.
 type Writer struct {
-	enabled      bool
-	s3cfg        *nsconfig.S3Config
-	baseDomain   string
-	nameservers  []nsconfig.NameserverSeed
-	ttl          uint32
-	nc           *nats.Conn
-	quotaEnabled bool
-	quotas       Quotas
+	enabled    bool
+	s3cfg      *nsconfig.S3Config
+	baseDomain string
+	// internalDomain is the private zone, for refusing reserved delegations.
+	internalDomain string
+	nameservers    []nsconfig.NameserverSeed
+	ttl            uint32
+	nc             *nats.Conn
+	quotaEnabled   bool
+	quotas         Quotas
 }
 
 // NewWriter resolves the northstar S3 endpoint/bucket from the node's
@@ -45,6 +47,7 @@ func NewWriter(cfg *config.Config, cluster *config.ClusterConfig, nc *nats.Conn)
 	w.enabled = true
 	w.s3cfg = zoneCfg.s3
 	w.baseDomain = strings.TrimSpace(zoneCfg.server.DefaultDomain)
+	w.internalDomain = ResolveInternalDomain(cfg)
 	// Share the bootstrap's derivation so a zone this writer materialises carries
 	// the same NS topology as one the bootstrap seeds; whichever wins the race,
 	// the zone is correct.

spinifex/domains/ec2/account/seal.go

@@ -0,0 +1,135 @@
+package account
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"log/slog"
+
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// sealMaxAttempts bounds SealAccount's retry loop against a revision that
+// keeps moving under concurrent writes, so it fails closed instead of
+// spinning.
+const sealMaxAttempts = 5
+
+// sealedRecord is the fixed value a seal writes. Clearing both booleans
+// retains no tenant setting; an older reader that ignores state sees the
+// AWS defaults.
+var sealedRecord = &AccountSettingsRecord{State: AccountSettingsStateSealed}
+
+// AccountSettingsSeal is SealAccount's result: the revision at which the
+// stream leader accepted the sealed value.
+type AccountSettingsSeal struct {
+	Revision uint64
+}
+
+// AccountTerminationChecker reports whether IAM holds accountID as TERMINATING.
+// A missing record is (false, nil); only a failed read is an error.
+type AccountTerminationChecker interface {
+	AccountTerminating(ctx context.Context, accountID string) (bool, error)
+}
+
+// SetAccountTerminationChecker wires the precondition; until it is set, SealAccount refuses.
+func (s *AccountSettingsServiceImpl) SetAccountTerminationChecker(c AccountTerminationChecker) {
+	s.terminationChecker = c
+}
+
+// checkTerminating runs before any settings-KV call, so a refusal never writes.
+// Not TERMINATING, including a missing IAM record, is IncorrectState; a read
+// failure or an unwired checker is InternalError.
+func (s *AccountSettingsServiceImpl) checkTerminating(ctx context.Context, accountID string) error {
+	if s.terminationChecker == nil {
+		slog.ErrorContext(ctx, "Account settings seal has no termination checker wired", "accountID", accountID)
+		return errors.New(awserrors.ErrorInternalError)
+	}
+
+	terminating, err := s.terminationChecker.AccountTerminating(ctx, accountID)
+	if err != nil {
+		slog.ErrorContext(ctx, "Failed to read account status for seal precondition", "accountID", accountID, "err", err)
+		return errors.New(awserrors.ErrorInternalError)
+	}
+	if !terminating {
+		return awserrors.Errorf(awserrors.ErrorIncorrectState, "account %s is not TERMINATING", accountID)
+	}
+	return nil
+}
+
+// isRevisionConflict reports a lost CAS race SealAccount should retry. Besides
+// the two sentinels, Create's retry against a delete marker returns a bare
+// *jetstream.APIError carrying a wrong-last-sequence code.
+func isRevisionConflict(err error) bool {
+	if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
+		return true
+	}
+	if apiErr, ok := errors.AsType[*jetstream.APIError](err); ok {
+		return apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence ||
+			apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequenceConstant
+	}
+	return false
+}
+
+// SealAccount seals accountID's settings record so no later mutation can
+// recreate it. It never relies on a read for its claim: the only success is a
+// leader-accepted write, evidenced by the returned revision.
+func (s *AccountSettingsServiceImpl) SealAccount(ctx context.Context, accountID string) (*AccountSettingsSeal, error) {
+	if accountID == "" {
+		return nil, awserrors.Errorf(awserrors.ErrorMissingParameter, "accountID is required to seal settings")
+	}
+	if !awsidentifiers.IsAccountID(accountID) {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "accountID %q is not a valid AWS account ID", accountID)
+	}
+	if accountID == awsidentifiers.GlobalAccountID {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "settings seal must not target the global account")
+	}
+
+	if err := s.checkTerminating(ctx, accountID); err != nil {
+		return nil, err
+	}
+
+	data, err := json.Marshal(sealedRecord)
+	if err != nil {
+		slog.ErrorContext(ctx, "Failed to encode sealed account settings record", "accountID", accountID, "err", err)
+		return nil, errors.New(awserrors.ErrorInternalError)
+	}
+
+	// settingsKey is never used here: a seal must act on exactly accountID
+	// and never fall back to the global key.
+	for range sealMaxAttempts {
+		var rev uint64
+		entry, getErr := s.settingsKV.Get(ctx, accountID)
+		switch {
+		case getErr == nil:
+			rev = entry.Revision()
+		case errors.Is(getErr, jetstream.ErrKeyNotFound):
+			rev = 0
+		default:
+			slog.ErrorContext(ctx, "Failed to read account settings for seal", "accountID", accountID, "err", getErr)
+			return nil, errors.New(awserrors.ErrorInternalError)
+		}
+
+		var newRev uint64
+		var writeErr error
+		if rev == 0 {
+			newRev, writeErr = s.settingsKV.Create(ctx, accountID, data)
+		} else {
+			newRev, writeErr = s.settingsKV.Update(ctx, accountID, data, rev)
+		}
+
+		switch {
+		case writeErr == nil:
+			return &AccountSettingsSeal{Revision: newRev}, nil
+		case isRevisionConflict(writeErr):
+			continue
+		default:
+			slog.ErrorContext(ctx, "Failed to write sealed account settings record", "accountID", accountID, "err", writeErr)
+			return nil, errors.New(awserrors.ErrorInternalError)
+		}
+	}
+
+	slog.ErrorContext(ctx, "Account settings seal exhausted attempts against concurrent writes", "accountID", accountID, "attempts", sealMaxAttempts)
+	return nil, errors.New(awserrors.ErrorInternalError)
+}

spinifex/domains/ec2/account/seal_test.go

@@ -0,0 +1,559 @@
+package account
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sync"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// stubTerminationChecker answers AccountTerminating from a fixed set, for
+// tests that exercise the KV paths rather than IAM itself.
+type stubTerminationChecker struct {
+	terminating map[string]bool
+	err         error
+}
+
+func (c *stubTerminationChecker) AccountTerminating(_ context.Context, accountID string) (bool, error) {
+	if c.err != nil {
+		return false, c.err
+	}
+	return c.terminating[accountID], nil
+}
+
+// terminatingChecker answers true for exactly accountID.
+func terminatingChecker(accountID string) *stubTerminationChecker {
+	return &stubTerminationChecker{terminating: map[string]bool{accountID: true}}
+}
+
+// lastSettingsSeq reports the account-settings stream's last sequence, so a
+// test can assert a failed attempt wrote nothing.
+func lastSettingsSeq(t *testing.T, svc *AccountSettingsServiceImpl) uint64 {
+	t.Helper()
+	stream, err := svc.js.Stream(context.Background(), "KV_"+KVBucketAccountSettings)
+	require.NoError(t, err)
+	info, err := stream.Info(context.Background())
+	require.NoError(t, err)
+	return info.State.LastSeq
+}
+
+// mutationCase is one of the four settings mutations, called uniformly so a
+// test can table over them.
+type mutationCase struct {
+	name string
+	call func(svc *AccountSettingsServiceImpl, ctx context.Context) error
+}
+
+func mutationTable() []mutationCase {
+	return []mutationCase{
+		{"EnableEbs", func(svc *AccountSettingsServiceImpl, ctx context.Context) error {
+			_, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
+			return err
+		}},
+		{"DisableEbs", func(svc *AccountSettingsServiceImpl, ctx context.Context) error {
+			_, err := svc.DisableEbsEncryptionByDefault(ctx, &ec2.DisableEbsEncryptionByDefaultInput{}, testAccountID)
+			return err
+		}},
+		{"EnableSerial", func(svc *AccountSettingsServiceImpl, ctx context.Context) error {
+			_, err := svc.EnableSerialConsoleAccess(ctx, &ec2.EnableSerialConsoleAccessInput{}, testAccountID)
+			return err
+		}},
+		{"DisableSerial", func(svc *AccountSettingsServiceImpl, ctx context.Context) error {
+			_, err := svc.DisableSerialConsoleAccess(ctx, &ec2.DisableSerialConsoleAccessInput{}, testAccountID)
+			return err
+		}},
+	}
+}
+
+// seedSettingsState prepares testAccountID's record for a race test: present
+// (an active record), absent (never written) or tombstone (written then
+// deleted, leaving a delete marker Get hides).
+func seedSettingsState(t *testing.T, svc *AccountSettingsServiceImpl, state string) {
+	t.Helper()
+	ctx := context.Background()
+	switch state {
+	case "present":
+		_, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
+		require.NoError(t, err)
+	case "tombstone":
+		_, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
+		require.NoError(t, err)
+		require.NoError(t, svc.settingsKV.Delete(ctx, testAccountID))
+	case "absent":
+	default:
+		t.Fatalf("unknown seed state %q", state)
+	}
+}
+
+// sealRaceKV wraps a real KeyValue so a test can inject a deterministic race
+// between a caller's read and its guarded write (Create or Update), instead
+// of relying on goroutine timing.
+type sealRaceKV struct {
+	jetstream.KeyValue
+
+	writeCalls  int
+	beforeWrite func(attempt int) error
+}
+
+func (f *sealRaceKV) Create(ctx context.Context, key string, value []byte, opts ...jetstream.KVCreateOpt) (uint64, error) {
+	f.writeCalls++
+	if f.beforeWrite != nil {
+		if err := f.beforeWrite(f.writeCalls); err != nil {
+			return 0, err
+		}
+	}
+	return f.KeyValue.Create(ctx, key, value, opts...)
+}
+
+func (f *sealRaceKV) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) {
+	f.writeCalls++
+	if f.beforeWrite != nil {
+		if err := f.beforeWrite(f.writeCalls); err != nil {
+			return 0, err
+		}
+	}
+	return f.KeyValue.Update(ctx, key, value, revision)
+}
+
+// staleGetKV always answers Get with a fixed snapshot taken at construction,
+// so a caller's CAS keeps retrying against a revision that can never win.
+type staleGetKV struct {
+	jetstream.KeyValue
+
+	entry jetstream.KeyValueEntry
+}
+
+func (f *staleGetKV) Get(context.Context, string) (jetstream.KeyValueEntry, error) {
+	return f.entry, nil
+}
+
+// staleNotFoundKV always answers Get as absent, regardless of what the real
+// bucket holds.
+type staleNotFoundKV struct {
+	jetstream.KeyValue
+}
+
+func (f *staleNotFoundKV) Get(context.Context, string) (jetstream.KeyValueEntry, error) {
+	return nil, jetstream.ErrKeyNotFound
+}
+
+// errGetKV always fails Get with a fixed store error.
+type errGetKV struct {
+	jetstream.KeyValue
+
+	err error
+}
+
+func (f *errGetKV) Get(context.Context, string) (jetstream.KeyValueEntry, error) {
+	return nil, f.err
+}
+
+func decodeSettings(t *testing.T, entry jetstream.KeyValueEntry) AccountSettingsRecord {
+	t.Helper()
+	var rec AccountSettingsRecord
+	require.NoError(t, json.Unmarshal(entry.Value(), &rec))
+	return rec
+}
+
+// A seal landing between a mutation's read and its guarded write is not
+// outlived: the mutation's retry sees the seal and is rejected, and nothing
+// it wrote survives.
+func TestSealRaceMutation_SealLandsBetweenReadAndWrite(t *testing.T) {
+	for _, m := range mutationTable() {
+		for _, state := range []string{"present", "absent", "tombstone"} {
+			t.Run(m.name+"_"+state, func(t *testing.T) {
+				svc := setupTestAccountService(t)
+				ctx := context.Background()
+				seedSettingsState(t, svc, state)
+
+				realKV := svc.settingsKV
+				sealer := &AccountSettingsServiceImpl{js: svc.js, settingsKV: realKV}
+				sealer.SetAccountTerminationChecker(terminatingChecker(testAccountID))
+
+				var sealed *AccountSettingsSeal
+				race := &sealRaceKV{KeyValue: realKV}
+				race.beforeWrite = func(attempt int) error {
+					if attempt != 1 {
+						return nil
+					}
+					var sealErr error
+					sealed, sealErr = sealer.SealAccount(ctx, testAccountID)
+					return sealErr
+				}
+				svc.settingsKV = race
+
+				err := m.call(svc, ctx)
+				require.Error(t, err)
+				assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked), "got %v", err)
+				require.NotNil(t, sealed)
+
+				entry, err := realKV.Get(ctx, testAccountID)
+				require.NoError(t, err)
+				assert.Equal(t, AccountSettingsStateSealed, decodeSettings(t, entry).State)
+				assert.Equal(t, sealed.Revision, entry.Revision(), "no write landed after the seal")
+			})
+		}
+	}
+}
+
+// A mutation landing inside a seal's read-write window does not survive it:
+// the seal retries against the mutation's write and wins, leaving the
+// sealed value with both booleans false.
+func TestSealRaceMutation_MutationLandsBetweenSealReadAndWrite(t *testing.T) {
+	for _, m := range mutationTable() {
+		for _, state := range []string{"present", "absent", "tombstone"} {
+			t.Run(m.name+"_"+state, func(t *testing.T) {
+				svc := setupTestAccountService(t)
+				ctx := context.Background()
+				seedSettingsState(t, svc, state)
+
+				realKV := svc.settingsKV
+				race := &sealRaceKV{KeyValue: realKV}
+				race.beforeWrite = func(attempt int) error {
+					if attempt != 1 {
+						return nil
+					}
+					_ = m.call(svc, ctx)
+					return nil
+				}
+
+				sealer := &AccountSettingsServiceImpl{js: svc.js, settingsKV: race}
+				sealer.SetAccountTerminationChecker(terminatingChecker(testAccountID))
+
+				seal, err := sealer.SealAccount(ctx, testAccountID)
+				require.NoError(t, err)
+
+				entry, err := realKV.Get(ctx, testAccountID)
+				require.NoError(t, err)
+				rec := decodeSettings(t, entry)
+				assert.Equal(t, AccountSettingsStateSealed, rec.State)
+				assert.False(t, rec.EbsEncryptionByDefault)
+				assert.False(t, rec.SerialConsoleAccess)
+				assert.Equal(t, seal.Revision, entry.Revision())
+			})
+		}
+	}
+}
+
+// None of the four mutations can act on an already-sealed record.
+func TestMutationOnSealedRecordIsRejected(t *testing.T) {
+	for _, m := range mutationTable() {
+		t.Run(m.name, func(t *testing.T) {
+			svc := setupTestAccountService(t)
+			ctx := context.Background()
+			svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
+			_, err := svc.SealAccount(ctx, testAccountID)
+			require.NoError(t, err)
+
+			before, err := svc.settingsKV.Get(ctx, testAccountID)
+			require.NoError(t, err)
+
+			err = m.call(svc, ctx)
+			require.Error(t, err)
+			assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked), "got %v", err)
+
+			after, err := svc.settingsKV.Get(ctx, testAccountID)
+			require.NoError(t, err)
+			assert.Equal(t, before.Revision(), after.Revision())
+		})
+	}
+}
+
+// Every settings mutation races one seal on a real bucket; afterwards the
+// record is sealed and stays sealed, so no success among them could have
+// landed after the seal.
+func TestSealConcurrentMutations(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+	svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
+	_, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
+	require.NoError(t, err)
+
+	mutations := mutationTable()
+	errs := make([]error, len(mutations))
+	var wg sync.WaitGroup
+	for i, m := range mutations {
+		wg.Add(1)
+		go func(i int, m mutationCase) {
+			defer wg.Done()
+			errs[i] = m.call(svc, ctx)
+		}(i, m)
+	}
+
+	var sealResult *AccountSettingsSeal
+	var sealErr error
+	wg.Go(func() {
+		sealResult, sealErr = svc.SealAccount(ctx, testAccountID)
+	})
+	wg.Wait()
+
+	require.NoError(t, sealErr)
+	require.NotNil(t, sealResult)
+	for i, err := range errs {
+		if err != nil {
+			assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked), "mutation %s: got %v", mutations[i].name, err)
+		}
+	}
+
+	entry, err := svc.settingsKV.Get(ctx, testAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, AccountSettingsStateSealed, decodeSettings(t, entry).State)
+	assert.Equal(t, sealResult.Revision, entry.Revision())
+
+	_, err = svc.GetEbsEncryptionByDefault(ctx, &ec2.GetEbsEncryptionByDefaultInput{}, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
+}
+
+// A Get pinned to a revision the record has already moved past exhausts the
+// mutation's retries without ever writing.
+func TestMutationStaleReadExhaustsWithoutWriting(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+
+	_, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
+	require.NoError(t, err)
+	stale, err := svc.settingsKV.Get(ctx, testAccountID)
+	require.NoError(t, err)
+
+	_, err = svc.DisableEbsEncryptionByDefault(ctx, &ec2.DisableEbsEncryptionByDefaultInput{}, testAccountID)
+	require.NoError(t, err)
+
+	before := lastSettingsSeq(t, svc)
+	svc.settingsKV = &staleGetKV{KeyValue: svc.settingsKV, entry: stale}
+
+	_, err = svc.EnableSerialConsoleAccess(ctx, &ec2.EnableSerialConsoleAccessInput{}, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
+	assert.Equal(t, before, lastSettingsSeq(t, svc))
+}
+
+// A Get that always reports absent, while a sealed record actually exists,
+// never gets to recreate it: every Create attempt conflicts.
+func TestMutationStaleAbsentDoesNotRecreate(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+	svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
+	_, err := svc.SealAccount(ctx, testAccountID)
+	require.NoError(t, err)
+
+	before := lastSettingsSeq(t, svc)
+	svc.settingsKV = &staleNotFoundKV{KeyValue: svc.settingsKV}
+
+	_, err = svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
+	assert.Equal(t, before, lastSettingsSeq(t, svc))
+}
+
+// A revision that keeps moving under sustained concurrent writes must not
+// let the seal spin forever: it gives up, bounded by its attempt budget,
+// and its own write never lands.
+func TestSealSustainedConflictFailsWithoutWriting(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+	svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
+	_, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
+	require.NoError(t, err)
+
+	realKV := svc.settingsKV
+	race := &sealRaceKV{KeyValue: realKV}
+	race.beforeWrite = func(int) error {
+		data, merr := json.Marshal(&AccountSettingsRecord{SerialConsoleAccess: true})
+		require.NoError(t, merr)
+		_, perr := realKV.Put(ctx, testAccountID, data)
+		return perr
+	}
+	svc.settingsKV = race
+
+	_, err = svc.SealAccount(ctx, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
+	assert.Equal(t, sealMaxAttempts, race.writeCalls)
+
+	entry, err := realKV.Get(ctx, testAccountID)
+	require.NoError(t, err)
+	assert.NotEqual(t, AccountSettingsStateSealed, decodeSettings(t, entry).State, "the seal's own write never landed")
+}
+
+// A store error on the seal's read reports failed, never written.
+func TestSealStoreErrorWritesNothing(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+	svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
+
+	before := lastSettingsSeq(t, svc)
+	svc.settingsKV = &errGetKV{KeyValue: svc.settingsKV, err: errors.New("kv unavailable")}
+
+	_, err := svc.SealAccount(ctx, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
+	assert.Equal(t, before, lastSettingsSeq(t, svc))
+}
+
+func TestIsRevisionConflictRecognisesTombstoneRetry(t *testing.T) {
+	assert.True(t, isRevisionConflict(jetstream.ErrKeyExists))
+	assert.True(t, isRevisionConflict(jetstream.ErrKeyRevisionMismatch))
+	assert.True(t, isRevisionConflict(&jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequence}))
+	assert.True(t, isRevisionConflict(&jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequenceConstant}))
+	assert.True(t, isRevisionConflict(fmt.Errorf("wrapped: %w", &jetstream.APIError{ErrorCode: jetstream.JSErrCodeStreamWrongLastSequence})))
+	assert.False(t, isRevisionConflict(errors.New("unrelated failure")))
+	assert.False(t, isRevisionConflict(nil))
+}
+
+// An account IAM does not hold as TERMINATING refuses the seal before any
+// settings-KV write, whether the key is present or absent.
+func TestSealRefusedForActiveAccount(t *testing.T) {
+	for _, state := range []string{"present", "absent"} {
+		t.Run(state, func(t *testing.T) {
+			svc := setupTestAccountService(t)
+			ctx := context.Background()
+			if state == "present" {
+				_, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
+				require.NoError(t, err)
+			}
+			svc.SetAccountTerminationChecker(&stubTerminationChecker{terminating: map[string]bool{}})
+
+			before := lastSettingsSeq(t, svc)
+			_, err := svc.SealAccount(ctx, testAccountID)
+			assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorIncorrectState), "got %v", err)
+			assert.Equal(t, before, lastSettingsSeq(t, svc))
+		})
+	}
+}
+
+func TestSealRefusedForMissingIAMRecord(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+	svc.SetAccountTerminationChecker(&stubTerminationChecker{terminating: map[string]bool{}})
+
+	before := lastSettingsSeq(t, svc)
+	_, err := svc.SealAccount(ctx, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorIncorrectState), "got %v", err)
+	assert.Equal(t, before, lastSettingsSeq(t, svc))
+}
+
+func TestSealRefusedWhenIAMReadFails(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+	svc.SetAccountTerminationChecker(&stubTerminationChecker{err: errors.New("iam unavailable")})
+
+	before := lastSettingsSeq(t, svc)
+	_, err := svc.SealAccount(ctx, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
+	assert.Equal(t, before, lastSettingsSeq(t, svc))
+}
+
+func TestSealRefusedWithNoCheckerWired(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+
+	before := lastSettingsSeq(t, svc)
+	_, err := svc.SealAccount(ctx, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
+	assert.Equal(t, before, lastSettingsSeq(t, svc))
+}
+
+func TestSealAcceptedForTerminatingAccount(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+	svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
+
+	seal, err := svc.SealAccount(ctx, testAccountID)
+	require.NoError(t, err)
+	assert.NotZero(t, seal.Revision)
+
+	entry, err := svc.settingsKV.Get(ctx, testAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, seal.Revision, entry.Revision())
+}
+
+// A sealed record stays sealed after the service is rebuilt over the same
+// JetStream, and every one of the six settings actions still refuses it.
+func TestSealSurvivesServiceReconstruction(t *testing.T) {
+	_, nc, _ := testutil.StartTestJetStream(t)
+	ctx := context.Background()
+
+	svc, err := NewAccountSettingsServiceImplWithNATS(ctx, nil, nc)
+	require.NoError(t, err)
+	svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
+	first, err := svc.SealAccount(ctx, testAccountID)
+	require.NoError(t, err)
+
+	rebuilt, err := NewAccountSettingsServiceImplWithNATS(ctx, nil, nc)
+	require.NoError(t, err)
+
+	entry, err := rebuilt.settingsKV.Get(ctx, testAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, AccountSettingsStateSealed, decodeSettings(t, entry).State)
+
+	_, err = rebuilt.GetEbsEncryptionByDefault(ctx, &ec2.GetEbsEncryptionByDefaultInput{}, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
+	_, err = rebuilt.GetSerialConsoleAccessStatus(ctx, &ec2.GetSerialConsoleAccessStatusInput{}, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
+	_, err = rebuilt.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
+	_, err = rebuilt.DisableEbsEncryptionByDefault(ctx, &ec2.DisableEbsEncryptionByDefaultInput{}, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
+	_, err = rebuilt.EnableSerialConsoleAccess(ctx, &ec2.EnableSerialConsoleAccessInput{}, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
+	_, err = rebuilt.DisableSerialConsoleAccess(ctx, &ec2.DisableSerialConsoleAccessInput{}, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorBlocked))
+
+	rebuilt.SetAccountTerminationChecker(terminatingChecker(testAccountID))
+	second, err := rebuilt.SealAccount(ctx, testAccountID)
+	require.NoError(t, err)
+	assert.Greater(t, second.Revision, first.Revision)
+}
+
+func TestSealAbsentKeyCreatesSealedRecord(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+	svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
+
+	_, err := svc.settingsKV.Get(ctx, testAccountID)
+	require.ErrorIs(t, err, jetstream.ErrKeyNotFound)
+
+	seal, err := svc.SealAccount(ctx, testAccountID)
+	require.NoError(t, err)
+
+	entry, err := svc.settingsKV.Get(ctx, testAccountID)
+	require.NoError(t, err)
+	assert.Equal(t, seal.Revision, entry.Revision())
+	assert.Equal(t, AccountSettingsStateSealed, decodeSettings(t, entry).State)
+}
+
+func TestSealRefusesEmptyInvalidAndGlobalAccount(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+
+	_, err := svc.SealAccount(ctx, "")
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorMissingParameter), "got %v", err)
+
+	_, err = svc.SealAccount(ctx, "not-an-account")
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidParameterValue), "got %v", err)
+
+	_, err = svc.SealAccount(ctx, awsidentifiers.GlobalAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidParameterValue), "got %v", err)
+}
+
+func TestSealLeavesOtherAccountsUntouched(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+	accountB := "222222222222"
+	_, err := svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, accountB)
+	require.NoError(t, err)
+
+	svc.SetAccountTerminationChecker(terminatingChecker(testAccountID))
+	_, err = svc.SealAccount(ctx, testAccountID)
+	require.NoError(t, err)
+
+	out, err := svc.GetEbsEncryptionByDefault(ctx, &ec2.GetEbsEncryptionByDefaultInput{}, accountB)
+	require.NoError(t, err)
+	assert.True(t, *out.EbsEncryptionByDefault)
+}

spinifex/domains/ec2/account/service_impl.go

@@ -5,12 +5,13 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"log/slog"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
@@ -20,14 +21,19 @@ import (
 const (
 	KVBucketAccountSettings        = "spinifex-ec2-account-settings"
 	KVBucketAccountSettingsVersion = 1
-	KeyEbsEncryptionDefault        = "ebs-encryption-default"
-	KeySerialConsoleAccess         = "serial-console-access"
 )
 
+// AccountSettingsState is a settings record's lifecycle state. "" is active;
+// a record written before the state field existed has none and stays active.
+type AccountSettingsState string
+
+const AccountSettingsStateSealed AccountSettingsState = "sealed"
+
 // AccountSettingsRecord represents stored account settings.
 type AccountSettingsRecord struct {
-	EbsEncryptionByDefault bool `json:"ebs_encryption_by_default"`
-	SerialConsoleAccess    bool `json:"serial_console_access"`
+	EbsEncryptionByDefault bool                 `json:"ebs_encryption_by_default"`
+	SerialConsoleAccess    bool                 `json:"serial_console_access"`
+	State                  AccountSettingsState `json:"state,omitempty"`
 }
 
 // AccountSettingsServiceImpl implements account settings operations with NATS JetStream persistence.
@@ -35,6 +41,10 @@ type AccountSettingsServiceImpl struct {
 	config     *config.Config
 	js         jetstream.JetStream
 	settingsKV jetstream.KeyValue
+
+	// terminationChecker is the seal precondition; SealAccount refuses until
+	// it is wired.
+	terminationChecker AccountTerminationChecker
 }
 
 var _ AccountSettingsService = (*AccountSettingsServiceImpl)(nil)
@@ -72,50 +82,99 @@ func settingsKey(accountID string) string {
 	return accountID
 }
 
-// getSettings retrieves current account settings.
+// errSettingsSealed marks a mutation or read that found a sealed record, so
+// callers map it to Blocked without logging it as a fault.
+var errSettingsSealed = errors.New(awserrors.ErrorBlocked)
+
+// errSettingsContended marks a mutation whose CAS retries were exhausted
+// under concurrent writers, bounding the retry loop instead of spinning.
+var errSettingsContended = errors.New("account settings: CAS retries exhausted")
+
+// checkActive rejects a sealed or unknown-state record; "" is active and
+// passes through. A mutation must never act on a record that failed this.
+func checkActive(rec *AccountSettingsRecord) error {
+	switch rec.State {
+	case "":
+		return nil
+	case AccountSettingsStateSealed:
+		return errSettingsSealed
+	default:
+		return fmt.Errorf("account settings: unknown state %q", rec.State)
+	}
+}
+
+// mapSettingsError turns a mutateSettings or getSettings failure into the AWS
+// error a caller returns: sealed is an expected, unlogged Blocked; everything
+// else is InternalError, logged with the account ID.
+func mapSettingsError(ctx context.Context, accountID string, err error) error {
+	switch {
+	case err == nil:
+		return nil
+	case errors.Is(err, errSettingsSealed):
+		return errors.New(awserrors.ErrorBlocked)
+	case errors.Is(err, errSettingsContended):
+		slog.ErrorContext(ctx, "Account settings CAS retries exhausted under contention", "accountID", accountID)
+	case errors.Is(err, kvutil.ErrRead):
+		slog.ErrorContext(ctx, "Failed to read account settings from KV", "accountID", accountID, "err", err)
+	case errors.Is(err, kvutil.ErrDecode):
+		slog.ErrorContext(ctx, "Corrupt account settings record in KV", "accountID", accountID, "err", err)
+	case errors.Is(err, kvutil.ErrEncode):
+		slog.ErrorContext(ctx, "Failed to marshal account settings record", "accountID", accountID, "err", err)
+	case errors.Is(err, kvutil.ErrWrite):
+		slog.ErrorContext(ctx, "Failed to write account settings to KV", "accountID", accountID, "err", err)
+	default:
+		slog.ErrorContext(ctx, "Account settings operation failed", "accountID", accountID, "err", err)
+	}
+	return errors.New(awserrors.ErrorInternalError)
+}
+
+// mutateSettings applies set to accountID's record at the revision it read.
+// changed is always true, so every success is a leader-accepted write over an
+// active value; a seal landing between the read and the write fails the CAS.
+func (s *AccountSettingsServiceImpl) mutateSettings(ctx context.Context, accountID string, set func(*AccountSettingsRecord)) error {
+	_, err := kvutil.Update(ctx, s.settingsKV, settingsKey(accountID), kvutil.CASConfig{
+		CreateIfAbsent: true,
+		Exhausted:      func(string, int) error { return errSettingsContended },
+	}, func(rec *AccountSettingsRecord) (bool, error) {
+		if err := checkActive(rec); err != nil {
+			return false, err
+		}
+		set(rec)
+		return true, nil
+	})
+	return mapSettingsError(ctx, accountID, err)
+}
+
+// getSettings retrieves current account settings, rejecting a sealed or
+// unknown-state record so a read never answers from behind a seal. Absent is
+// the AWS defaults; a read is a direct get and can be stale.
 func (s *AccountSettingsServiceImpl) getSettings(ctx context.Context, accountID string) (*AccountSettingsRecord, error) {
 	entry, err := s.settingsKV.Get(ctx, settingsKey(accountID))
 	if err != nil {
 		if errors.Is(err, jetstream.ErrKeyNotFound) {
-			return &AccountSettingsRecord{
-				EbsEncryptionByDefault: false,
-				SerialConsoleAccess:    false,
-			}, nil
+			return &AccountSettingsRecord{}, nil
 		}
-		return nil, fmt.Errorf("failed to get account settings: %w", err)
+		slog.ErrorContext(ctx, "Failed to read account settings from KV", "accountID", accountID, "err", err)
+		return nil, errors.New(awserrors.ErrorInternalError)
 	}
 
 	var settings AccountSettingsRecord
 	if err := json.Unmarshal(entry.Value(), &settings); err != nil {
-		return nil, err
+		slog.ErrorContext(ctx, "Corrupt account settings record in KV", "accountID", accountID, "err", err)
+		return nil, errors.New(awserrors.ErrorInternalError)
+	}
+	if err := checkActive(&settings); err != nil {
+		return nil, mapSettingsError(ctx, accountID, err)
 	}
 
 	return &settings, nil
 }
 
-// saveSettings saves current account settings.
-func (s *AccountSettingsServiceImpl) saveSettings(ctx context.Context, settings *AccountSettingsRecord, accountID string) error {
-	data, err := json.Marshal(settings)
-	if err != nil {
-		return err
-	}
-
-	_, err = s.settingsKV.Put(ctx, settingsKey(accountID), data)
-	return err
-}
-
 // EnableEbsEncryptionByDefault enables EBS encryption by default for the account.
 func (s *AccountSettingsServiceImpl) EnableEbsEncryptionByDefault(ctx context.Context, input *ec2.EnableEbsEncryptionByDefaultInput, accountID string) (*ec2.EnableEbsEncryptionByDefaultOutput, error) {
 	slog.InfoContext(ctx, "EnableEbsEncryptionByDefault called", "accountID", accountID)
 
-	settings, err := s.getSettings(ctx, accountID)
-	if err != nil {
-		return nil, err
-	}
-
-	settings.EbsEncryptionByDefault = true
-
-	if err := s.saveSettings(ctx, settings, accountID); err != nil {
+	if err := s.mutateSettings(ctx, accountID, func(rec *AccountSettingsRecord) { rec.EbsEncryptionByDefault = true }); err != nil {
 		return nil, err
 	}
 
@@ -128,14 +187,7 @@ func (s *AccountSettingsServiceImpl) EnableEbsEncryptionByDefault(ctx context.Co
 func (s *AccountSettingsServiceImpl) DisableEbsEncryptionByDefault(ctx context.Context, input *ec2.DisableEbsEncryptionByDefaultInput, accountID string) (*ec2.DisableEbsEncryptionByDefaultOutput, error) {
 	slog.InfoContext(ctx, "DisableEbsEncryptionByDefault called", "accountID", accountID)
 
-	settings, err := s.getSettings(ctx, accountID)
-	if err != nil {
-		return nil, err
-	}
-
-	settings.EbsEncryptionByDefault = false
-
-	if err := s.saveSettings(ctx, settings, accountID); err != nil {
+	if err := s.mutateSettings(ctx, accountID, func(rec *AccountSettingsRecord) { rec.EbsEncryptionByDefault = false }); err != nil {
 		return nil, err
 	}
 
@@ -176,14 +228,7 @@ func (s *AccountSettingsServiceImpl) GetSerialConsoleAccessStatus(ctx context.Co
 func (s *AccountSettingsServiceImpl) EnableSerialConsoleAccess(ctx context.Context, input *ec2.EnableSerialConsoleAccessInput, accountID string) (*ec2.EnableSerialConsoleAccessOutput, error) {
 	slog.InfoContext(ctx, "EnableSerialConsoleAccess called", "accountID", accountID)
 
-	settings, err := s.getSettings(ctx, accountID)
-	if err != nil {
-		return nil, err
-	}
-
-	settings.SerialConsoleAccess = true
-
-	if err := s.saveSettings(ctx, settings, accountID); err != nil {
+	if err := s.mutateSettings(ctx, accountID, func(rec *AccountSettingsRecord) { rec.SerialConsoleAccess = true }); err != nil {
 		return nil, err
 	}
 
@@ -196,14 +241,7 @@ func (s *AccountSettingsServiceImpl) EnableSerialConsoleAccess(ctx context.Conte
 func (s *AccountSettingsServiceImpl) DisableSerialConsoleAccess(ctx context.Context, input *ec2.DisableSerialConsoleAccessInput, accountID string) (*ec2.DisableSerialConsoleAccessOutput, error) {
 	slog.InfoContext(ctx, "DisableSerialConsoleAccess called", "accountID", accountID)
 
-	settings, err := s.getSettings(ctx, accountID)
-	if err != nil {
-		return nil, err
-	}
-
-	settings.SerialConsoleAccess = false
-
-	if err := s.saveSettings(ctx, settings, accountID); err != nil {
+	if err := s.mutateSettings(ctx, accountID, func(rec *AccountSettingsRecord) { rec.SerialConsoleAccess = false }); err != nil {
 		return nil, err
 	}
 

spinifex/domains/ec2/account/service_impl_test.go

@@ -2,11 +2,13 @@ package account
 
 import (
 	"context"
+	"encoding/json"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/internal/testkit"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -143,3 +145,44 @@ func TestSettingsKey_EmptyAccountIDFallback(t *testing.T) {
 	assert.Equal(t, awsidentifiers.GlobalAccountID, settingsKey(""))
 	assert.Equal(t, "123456789012", settingsKey("123456789012"))
 }
+
+// A raw pre-seal record, with no state field at all, is readable and
+// mutable, and a mutation writes it back without adding one.
+func TestRecordWithoutStateIsActive(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+	_, err := svc.settingsKV.Put(ctx, testAccountID, []byte(`{"ebs_encryption_by_default":true,"serial_console_access":false}`))
+	require.NoError(t, err)
+
+	out, err := svc.GetEbsEncryptionByDefault(ctx, &ec2.GetEbsEncryptionByDefaultInput{}, testAccountID)
+	require.NoError(t, err)
+	assert.True(t, *out.EbsEncryptionByDefault)
+
+	_, err = svc.EnableSerialConsoleAccess(ctx, &ec2.EnableSerialConsoleAccessInput{}, testAccountID)
+	require.NoError(t, err)
+
+	entry, err := svc.settingsKV.Get(ctx, testAccountID)
+	require.NoError(t, err)
+	assert.NotContains(t, string(entry.Value()), "state")
+}
+
+// A record with an unrecognised state fails closed rather than being treated
+// as active or sealed, for both a read and a mutation.
+func TestUnknownStateFailsClosed(t *testing.T) {
+	svc := setupTestAccountService(t)
+	ctx := context.Background()
+	_, err := svc.settingsKV.Put(ctx, testAccountID, []byte(`{"ebs_encryption_by_default":false,"serial_console_access":false,"state":"quarantined"}`))
+	require.NoError(t, err)
+
+	_, err = svc.GetEbsEncryptionByDefault(ctx, &ec2.GetEbsEncryptionByDefaultInput{}, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
+
+	_, err = svc.EnableEbsEncryptionByDefault(ctx, &ec2.EnableEbsEncryptionByDefaultInput{}, testAccountID)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInternalError), "got %v", err)
+
+	entry, err := svc.settingsKV.Get(ctx, testAccountID)
+	require.NoError(t, err)
+	var rec AccountSettingsRecord
+	require.NoError(t, json.Unmarshal(entry.Value(), &rec))
+	assert.EqualValues(t, "quarantined", rec.State, "the unknown-state record was never overwritten")
+}

spinifex/domains/ec2/account/settings_writes_guarded_test.go

@@ -0,0 +1,68 @@
+package account
+
+//test:in-package — pins what AccountSettingsSealCapabilityVersion 1 asserts:
+// nothing in this package writes the settings KV except through the
+// revision-guarded helpers.
+
+import (
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"io/fs"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+)
+
+func accountPackageDir(t *testing.T) string {
+	t.Helper()
+	_, file, _, ok := runtime.Caller(0)
+	if !ok {
+		t.Fatal("cannot locate test source")
+	}
+	return filepath.Dir(file)
+}
+
+// The capability daemons report as version 1 is a claim about every writer in
+// the binary, not just the responder: no non-test file in this package may
+// call Put or PutString, which bypass the revision-guarded mutate/seal paths.
+func TestAccountSettingsWritesAreRevisionGuarded(t *testing.T) {
+	root := accountPackageDir(t)
+	fset := token.NewFileSet()
+	scanned := 0
+
+	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+		if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
+			return err
+		}
+		f, perr := parser.ParseFile(fset, path, nil, 0)
+		if perr != nil {
+			return perr
+		}
+		scanned++
+
+		ast.Inspect(f, func(n ast.Node) bool {
+			call, ok := n.(*ast.CallExpr)
+			if !ok {
+				return true
+			}
+			sel, ok := call.Fun.(*ast.SelectorExpr)
+			if !ok {
+				return true
+			}
+			if sel.Sel.Name == "Put" || sel.Sel.Name == "PutString" {
+				t.Errorf("%s: unconditional %s call bypasses the revision-guarded settings write; use mutateSettings or SealAccount's CAS loop instead",
+					fset.Position(call.Pos()), sel.Sel.Name)
+			}
+			return true
+		})
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if scanned == 0 {
+		t.Fatalf("scanned no Go files under %s; the invariant would pass vacuously", root)
+	}
+}

spinifex/domains/ec2/awsapi/instance/DescribeInstanceTypes.go

@@ -3,7 +3,6 @@ package instance
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"log/slog"
@@ -101,10 +100,10 @@ func DescribeInstanceTypes(ctx context.Context, input *ec2.DescribeInstanceTypes
 
 		if missing := missingInstanceTypes(finalInstanceTypes, input.InstanceTypes); len(missing) > 0 {
 			if !proveAbsence || !everyNodeAnswered(nodeIDs, answered, sum) {
-				return nil, errors.Join(
+				return nil, awserrors.WithRetryAfter(
 					awserrors.Errorf(awserrors.ErrorServiceUnavailable,
 						"cannot confirm instance type(s) %s: describe fan-out did not complete", strings.Join(missing, ", ")),
-					awserrors.RetryAfter(awserrors.ErrorServiceUnavailable, describeIncompleteRetryAfter),
+					describeIncompleteRetryAfter,
 				)
 			}
 			return nil, awserrors.Errorf(awserrors.ErrorInvalidInstanceType,

spinifex/domains/ec2/awsapi/instance/DescribeInstances.go

@@ -126,10 +126,10 @@ func DescribeInstancesChecked(ctx context.Context, input *ec2.DescribeInstancesI
 			// An incomplete sweep cannot prove absence, so a missing id is
 			// unresolved, not gone. NotFound would be a false negative and an
 			// empty success would look identical to a real one, so return 503.
-			return nil, errors.Join(
+			return nil, awserrors.WithRetryAfter(
 				awserrors.Errorf(awserrors.ErrorServiceUnavailable,
 					"cannot confirm status of instance(s) %s: describe fan-out did not complete", strings.Join(missing, ", ")),
-				awserrors.RetryAfter(awserrors.ErrorServiceUnavailable, describeIncompleteRetryAfter),
+				describeIncompleteRetryAfter,
 			)
 		}
 	}
@@ -198,10 +198,10 @@ func DescribeAccountInstancesComplete(ctx context.Context, natsConn *nats.Conn,
 }
 
 func incompleteListingError() error {
-	return errors.Join(
+	return awserrors.WithRetryAfter(
 		awserrors.Errorf(awserrors.ErrorServiceUnavailable,
 			"cannot list instances: not every node and instance store answered"),
-		awserrors.RetryAfter(awserrors.ErrorServiceUnavailable, describeIncompleteRetryAfter),
+		describeIncompleteRetryAfter,
 	)
 }
 

spinifex/domains/route53/activation.go

@@ -0,0 +1,187 @@
+package route53
+
+import (
+	"context"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/route53/ext"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+)
+
+// GetHostedZoneActivation reports whether a zone is served and why not.
+func (s *Service) GetHostedZoneActivation(ctx context.Context, in *ext.GetHostedZoneActivationInput, account string) (*ext.GetHostedZoneActivationOutput, error) {
+	id := bareZoneID(aws.StringValue(in.Id))
+	zone, err := s.store.GetZone(ctx, account, id)
+	if err != nil {
+		return nil, s.zoneErr(id, err)
+	}
+	return &ext.GetHostedZoneActivationOutput{HostedZoneId: aws.String(hostedZonePrefix + zone.ID), Activation: activation(zone)}, nil
+}
+
+// VerifyHostedZoneDelegation checks a zone's delegation now, the console's
+// "Check now", rather than at its next scheduled check.
+func (s *Service) VerifyHostedZoneDelegation(ctx context.Context, in *ext.VerifyHostedZoneDelegationInput, account string) (*ext.VerifyHostedZoneDelegationOutput, error) {
+	id := bareZoneID(aws.StringValue(in.Id))
+	if s.Verifier == nil {
+		return nil, awserrors.Errorf(awserrors.ErrorServiceUnavailable, "delegation checks are not available on this cluster")
+	}
+	zone, err := s.Verifier.Verify(ctx, account, id)
+	if err != nil {
+		return nil, s.zoneErr(id, err)
+	}
+	return &ext.VerifyHostedZoneDelegationOutput{HostedZoneId: aws.String(hostedZonePrefix + zone.ID), Activation: activation(zone)}, nil
+}
+
+func activation(z *ZoneRecord) *ext.Activation {
+	a := &ext.Activation{
+		State:            aws.String(z.State),
+		Reason:           nonEmpty(z.StateReason),
+		NameServers:      aws.StringSlice(z.Nameservers),
+		DelegatedTo:      aws.StringSlice(z.Delegation),
+		DelegationSource: nonEmpty(z.DelegationSource),
+		Override:         nonEmpty(z.Override),
+	}
+	if !z.CheckedAt.IsZero() {
+		a.CheckedAt = aws.Time(z.CheckedAt)
+	}
+	if z.Override != "" {
+		a.Reason = nonEmpty(z.OverrideReason)
+	}
+	return a
+}
+
+// AdminZoneQuery selects zones across every account, by ID or by name.
+type AdminZoneQuery struct {
+	ZoneID string `json:"zone_id,omitempty"`
+	Name   string `json:"name,omitempty"`
+}
+
+// AdminZones is the zones a query matched.
+type AdminZones struct {
+	Zones []ZoneRecord `json:"zones"`
+}
+
+// AdminOverrideInput sets or clears an operator override on a zone.
+type AdminOverrideInput struct {
+	ZoneID string `json:"zone_id"`
+	// Override is StateActive, StateSuspended, or "" to hand the zone back to
+	// the verifier.
+	Override string `json:"override"`
+	Reason   string `json:"reason"`
+}
+
+// AdminDelegateInput names the platform-zone delegations to write or, with
+// Remove, withdraw.
+type AdminDelegateInput struct {
+	Names  []string `json:"names"`
+	Remove bool     `json:"remove,omitempty"`
+}
+
+// AdminDelegateOutput reports how many record changes were applied.
+type AdminDelegateOutput struct {
+	Applied int `json:"applied"`
+}
+
+// AdminFindZones lists the zones matching q in every account. It is for the
+// operator and is never reachable through the gateway.
+func (s *Service) AdminFindZones(ctx context.Context, q *AdminZoneQuery, _ string) (*AdminZones, error) {
+	zones, err := s.store.ListAllZones(ctx)
+	if err != nil {
+		return nil, internal("list hosted zones", err)
+	}
+	id, name := bareZoneID(q.ZoneID), normalizeName(q.Name)
+	out := &AdminZones{Zones: []ZoneRecord{}}
+	for _, z := range zones {
+		if (id == "" || z.ID == id) && (name == "" || z.Name == name) {
+			out.Zones = append(out.Zones, z)
+		}
+	}
+	slices.SortFunc(out.Zones, func(a, b ZoneRecord) int { return olderFirst(&a, &b) })
+	return out, nil
+}
+
+// AdminVerify checks one zone now, in whichever account holds it.
+func (s *Service) AdminVerify(ctx context.Context, q *AdminZoneQuery, _ string) (*AdminZones, error) {
+	z, err := s.adminZone(ctx, q.ZoneID)
+	if err != nil {
+		return nil, err
+	}
+	if s.Verifier == nil {
+		return nil, awserrors.Errorf(awserrors.ErrorServiceUnavailable, "delegation checks are not available on this cluster")
+	}
+	checked, err := s.Verifier.Verify(ctx, z.AccountID, z.ID)
+	if err != nil {
+		return nil, s.zoneErr(z.ID, err)
+	}
+	return &AdminZones{Zones: []ZoneRecord{*checked}}, nil
+}
+
+// AdminOverride activates or suspends a zone whatever its delegation, or
+// clears an override. Activation refuses a name active in another zone.
+func (s *Service) AdminOverride(ctx context.Context, in *AdminOverrideInput, _ string) (*AdminZones, error) {
+	if in.Override != "" && strings.TrimSpace(in.Reason) == "" {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "an override needs a reason")
+	}
+	z, err := s.adminZone(ctx, in.ZoneID)
+	if err != nil {
+		return nil, err
+	}
+	switch in.Override {
+	case StateActive:
+		zones, err := s.store.ListAllZones(ctx)
+		if err != nil {
+			return nil, internal("list hosted zones", err)
+		}
+		for _, other := range zones {
+			if other.Name == z.Name && other.ID != z.ID && other.State == StateActive {
+				return nil, awserrors.Errorf(awserrors.ErrorConflictingDomainExists, "%s is already active in account %s as %s", strings.TrimSuffix(z.Name, "."), other.AccountID, other.ID)
+			}
+		}
+	case StateSuspended, "":
+	default:
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "override must be %s, %s or empty", StateActive, StateSuspended)
+	}
+	err = s.store.MutateZone(ctx, z.AccountID, z.ID, func(r *ZoneRecord) (bool, error) {
+		switch in.Override {
+		case "":
+			// The verifier decides again from here, starting with a check.
+			if r.State == StateSuspended || r.Override == StateActive {
+				r.State = StatePendingNameservers
+			}
+			r.CheckedAt = time.Time{}
+		default:
+			r.State = in.Override
+		}
+		r.Override, r.OverrideReason = in.Override, strings.TrimSpace(in.Reason)
+		r.FailingSince = time.Time{}
+		return true, nil
+	})
+	if err != nil {
+		return nil, s.zoneErr(z.ID, err)
+	}
+	updated, err := s.store.GetZone(ctx, z.AccountID, z.ID)
+	if err != nil {
+		return nil, s.zoneErr(z.ID, err)
+	}
+	return &AdminZones{Zones: []ZoneRecord{*updated}}, nil
+}
+
+func (s *Service) adminZone(ctx context.Context, rawID string) (*ZoneRecord, error) {
+	id := bareZoneID(rawID)
+	if id == "" {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "a hosted zone ID is required")
+	}
+	found, err := s.AdminFindZones(ctx, &AdminZoneQuery{ZoneID: id}, "")
+	if err != nil {
+		return nil, err
+	}
+	if len(found.Zones) == 0 {
+		return nil, s.zoneErr(id, kvstore.ErrNotFound)
+	}
+	return &found.Zones[0], nil
+}

spinifex/domains/route53/activation_test.go

@@ -0,0 +1,111 @@
+package route53_test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/route53"
+	"github.com/mulgadc/spinifex/spinifex/domains/route53/ext"
+)
+
+func TestActivationIsReportedAndCheckedOnDemand(t *testing.T) {
+	svc, v, dns, _ := newVerifier(t)
+	svc.Verifier = v
+	id := createZone(t, svc, acctA, "demand.example")
+
+	got, err := svc.GetHostedZoneActivation(ctx, &ext.GetHostedZoneActivationInput{Id: aws.String(id)}, acctA)
+	require.NoError(t, err)
+	assert.Equal(t, route53.StatePendingNameservers, aws.StringValue(got.Activation.State))
+	assert.Nil(t, got.Activation.CheckedAt, "never checked")
+
+	dns.parents["demand.example"] = ours()
+	verified, err := svc.VerifyHostedZoneDelegation(ctx, &ext.VerifyHostedZoneDelegationInput{Id: aws.String(id)}, acctA)
+	require.NoError(t, err)
+	assert.Equal(t, route53.StateActive, aws.StringValue(verified.Activation.State))
+	assert.Equal(t, ours(), aws.StringValueSlice(verified.Activation.DelegatedTo))
+	assert.NotNil(t, verified.Activation.CheckedAt)
+
+	_, err = svc.GetHostedZoneActivation(ctx, &ext.GetHostedZoneActivationInput{Id: aws.String(id)}, acctB)
+	requireCode(t, err, "NoSuchHostedZone")
+	_, err = svc.VerifyHostedZoneDelegation(ctx, &ext.VerifyHostedZoneDelegationInput{Id: aws.String(id)}, acctB)
+	requireCode(t, err, "NoSuchHostedZone")
+}
+
+func TestAdminFindsZonesAcrossAccounts(t *testing.T) {
+	svc, _, _, c := newVerifier(t)
+	a := createZone(t, svc, acctA, "shared.example")
+	c.advance(time.Second)
+	createZone(t, svc, acctB, "shared.example")
+	createZone(t, svc, acctB, "other.example")
+
+	byName, err := svc.AdminFindZones(ctx, &route53.AdminZoneQuery{Name: "Shared.Example"}, "")
+	require.NoError(t, err)
+	require.Len(t, byName.Zones, 2)
+	assert.Equal(t, acctA, byName.Zones[0].AccountID, "oldest first")
+
+	byID, err := svc.AdminFindZones(ctx, &route53.AdminZoneQuery{ZoneID: a}, "")
+	require.NoError(t, err)
+	require.Len(t, byID.Zones, 1)
+	assert.Equal(t, zoneID(a), byID.Zones[0].ID)
+}
+
+func TestAdminOverrideActivatesSuspendsAndReleases(t *testing.T) {
+	svc, v, dns, c := newVerifier(t)
+	svc.Verifier = v
+	held := createZone(t, svc, acctA, "contested.example")
+	c.advance(time.Second)
+	squat := createZone(t, svc, acctB, "contested.example")
+
+	_, err := svc.AdminOverride(ctx, &route53.AdminOverrideInput{ZoneID: held, Override: route53.StateActive}, "")
+	requireCode(t, err, "InvalidInput")
+
+	out, err := svc.AdminOverride(ctx, &route53.AdminOverrideInput{ZoneID: held, Override: route53.StateActive, Reason: "air-gapped root"}, "")
+	require.NoError(t, err)
+	assert.Equal(t, route53.StateActive, out.Zones[0].State)
+
+	_, err = svc.AdminOverride(ctx, &route53.AdminOverrideInput{ZoneID: squat, Override: route53.StateActive, Reason: "mine"}, "")
+	requireCode(t, err, "ConflictingDomainExists")
+
+	// The delegation check never demotes an operator's activation.
+	pass(t, v)
+	assert.Equal(t, route53.StateActive, zone(t, svc, acctA, held).State)
+
+	_, err = svc.AdminOverride(ctx, &route53.AdminOverrideInput{ZoneID: held, Override: route53.StateSuspended, Reason: "dispute"}, "")
+	require.NoError(t, err)
+	dns.parents["contested.example"] = ours()
+	c.advance(time.Minute)
+	pass(t, v)
+	assert.Equal(t, route53.StateSuspended, zone(t, svc, acctA, held).State)
+	assert.Equal(t, route53.StateActive, zone(t, svc, acctB, squat).State, "suspending a zone frees its name")
+
+	_, err = svc.AdminOverride(ctx, &route53.AdminOverrideInput{ZoneID: held}, "")
+	require.NoError(t, err)
+	z := zone(t, svc, acctA, held)
+	assert.Equal(t, route53.StatePendingNameservers, z.State)
+	assert.Empty(t, z.Override)
+	pass(t, v)
+	assert.Equal(t, "the domain is active in another account", zone(t, svc, acctA, held).StateReason)
+
+	_, err = svc.AdminOverride(ctx, &route53.AdminOverrideInput{ZoneID: "ZNOPE"}, "")
+	requireCode(t, err, "NoSuchHostedZone")
+	_, err = svc.AdminOverride(ctx, &route53.AdminOverrideInput{ZoneID: held, Override: "BOGUS", Reason: "x"}, "")
+	requireCode(t, err, "InvalidInput")
+}
+
+func TestAdminVerifyChecksInTheOwningAccount(t *testing.T) {
+	svc, v, dns, _ := newVerifier(t)
+	id := createZone(t, svc, acctB, "admin.example")
+	_, err := svc.AdminVerify(ctx, &route53.AdminZoneQuery{ZoneID: id}, "")
+	requireCode(t, err, "ServiceUnavailable")
+
+	svc.Verifier = v
+	dns.parents["admin.example"] = ours()
+	out, err := svc.AdminVerify(ctx, &route53.AdminZoneQuery{ZoneID: id}, "")
+	require.NoError(t, err)
+	assert.Equal(t, route53.StateActive, out.Zones[0].State)
+	assert.Equal(t, acctB, out.Zones[0].AccountID)
+}

spinifex/domains/route53/awsapi/authz.go

@@ -0,0 +1,60 @@
+package awsapi
+
+import (
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/route53"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/route53/ext"
+)
+
+// Route 53 ARNs carry no region or account, so the policy gate cannot scope
+// them to the caller; the daemon looks every ID up within the caller's account.
+const arnPrefix = "arn:aws:route53:::"
+
+// ResourceARNs returns the resources an action acts on, for the policy gate.
+// Actions that name no zone or change are authorized against "*".
+func ResourceARNs(input any) []string {
+	switch in := input.(type) {
+	case *route53.GetHostedZoneInput:
+		return zoneARN(in.Id)
+	case *route53.UpdateHostedZoneCommentInput:
+		return zoneARN(in.Id)
+	case *route53.DeleteHostedZoneInput:
+		return zoneARN(in.Id)
+	case *route53.ChangeResourceRecordSetsInput:
+		return zoneARN(in.HostedZoneId)
+	case *route53.ListResourceRecordSetsInput:
+		return zoneARN(in.HostedZoneId)
+	case *ext.GetHostedZoneActivationInput:
+		return zoneARN(in.Id)
+	case *ext.VerifyHostedZoneDelegationInput:
+		return zoneARN(in.Id)
+	case *route53.GetChangeInput:
+		return []string{arnPrefix + "change/" + strings.TrimPrefix(aws.StringValue(in.Id), "/change/")}
+	case *route53.ChangeTagsForResourceInput:
+		return tagARNs(in.ResourceType, in.ResourceId)
+	case *route53.ListTagsForResourceInput:
+		return tagARNs(in.ResourceType, in.ResourceId)
+	case *route53.ListTagsForResourcesInput:
+		var out []string
+		for _, id := range in.ResourceIds {
+			out = append(out, tagARNs(in.ResourceType, id)...)
+		}
+		if len(out) == 0 {
+			return []string{"*"}
+		}
+		return out
+	default:
+		return []string{"*"}
+	}
+}
+
+func zoneARN(id *string) []string {
+	return []string{arnPrefix + "hostedzone/" + strings.TrimPrefix(aws.StringValue(id), "/hostedzone/")}
+}
+
+func tagARNs(resourceType, id *string) []string {
+	return []string{arnPrefix + aws.StringValue(resourceType) + "/" + aws.StringValue(id)}
+}

spinifex/domains/route53/awsapi/codec.go

@@ -0,0 +1,103 @@
+package awsapi
+
+import (
+	"bytes"
+	"encoding/xml"
+	"errors"
+	"log/slog"
+	"net/http"
+	"net/url"
+	"reflect"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	awsxml "github.com/mulgadc/spinifex/spinifex/foundation/aws/xml"
+)
+
+// XMLNamespace is the namespace every Route 53 request and response carries.
+const XMLNamespace = "https://route53.amazonaws.com/doc/2013-04-01/"
+
+// bindInput fills an SDK input struct the way the REST-XML protocol lays it
+// out: members tagged location:"uri" from the path, location:"querystring"
+// from the query, and the rest from the XML body.
+func bindInput(input any, path map[string]string, query url.Values, body []byte) error {
+	v := reflect.ValueOf(input).Elem()
+	t := v.Type()
+	for i := range t.NumField() {
+		f := t.Field(i)
+		var raw string
+		var ok bool
+		switch f.Tag.Get("location") {
+		case "uri":
+			raw, ok = path[f.Tag.Get("locationName")]
+		case "querystring":
+			raw, ok = query.Get(f.Tag.Get("locationName")), query.Has(f.Tag.Get("locationName"))
+		default:
+			continue
+		}
+		if !ok || f.Type != reflect.TypeFor[*string]() {
+			continue
+		}
+		v.Field(i).Set(reflect.ValueOf(aws.String(raw)))
+	}
+	if len(bytes.TrimSpace(body)) == 0 {
+		return nil
+	}
+	if err := xmlutil.UnmarshalXML(input, xml.NewDecoder(bytes.NewReader(body)), ""); err != nil {
+		return awserrors.Errorf(awserrors.ErrorInvalidInput, "the request body is not valid XML: %v", err)
+	}
+	return nil
+}
+
+// writeXML renders output under a <ActionResponse> root in the Route 53
+// namespace, plus the Location header CreateHostedZone carries.
+func writeXML(w http.ResponseWriter, r *http.Request, action string, status int, output any) error {
+	if loc := locationHeader(output); loc != "" {
+		w.Header().Set("Location", "https://"+r.Host+loc)
+	}
+	body, err := awsxml.Marshal(rootElement(action+"Response", output))
+	if err != nil {
+		slog.ErrorContext(r.Context(), "Route 53: marshal response", "action", action, "err", err)
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+	w.Header().Set("Content-Type", "text/xml")
+	w.WriteHeader(status)
+	_, _ = w.Write([]byte(xml.Header))
+	_, _ = w.Write(body)
+	return nil
+}
+
+func rootElement(name string, payload any) any {
+	t := reflect.StructOf([]reflect.StructField{{
+		Name: "Value",
+		Type: reflect.TypeOf(payload),
+		Tag:  reflect.StructTag(`locationName:"` + name + `" xmlURI:"` + XMLNamespace + `"`),
+	}})
+	v := reflect.New(t).Elem()
+	v.Field(0).Set(reflect.ValueOf(payload))
+	return v.Interface()
+}
+
+func locationHeader(output any) string {
+	v := reflect.ValueOf(output)
+	if v.Kind() != reflect.Pointer || v.IsNil() {
+		return ""
+	}
+	f := v.Elem().FieldByName("Location")
+	if !f.IsValid() || f.Kind() != reflect.Pointer || f.IsNil() || f.Elem().Kind() != reflect.String {
+		return ""
+	}
+	return f.Elem().String()
+}
+
+func patternNames(pattern string) []string {
+	var names []string
+	for seg := range strings.SplitSeq(pattern, "/") {
+		if strings.HasPrefix(seg, "{") && strings.HasSuffix(seg, "}") {
+			names = append(names, seg[1:len(seg)-1])
+		}
+	}
+	return names
+}

spinifex/domains/route53/awsapi/registration.go

@@ -0,0 +1,176 @@
+// Package awsapi adapts Route 53's REST-XML API to the route53 domain's
+// NATS-backed service. Generic gateway routing and authorization stay at the
+// ingress boundary.
+package awsapi
+
+import (
+	"context"
+	"errors"
+	"io"
+	"log/slog"
+	"net/http"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/route53"
+	"github.com/mulgadc/bluebottle/pkg/sigv4"
+	"github.com/mulgadc/spinifex/spinifex/domains/route53/ext"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/rest"
+	"github.com/nats-io/nats.go"
+)
+
+// ServiceName is the SigV4 credential-scope service Route 53 clients sign for.
+const ServiceName = "route53"
+
+// natsTimeout bounds the gateway's wait for the daemon's answer.
+const natsTimeout = 30 * time.Second
+
+// Deps are the composed capabilities the Route 53 control plane dispatches
+// to. A nil NATS connection answers every action with ServerInternal.
+type Deps struct {
+	NATS *nats.Conn
+}
+
+// operation is one Route 53 action: its wire input and output types and the
+// status a success returns.
+type operation struct {
+	newInput func() any
+	relay    func(ctx context.Context, nc *nats.Conn, subject string, in any, account string) (any, error)
+	status   int
+}
+
+func op[I, O any](status int) operation {
+	return operation{
+		newInput: func() any { return new(I) },
+		relay: func(ctx context.Context, nc *nats.Conn, subject string, in any, account string) (any, error) {
+			return natsmsg.NATSRequest[O](ctx, nc, subject, in, natsTimeout, account)
+		},
+		status: status,
+	}
+}
+
+const base = "/2013-04-01"
+
+// routes is the Route 53 REST surface Spinifex serves, paths as in the
+// 2013-04-01 model.
+var routes = []rest.Route[operation]{
+	{Method: http.MethodPost, Pattern: base + "/hostedzone", Action: "CreateHostedZone", Handler: op[route53.CreateHostedZoneInput, route53.CreateHostedZoneOutput](http.StatusCreated)},
+	{Method: http.MethodGet, Pattern: base + "/hostedzone", Action: "ListHostedZones", Handler: op[route53.ListHostedZonesInput, route53.ListHostedZonesOutput](http.StatusOK)},
+	{Method: http.MethodGet, Pattern: base + "/hostedzonesbyname", Action: "ListHostedZonesByName", Handler: op[route53.ListHostedZonesByNameInput, route53.ListHostedZonesByNameOutput](http.StatusOK)},
+	{Method: http.MethodGet, Pattern: base + "/hostedzonecount", Action: "GetHostedZoneCount", Handler: op[route53.GetHostedZoneCountInput, route53.GetHostedZoneCountOutput](http.StatusOK)},
+	{Method: http.MethodGet, Pattern: base + "/hostedzone/{Id}", Action: "GetHostedZone", Handler: op[route53.GetHostedZoneInput, route53.GetHostedZoneOutput](http.StatusOK)},
+	{Method: http.MethodPost, Pattern: base + "/hostedzone/{Id}", Action: "UpdateHostedZoneComment", Handler: op[route53.UpdateHostedZoneCommentInput, route53.UpdateHostedZoneCommentOutput](http.StatusOK)},
+	{Method: http.MethodDelete, Pattern: base + "/hostedzone/{Id}", Action: "DeleteHostedZone", Handler: op[route53.DeleteHostedZoneInput, route53.DeleteHostedZoneOutput](http.StatusOK)},
+	{Method: http.MethodPost, Pattern: base + "/hostedzone/{Id}/rrset", Action: "ChangeResourceRecordSets", Handler: op[route53.ChangeResourceRecordSetsInput, route53.ChangeResourceRecordSetsOutput](http.StatusOK)},
+	{Method: http.MethodGet, Pattern: base + "/hostedzone/{Id}/rrset", Action: "ListResourceRecordSets", Handler: op[route53.ListResourceRecordSetsInput, route53.ListResourceRecordSetsOutput](http.StatusOK)},
+	{Method: http.MethodGet, Pattern: base + "/change/{Id}", Action: "GetChange", Handler: op[route53.GetChangeInput, route53.GetChangeOutput](http.StatusOK)},
+	{Method: http.MethodPost, Pattern: base + "/tags/{ResourceType}/{ResourceId}", Action: "ChangeTagsForResource", Handler: op[route53.ChangeTagsForResourceInput, route53.ChangeTagsForResourceOutput](http.StatusOK)},
+	{Method: http.MethodGet, Pattern: base + "/tags/{ResourceType}/{ResourceId}", Action: "ListTagsForResource", Handler: op[route53.ListTagsForResourceInput, route53.ListTagsForResourceOutput](http.StatusOK)},
+	{Method: http.MethodPost, Pattern: base + "/tags/{ResourceType}", Action: "ListTagsForResources", Handler: op[route53.ListTagsForResourcesInput, route53.ListTagsForResourcesOutput](http.StatusOK)},
+
+	// Spinifex extensions, which standard SDKs never call.
+	{Method: http.MethodGet, Pattern: base + "/hostedzone/{Id}/activation", Action: ext.ActionGetHostedZoneActivation, Handler: op[ext.GetHostedZoneActivationInput, ext.GetHostedZoneActivationOutput](http.StatusOK)},
+	{Method: http.MethodPost, Pattern: base + "/hostedzone/{Id}/activation", Action: ext.ActionVerifyHostedZoneDelegation, Handler: op[ext.VerifyHostedZoneDelegationInput, ext.VerifyHostedZoneDelegationOutput](http.StatusOK)},
+}
+
+// extensions are served but are not operations of the AWS model.
+var extensions = []string{ext.ActionGetHostedZoneActivation, ext.ActionVerifyHostedZoneDelegation}
+
+var router = rest.NewRouter(ServiceName, routes)
+
+// NewRegistration returns the Route 53 REST-XML control-plane registration.
+func NewRegistration(deps Deps) dispatch.Registration {
+	return dispatch.Registration{
+		Service:       ServiceName,
+		ResolveAction: resolveAction,
+		Dispatch:      deps.dispatch,
+		Errors:        dispatch.ErrorEnvelopeXML,
+		Inventory:     OperationInventory(),
+	}
+}
+
+// ExtensionActions lists the Spinifex actions served beside the AWS model's.
+func ExtensionActions() []string { return slices.Clone(extensions) }
+
+// OperationInventory lists the AWS model actions the registration serves.
+func OperationInventory() dispatch.Inventory {
+	names := make([]string, 0, len(routes))
+	for _, r := range routes {
+		if !slices.Contains(extensions, r.Action) {
+			names = append(names, r.Action)
+		}
+	}
+	slices.Sort(names)
+	return dispatch.Inventory{Registered: names}
+}
+
+func resolveAction(r *http.Request) string {
+	action, _, _, _ := lookup(r)
+	return action
+}
+
+// lookup matches a request to its route. The model's ChangeResourceRecordSets
+// path ends in a slash, which SDKs send and the router does not match.
+func lookup(r *http.Request) (string, []string, operation, bool) {
+	return router.Lookup(r.Method, strings.TrimSuffix(r.URL.EscapedPath(), "/"))
+}
+
+// dispatch binds the request, authorizes it against the zone or change it
+// names, relays it to the daemon and writes the XML response.
+func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
+	r := inv.Request
+	action, params, operation, ok := lookup(r)
+	if !ok {
+		slog.DebugContext(r.Context(), "Route 53: no route", "method", r.Method, "path", r.URL.Path)
+		return errors.New(awserrors.ErrorInvalidAction)
+	}
+	if inv.AccountID == "" {
+		slog.ErrorContext(r.Context(), "Route 53 dispatch: no account ID in auth context")
+		return errors.New(awserrors.ErrorInternalError)
+	}
+	body, err := io.ReadAll(io.LimitReader(r.Body, sigv4.MaxPayloadLen+1))
+	if err != nil {
+		return errors.New(awserrors.ErrorInvalidInput)
+	}
+	if int64(len(body)) > sigv4.MaxPayloadLen {
+		return errors.New(awserrors.ErrorRequestEntityTooLarge)
+	}
+
+	input := operation.newInput()
+	if err := bindInput(input, pathParams(action, params), r.URL.Query(), body); err != nil {
+		return err
+	}
+	if err := inv.Authorize(ServiceName, action, ResourceARNs(input), nil); err != nil {
+		return err
+	}
+	if d.NATS == nil {
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+	output, err := operation.relay(r.Context(), d.NATS, ServiceName+"."+action, input, inv.AccountID)
+	if err != nil {
+		return err
+	}
+	return writeXML(w, r, action, operation.status, output)
+}
+
+// pathParams names the router's positional params by the route's pattern.
+func pathParams(action string, params []string) map[string]string {
+	for _, rt := range routes {
+		if rt.Action != action {
+			continue
+		}
+		names := patternNames(rt.Pattern)
+		out := make(map[string]string, len(names))
+		for i, n := range names {
+			if i < len(params) {
+				out[n] = params[i]
+			}
+		}
+		return out
+	}
+	return nil
+}

spinifex/domains/route53/awsapi/registration_test.go

@@ -0,0 +1,324 @@
+package awsapi_test
+
+import (
+	"context"
+	"encoding/json"
+	"encoding/xml"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/aws/session"
+	r53 "github.com/aws/aws-sdk-go/service/route53"
+	"github.com/mulgadc/bluebottle/pkg/iampolicy"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/route53"
+	"github.com/mulgadc/spinifex/spinifex/domains/route53/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/domains/route53/ext"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
+)
+
+const account = "000000000011"
+
+type authorized struct {
+	mu    sync.Mutex
+	calls []string
+}
+
+func (a *authorized) record(service, action string, resources []string, _ iampolicy.ConditionKeys) error {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	a.calls = append(a.calls, fmt.Sprintf("%s:%s %s", service, action, strings.Join(resources, ",")))
+	return nil
+}
+
+// serveDomain answers route53.<Method> the way the daemon does, from the real
+// service, so the SDK below exercises the whole wire path.
+func serveDomain(t *testing.T, nc *nats.Conn, svc *route53.Service) {
+	t.Helper()
+	v := reflect.ValueOf(svc)
+	for _, action := range append(awsapi.OperationInventory().Registered, awsapi.ExtensionActions()...) {
+		m := v.MethodByName(action)
+		require.True(t, m.IsValid(), "the service has no method for %s", action)
+		inType := m.Type().In(1).Elem()
+		sub, err := nc.Subscribe("route53."+action, func(msg *nats.Msg) {
+			in := reflect.New(inType)
+			require.NoError(t, json.Unmarshal(msg.Data, in.Interface()))
+			out := m.Call([]reflect.Value{reflect.ValueOf(context.Background()), in, reflect.ValueOf(natsmsg.AccountIDFromMsg(msg))})
+			if err, _ := reflect.TypeAssert[error](out[1]); err != nil {
+				_, message, _ := awserrors.ResolveErrorDetail(err)
+				_ = msg.Respond(awserrors.GenerateErrorPayloadWithMessage(awserrors.ValidErrorCodeFromError(err), message))
+				return
+			}
+			body, err := json.Marshal(out[0].Interface())
+			require.NoError(t, err)
+			_ = msg.Respond(body)
+		})
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = sub.Unsubscribe() })
+	}
+}
+
+// newClient returns an SDK client whose requests run through the
+// registration, with errors rendered as the gateway renders them.
+func newClient(t *testing.T) (*r53.Route53, *authorized) {
+	t.Helper()
+	_, nc, js := testutil.StartTestJetStream(t)
+	svc := route53.NewService(route53.NewStore(js))
+	svc.Nameservers = func() []string { return []string{"ns1.spx3.net", "ns2.spx3.net"} }
+	serveDomain(t, nc, svc)
+
+	reg := awsapi.NewRegistration(awsapi.Deps{NATS: nc})
+	auth := &authorized{}
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		err := reg.Dispatch(w, dispatch.Invocation{Request: r, AccountID: account, Region: "us-east-1", Authorize: auth.record})
+		if err == nil {
+			return
+		}
+		code, message, _ := awserrors.ResolveErrorDetail(err)
+		if code == "" {
+			code = err.Error()
+		}
+		meta := awserrors.LookupErrorMessage(awsapi.ServiceName, code)
+		if message == "" {
+			message = meta.Message
+		}
+		w.Header().Set("Content-Type", "text/xml")
+		w.WriteHeader(meta.HTTPCode)
+		_ = xml.NewEncoder(w).Encode(struct {
+			XMLName xml.Name `xml:"ErrorResponse"`
+			Type    string   `xml:"Error>Type"`
+			Code    string   `xml:"Error>Code"`
+			Message string   `xml:"Error>Message"`
+		}{Type: "Sender", Code: code, Message: message})
+	}))
+	t.Cleanup(srv.Close)
+
+	sess, err := session.NewSession(&aws.Config{
+		Endpoint:    aws.String(srv.URL),
+		Region:      aws.String("us-east-1"),
+		Credentials: credentials.NewStaticCredentials("AKID", "secret", ""),
+		MaxRetries:  aws.Int(0),
+	})
+	require.NoError(t, err)
+	return r53.New(sess), auth
+}
+
+func requireAWSCode(t *testing.T, err error, code string) {
+	t.Helper()
+	var aerr awserr.Error
+	require.ErrorAs(t, err, &aerr)
+	assert.Equal(t, code, aerr.Code(), aerr.Message())
+}
+
+func TestSDKDrivesTheTerraformSequence(t *testing.T) {
+	client, auth := newClient(t)
+
+	created, err := client.CreateHostedZone(&r53.CreateHostedZoneInput{
+		Name: aws.String("example.com"), CallerReference: aws.String("terraform-1"),
+		HostedZoneConfig: &r53.HostedZoneConfig{Comment: aws.String("Managed by Terraform")},
+	})
+	require.NoError(t, err)
+	zoneID := aws.StringValue(created.HostedZone.Id)
+	bare := strings.TrimPrefix(zoneID, "/hostedzone/")
+	assert.True(t, strings.HasSuffix(aws.StringValue(created.Location), "/2013-04-01/hostedzone/"+bare), aws.StringValue(created.Location))
+	assert.Equal(t, []string{"ns1.spx3.net", "ns2.spx3.net"}, aws.StringValueSlice(created.DelegationSet.NameServers))
+	assert.Equal(t, "INSYNC", aws.StringValue(created.ChangeInfo.Status))
+	assert.False(t, created.ChangeInfo.SubmittedAt.IsZero())
+
+	_, err = client.GetChange(&r53.GetChangeInput{Id: created.ChangeInfo.Id})
+	require.NoError(t, err)
+
+	got, err := client.GetHostedZone(&r53.GetHostedZoneInput{Id: aws.String(zoneID)})
+	require.NoError(t, err)
+	assert.Equal(t, "example.com.", aws.StringValue(got.HostedZone.Name))
+	assert.Equal(t, "Managed by Terraform", aws.StringValue(got.HostedZone.Config.Comment))
+
+	tags, err := client.ListTagsForResource(&r53.ListTagsForResourceInput{ResourceType: aws.String("hostedzone"), ResourceId: aws.String(bare)})
+	require.NoError(t, err)
+	assert.Equal(t, bare, aws.StringValue(tags.ResourceTagSet.ResourceId))
+
+	record := &r53.ResourceRecordSet{
+		Name: aws.String("www.example.com"), Type: aws.String("A"), TTL: aws.Int64(300),
+		ResourceRecords: []*r53.ResourceRecord{{Value: aws.String("192.0.2.10")}},
+	}
+	changed, err := client.ChangeResourceRecordSets(&r53.ChangeResourceRecordSetsInput{
+		HostedZoneId: aws.String(bare),
+		ChangeBatch:  &r53.ChangeBatch{Comment: aws.String("Managed by Terraform"), Changes: []*r53.Change{{Action: aws.String("CREATE"), ResourceRecordSet: record}}},
+	})
+	require.NoError(t, err)
+	assert.Equal(t, "Managed by Terraform", aws.StringValue(changed.ChangeInfo.Comment))
+
+	sets, err := client.ListResourceRecordSets(&r53.ListResourceRecordSetsInput{
+		HostedZoneId: aws.String(bare), MaxItems: aws.String("1"),
+		StartRecordName: aws.String("www.example.com."), StartRecordType: aws.String("A"),
+	})
+	require.NoError(t, err)
+	require.Len(t, sets.ResourceRecordSets, 1)
+	assert.Equal(t, "www.example.com.", aws.StringValue(sets.ResourceRecordSets[0].Name))
+	assert.Equal(t, "192.0.2.10", aws.StringValue(sets.ResourceRecordSets[0].ResourceRecords[0].Value))
+	assert.Equal(t, int64(300), aws.Int64Value(sets.ResourceRecordSets[0].TTL))
+
+	_, err = client.ChangeResourceRecordSets(&r53.ChangeResourceRecordSetsInput{
+		HostedZoneId: aws.String(bare),
+		ChangeBatch:  &r53.ChangeBatch{Changes: []*r53.Change{{Action: aws.String("CREATE"), ResourceRecordSet: record}}},
+	})
+	requireAWSCode(t, err, "InvalidChangeBatch")
+
+	_, err = client.DeleteHostedZone(&r53.DeleteHostedZoneInput{Id: aws.String(bare)})
+	requireAWSCode(t, err, "HostedZoneNotEmpty")
+
+	record.Name = aws.String("www.example.com.")
+	_, err = client.ChangeResourceRecordSets(&r53.ChangeResourceRecordSetsInput{
+		HostedZoneId: aws.String(bare),
+		ChangeBatch:  &r53.ChangeBatch{Comment: aws.String("Deleted by Terraform"), Changes: []*r53.Change{{Action: aws.String("DELETE"), ResourceRecordSet: record}}},
+	})
+	require.NoError(t, err)
+	deleted, err := client.DeleteHostedZone(&r53.DeleteHostedZoneInput{Id: aws.String(bare)})
+	require.NoError(t, err)
+	_, err = client.GetChange(&r53.GetChangeInput{Id: deleted.ChangeInfo.Id})
+	require.NoError(t, err)
+
+	_, err = client.GetHostedZone(&r53.GetHostedZoneInput{Id: aws.String(bare)})
+	requireAWSCode(t, err, "NoSuchHostedZone")
+
+	assert.Contains(t, auth.calls, "route53:GetHostedZone arn:aws:route53:::hostedzone/"+bare)
+	assert.Contains(t, auth.calls, "route53:ChangeResourceRecordSets arn:aws:route53:::hostedzone/"+bare)
+	assert.Contains(t, auth.calls, "route53:ListTagsForResource arn:aws:route53:::hostedzone/"+bare)
+	assert.Contains(t, auth.calls, "route53:CreateHostedZone *")
+}
+
+func TestSDKListsAndTags(t *testing.T) {
+	client, _ := newClient(t)
+	var ids []string
+	for _, name := range []string{"b.example.com", "example.com", "example.org"} {
+		out, err := client.CreateHostedZone(&r53.CreateHostedZoneInput{Name: aws.String(name), CallerReference: aws.String(name)})
+		require.NoError(t, err)
+		ids = append(ids, strings.TrimPrefix(aws.StringValue(out.HostedZone.Id), "/hostedzone/"))
+	}
+
+	var listed []string
+	require.NoError(t, client.ListHostedZonesPages(&r53.ListHostedZonesInput{MaxItems: aws.String("2")}, func(p *r53.ListHostedZonesOutput, _ bool) bool {
+		for _, z := range p.HostedZones {
+			listed = append(listed, aws.StringValue(z.Name))
+		}
+		return true
+	}))
+	assert.ElementsMatch(t, []string{"b.example.com.", "example.com.", "example.org."}, listed)
+
+	byName, err := client.ListHostedZonesByName(&r53.ListHostedZonesByNameInput{DNSName: aws.String("example.com"), MaxItems: aws.String("2")})
+	require.NoError(t, err)
+	require.Len(t, byName.HostedZones, 2)
+	assert.Equal(t, "example.com.", aws.StringValue(byName.HostedZones[0].Name))
+	assert.Equal(t, "b.example.com.", aws.StringValue(byName.HostedZones[1].Name))
+	assert.True(t, aws.BoolValue(byName.IsTruncated))
+	assert.Equal(t, "example.org.", aws.StringValue(byName.NextDNSName))
+
+	count, err := client.GetHostedZoneCount(&r53.GetHostedZoneCountInput{})
+	require.NoError(t, err)
+	assert.Equal(t, int64(3), aws.Int64Value(count.HostedZoneCount))
+
+	_, err = client.UpdateHostedZoneComment(&r53.UpdateHostedZoneCommentInput{Id: aws.String(ids[0]), Comment: aws.String("moved")})
+	require.NoError(t, err)
+
+	_, err = client.ChangeTagsForResource(&r53.ChangeTagsForResourceInput{
+		ResourceType: aws.String("hostedzone"), ResourceId: aws.String(ids[0]),
+		AddTags: []*r53.Tag{{Key: aws.String("env"), Value: aws.String("prod")}},
+	})
+	require.NoError(t, err)
+	many, err := client.ListTagsForResources(&r53.ListTagsForResourcesInput{ResourceType: aws.String("hostedzone"), ResourceIds: aws.StringSlice(ids[:2])})
+	require.NoError(t, err)
+	require.Len(t, many.ResourceTagSets, 2)
+	assert.Equal(t, "env", aws.StringValue(many.ResourceTagSets[0].Tags[0].Key))
+}
+
+func TestUnservedRouteIsInvalidAction(t *testing.T) {
+	client, _ := newClient(t)
+	_, err := client.CreateHealthCheck(&r53.CreateHealthCheckInput{
+		CallerReference:   aws.String("hc"),
+		HealthCheckConfig: &r53.HealthCheckConfig{Type: aws.String("HTTP"), FullyQualifiedDomainName: aws.String("example.com")},
+	})
+	requireAWSCode(t, err, "InvalidAction")
+}
+
+func TestOperationInventoryMatchesTheServiceMethods(t *testing.T) {
+	inv := awsapi.OperationInventory()
+	svc := reflect.TypeFor[*route53.Service]()
+	for _, action := range append(inv.Registered, awsapi.ExtensionActions()...) {
+		_, ok := svc.MethodByName(action)
+		assert.True(t, ok, "%s is routed but the service does not implement it", action)
+	}
+	reg := awsapi.NewRegistration(awsapi.Deps{})
+	assert.Equal(t, dispatch.ErrorEnvelopeXML, reg.Errors)
+	assert.Equal(t, "route53", reg.Service)
+}
+
+func TestResolveActionMatchesEveryRoute(t *testing.T) {
+	reg := awsapi.NewRegistration(awsapi.Deps{})
+	cases := map[string]string{
+		"POST /2013-04-01/hostedzone":                    "CreateHostedZone",
+		"GET /2013-04-01/hostedzone":                     "ListHostedZones",
+		"GET /2013-04-01/hostedzonesbyname":              "ListHostedZonesByName",
+		"GET /2013-04-01/hostedzonecount":                "GetHostedZoneCount",
+		"GET /2013-04-01/hostedzone/Z1":                  "GetHostedZone",
+		"POST /2013-04-01/hostedzone/Z1":                 "UpdateHostedZoneComment",
+		"DELETE /2013-04-01/hostedzone/Z1":               "DeleteHostedZone",
+		"POST /2013-04-01/hostedzone/Z1/rrset/":          "ChangeResourceRecordSets",
+		"GET /2013-04-01/hostedzone/Z1/rrset":            "ListResourceRecordSets",
+		"GET /2013-04-01/change/C1":                      "GetChange",
+		"POST /2013-04-01/tags/hostedzone/Z1":            "ChangeTagsForResource",
+		"GET /2013-04-01/tags/hostedzone/Z1":             "ListTagsForResource",
+		"POST /2013-04-01/tags/hostedzone":               "ListTagsForResources",
+		"GET /2013-04-01/hostedzone/Z1/activation":       "GetHostedZoneActivation",
+		"POST /2013-04-01/hostedzone/Z1/activation":      "VerifyHostedZoneDelegation",
+		"POST /2013-04-01/healthcheck":                   "",
+		"GET /2013-04-01/hostedzone/Z1/rrset/extra/more": "",
+	}
+	for req, want := range cases {
+		method, path, _ := strings.Cut(req, " ")
+		r := httptest.NewRequest(method, path, nil)
+		assert.Equal(t, want, reg.ResolveAction(r), req)
+	}
+	assert.Len(t, awsapi.OperationInventory().Registered, 13, "extensions are not AWS model operations")
+}
+
+// The extensions are REST-XML like the rest of the API, so an SDK client can
+// call them as custom operations; this is how the console and spx do.
+func TestExtensionsRoundTripThroughTheSDK(t *testing.T) {
+	client, auth := newClient(t)
+	zone, err := client.CreateHostedZone(&r53.CreateHostedZoneInput{Name: aws.String("example.com"), CallerReference: aws.String("ext")})
+	require.NoError(t, err)
+	id := strings.TrimPrefix(aws.StringValue(zone.HostedZone.Id), "/hostedzone/")
+
+	out := &ext.GetHostedZoneActivationOutput{}
+	req := client.NewRequest(&request.Operation{Name: ext.ActionGetHostedZoneActivation, HTTPMethod: http.MethodGet, HTTPPath: "/2013-04-01/hostedzone/{Id}/activation"},
+		&ext.GetHostedZoneActivationInput{Id: aws.String(id)}, out)
+	require.NoError(t, req.Send())
+	assert.Equal(t, "/hostedzone/"+id, aws.StringValue(out.HostedZoneId))
+	require.NotNil(t, out.Activation)
+	assert.Equal(t, route53.StatePendingNameservers, aws.StringValue(out.Activation.State))
+	assert.Equal(t, []string{"ns1.spx3.net", "ns2.spx3.net"}, aws.StringValueSlice(out.Activation.NameServers))
+	assert.Contains(t, auth.calls, "route53:GetHostedZoneActivation arn:aws:route53:::hostedzone/"+id)
+
+	verify := client.NewRequest(&request.Operation{Name: ext.ActionVerifyHostedZoneDelegation, HTTPMethod: http.MethodPost, HTTPPath: "/2013-04-01/hostedzone/{Id}/activation"},
+		&ext.VerifyHostedZoneDelegationInput{Id: aws.String(id)}, &ext.VerifyHostedZoneDelegationOutput{})
+	requireAWSCode(t, verify.Send(), "ServiceUnavailable")
+
+	missing := client.NewRequest(&request.Operation{Name: ext.ActionGetHostedZoneActivation, HTTPMethod: http.MethodGet, HTTPPath: "/2013-04-01/hostedzone/{Id}/activation"},
+		&ext.GetHostedZoneActivationInput{Id: aws.String("ZNOPE")}, &ext.GetHostedZoneActivationOutput{})
+	requireAWSCode(t, missing.Send(), "NoSuchHostedZone")
+}

spinifex/domains/route53/changes.go

@@ -0,0 +1,278 @@
+package route53
+
+import (
+	"fmt"
+	"net"
+	"slices"
+	"strconv"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/route53"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// AWS default and hard quotas that bound a change batch and a zone.
+const (
+	maxRecordSetsPerZone       = 10_000
+	maxRecordsPerChangeRequest = 1_000
+	maxValueCharsPerChange     = 32_000
+	maxValuesPerRecordSet      = 400
+	maxTTL                     = 2_147_483_647
+	defaultNSTTL               = 172_800
+	defaultSOATTL              = 900
+)
+
+// supportedTypes are the record types a simple-routing record set may carry.
+var supportedTypes = []string{"A", "AAAA", "CAA", "CNAME", "MX", "NS", "PTR", "SOA", "SPF", "SRV", "TXT"}
+
+// applyChangeBatch applies batch to a copy of sets in order, as Route 53 does,
+// and returns the result only if every change is valid. Every problem found is
+// reported in one InvalidChangeBatch, so the caller sees them all at once.
+func applyChangeBatch(zone string, sets []RRSet, batch *route53.ChangeBatch) ([]RRSet, error) {
+	if batch == nil || len(batch.Changes) == 0 {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "ChangeBatch must contain at least one change")
+	}
+	if err := checkBatchSize(batch); err != nil {
+		return nil, err
+	}
+	out := slices.Clone(sets)
+	var problems []string
+	for _, ch := range batch.Changes {
+		if ch == nil || ch.ResourceRecordSet == nil {
+			return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "every change needs an Action and a ResourceRecordSet")
+		}
+		set, err := toRRSet(zone, ch.ResourceRecordSet)
+		if err != nil {
+			if code, _, ok := awserrors.ResolveErrorDetail(err); ok && code != awserrors.ErrorInvalidChangeBatch {
+				return nil, err
+			}
+			problems = append(problems, messageOf(err))
+			continue
+		}
+		var msg string
+		out, msg = applyOne(zone, out, aws.StringValue(ch.Action), set)
+		if msg != "" {
+			problems = append(problems, msg)
+		}
+	}
+	if len(problems) == 0 && len(out) > maxRecordSetsPerZone {
+		problems = append(problems, fmt.Sprintf("The hosted zone %s would exceed the limit of %d resource record sets", zone, maxRecordSetsPerZone))
+	}
+	if len(problems) > 0 {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidChangeBatch, "[%s]", strings.Join(problems, ", "))
+	}
+	return out, nil
+}
+
+func checkBatchSize(batch *route53.ChangeBatch) error {
+	records, chars := 0, 0
+	for _, ch := range batch.Changes {
+		if ch == nil || ch.ResourceRecordSet == nil {
+			continue
+		}
+		weight := 1
+		if aws.StringValue(ch.Action) == route53.ChangeActionUpsert {
+			weight = 2
+		}
+		for _, rr := range ch.ResourceRecordSet.ResourceRecords {
+			records += weight
+			chars += weight * len(aws.StringValue(rr.Value))
+		}
+	}
+	if records > maxRecordsPerChangeRequest {
+		return awserrors.Errorf(awserrors.ErrorInvalidChangeBatch, "[Number of records limit of %d exceeded.]", maxRecordsPerChangeRequest)
+	}
+	if chars > maxValueCharsPerChange {
+		return awserrors.Errorf(awserrors.ErrorInvalidChangeBatch, "[Number of characters limit of %d exceeded.]", maxValueCharsPerChange)
+	}
+	return nil
+}
+
+func applyOne(zone string, sets []RRSet, action string, set RRSet) ([]RRSet, string) {
+	at := slices.IndexFunc(sets, func(s RRSet) bool { return s.Name == set.Name && s.Type == set.Type })
+	label := fmt.Sprintf("[name='%s', type='%s']", set.Name, set.Type)
+	apexDefault := set.Name == zone && (set.Type == "SOA" || set.Type == "NS")
+
+	switch action {
+	case route53.ChangeActionCreate:
+		if at >= 0 {
+			return sets, "Tried to create resource record set " + label + " but it already exists"
+		}
+		if msg := conflicts(zone, sets, set); msg != "" {
+			return sets, msg
+		}
+		return append(sets, set), ""
+	case route53.ChangeActionUpsert:
+		if msg := conflicts(zone, sets, set); msg != "" {
+			return sets, msg
+		}
+		if at >= 0 {
+			sets[at] = set
+			return sets, ""
+		}
+		return append(sets, set), ""
+	case route53.ChangeActionDelete:
+		if at < 0 {
+			return sets, "Tried to delete resource record set " + label + " but it was not found"
+		}
+		if !sameRRSet(sets[at], set) {
+			return sets, "Tried to delete resource record set " + label + " but the values provided do not match the current values"
+		}
+		if apexDefault {
+			return sets, fmt.Sprintf("A HostedZone must contain at least one %s record for the zone itself.", set.Type)
+		}
+		return slices.Delete(sets, at, at+1), ""
+	default:
+		return sets, fmt.Sprintf("Invalid action %q for resource record set %s", action, label)
+	}
+}
+
+// conflicts applies the per-name rules: a CNAME stands alone at its name and
+// never at the apex, and SOA exists only at the apex.
+func conflicts(zone string, sets []RRSet, set RRSet) string {
+	if set.Type == "SOA" && set.Name != zone {
+		return fmt.Sprintf("RRSet of type SOA with DNS name %s is not permitted because SOA records are only permitted at the zone apex", set.Name)
+	}
+	if set.Type == "CNAME" && set.Name == zone {
+		return fmt.Sprintf("RRSet of type CNAME with DNS name %s is not permitted at apex in zone %s", set.Name, zone)
+	}
+	for _, s := range sets {
+		if s.Name != set.Name || s.Type == set.Type {
+			continue
+		}
+		if s.Type == "CNAME" || set.Type == "CNAME" {
+			return fmt.Sprintf("RRSet of type %s with DNS name %s is not permitted because a conflicting RRSet of type %s with the same DNS name already exists in zone %s", set.Type, set.Name, s.Type, zone)
+		}
+	}
+	return ""
+}
+
+func sameRRSet(a, b RRSet) bool {
+	return a.TTL == b.TTL && slices.Equal(sortedCopy(a.Values), sortedCopy(b.Values))
+}
+
+func sortedCopy(v []string) []string {
+	c := slices.Clone(v)
+	slices.Sort(c)
+	return c
+}
+
+// toRRSet validates one wire record set and converts it. Routing policies,
+// alias targets and health checks are refused with InvalidInput.
+func toRRSet(zone string, in *route53.ResourceRecordSet) (RRSet, error) {
+	if in.AliasTarget != nil {
+		return RRSet{}, awserrors.Errorf(awserrors.ErrorInvalidInput, "alias records are not supported")
+	}
+	if in.SetIdentifier != nil || in.Weight != nil || in.Region != nil || in.Failover != nil ||
+		in.GeoLocation != nil || in.GeoProximityLocation != nil || in.MultiValueAnswer != nil || in.CidrRoutingConfig != nil {
+		return RRSet{}, awserrors.Errorf(awserrors.ErrorInvalidInput, "only simple routing is supported; SetIdentifier and routing policies are not")
+	}
+	if in.HealthCheckId != nil || in.TrafficPolicyInstanceId != nil {
+		return RRSet{}, awserrors.Errorf(awserrors.ErrorInvalidInput, "health checks and traffic policies are not supported")
+	}
+	rrType := strings.ToUpper(aws.StringValue(in.Type))
+	if !slices.Contains(supportedTypes, rrType) {
+		return RRSet{}, awserrors.Errorf(awserrors.ErrorInvalidInput, "record type %q is not supported", rrType)
+	}
+	name := normalizeName(aws.StringValue(in.Name))
+	if err := validateRecordName(name, zone); err != nil {
+		return RRSet{}, awserrors.Errorf(awserrors.ErrorInvalidChangeBatch, "%s", messageOf(err))
+	}
+	if in.TTL == nil {
+		return RRSet{}, awserrors.Errorf(awserrors.ErrorInvalidChangeBatch, "Invalid request: Expected exactly one of [AliasTarget, all of [TTL, and ResourceRecords]], but found none in Change with [Name=%s, Type=%s, SetIdentifier=null]", name, rrType)
+	}
+	ttl := aws.Int64Value(in.TTL)
+	if ttl < 0 || ttl > maxTTL {
+		return RRSet{}, awserrors.Errorf(awserrors.ErrorInvalidInput, "TTL %d is out of range", ttl)
+	}
+	if len(in.ResourceRecords) == 0 {
+		return RRSet{}, awserrors.Errorf(awserrors.ErrorInvalidInput, "ResourceRecords must contain at least one record")
+	}
+	if len(in.ResourceRecords) > maxValuesPerRecordSet {
+		return RRSet{}, awserrors.Errorf(awserrors.ErrorInvalidChangeBatch, "RRSet %s %s has more than %d values", name, rrType, maxValuesPerRecordSet)
+	}
+	set := RRSet{Name: name, Type: rrType, TTL: ttl}
+	for _, rr := range in.ResourceRecords {
+		v := strings.TrimSpace(aws.StringValue(rr.Value))
+		if err := validateValue(rrType, v); err != nil {
+			return RRSet{}, awserrors.Errorf(awserrors.ErrorInvalidChangeBatch, "Invalid Resource Record: %s for %s %s", err, name, rrType)
+		}
+		set.Values = append(set.Values, v)
+	}
+	if (rrType == "CNAME" || rrType == "SOA") && len(set.Values) > 1 {
+		return RRSet{}, awserrors.Errorf(awserrors.ErrorInvalidChangeBatch, "RRSet of type %s with DNS name %s has more than one value", rrType, name)
+	}
+	return set, nil
+}
+
+// validateValue checks a record's RDATA in its presentation format.
+func validateValue(rrType, v string) error {
+	if v == "" {
+		return fmt.Errorf("empty value")
+	}
+	f := strings.Fields(v)
+	switch rrType {
+	case "A":
+		if ip := net.ParseIP(v); ip == nil || ip.To4() == nil || strings.Contains(v, ":") {
+			return fmt.Errorf("value %q is not a valid IPv4 address", v)
+		}
+	case "AAAA":
+		if ip := net.ParseIP(v); ip == nil || !strings.Contains(v, ":") {
+			return fmt.Errorf("value %q is not a valid IPv6 address", v)
+		}
+	case "CNAME", "NS", "PTR":
+		return validHostValue(v)
+	case "MX":
+		if len(f) != 2 || !validUint16(f[0]) {
+			return fmt.Errorf("value %q is not \"priority host\"", v)
+		}
+		return validHostValue(f[1])
+	case "SRV":
+		if len(f) != 4 || !validUint16(f[0]) || !validUint16(f[1]) || !validUint16(f[2]) {
+			return fmt.Errorf("value %q is not \"priority weight port target\"", v)
+		}
+		return validHostValue(f[3])
+	case "CAA":
+		if len(f) < 3 || !validUint8(f[0]) {
+			return fmt.Errorf("value %q is not \"flags tag value\"", v)
+		}
+	case "SOA":
+		if len(f) != 7 {
+			return fmt.Errorf("value %q is not a seven-field SOA", v)
+		}
+	case "TXT", "SPF":
+		if _, err := characterStrings(v); err != nil {
+			return fmt.Errorf("FATAL problem: %w encountered with '%s'", err, v)
+		}
+	}
+	return nil
+}
+
+func validHostValue(v string) error {
+	if v == "." {
+		return nil
+	}
+	if _, err := splitLabels(normalizeName(v), true); err != nil {
+		return fmt.Errorf("value %q is not a valid host name", v)
+	}
+	return nil
+}
+
+func validUint16(s string) bool {
+	_, err := strconv.ParseUint(s, 10, 16)
+	return err == nil
+}
+
+func validUint8(s string) bool {
+	_, err := strconv.ParseUint(s, 10, 8)
+	return err == nil
+}
+
+// messageOf returns the client message an awserrors error carries.
+func messageOf(err error) string {
+	if _, msg, ok := awserrors.ResolveErrorDetail(err); ok && msg != "" {
+		return msg
+	}
+	return err.Error()
+}

spinifex/domains/route53/ext/ext.go

@@ -0,0 +1,57 @@
+// Package ext holds the wire types of the Spinifex extensions to the Route 53
+// API: the activation a hosted zone needs before it is served, which AWS has
+// no equivalent of. Standard SDKs never call them.
+package ext
+
+import "time"
+
+// Actions served on the route53 signing service beside the AWS model's.
+const (
+	ActionGetHostedZoneActivation    = "GetHostedZoneActivation"
+	ActionVerifyHostedZoneDelegation = "VerifyHostedZoneDelegation"
+)
+
+// GetHostedZoneActivationInput names the zone whose activation to report.
+type GetHostedZoneActivationInput struct {
+	_  struct{} `locationName:"GetHostedZoneActivationRequest" type:"structure"`
+	Id *string  `location:"uri" locationName:"Id" type:"string" required:"true"`
+}
+
+// VerifyHostedZoneDelegationInput names the zone to check now.
+type VerifyHostedZoneDelegationInput struct {
+	_  struct{} `locationName:"VerifyHostedZoneDelegationRequest" type:"structure"`
+	Id *string  `location:"uri" locationName:"Id" type:"string" required:"true"`
+}
+
+// Activation is whether northstar serves a zone, and what the last check of
+// its delegation found.
+type Activation struct {
+	_ struct{} `type:"structure"`
+
+	// State is PENDING_NAMESERVERS, ACTIVE, NAMESERVERS_LOST or SUSPENDED.
+	State  *string `type:"string"`
+	Reason *string `type:"string"`
+	// NameServers are the servers the parent must delegate the zone to.
+	NameServers []*string  `locationNameList:"NameServer" type:"list"`
+	CheckedAt   *time.Time `type:"timestamp"`
+	// DelegatedTo is the NS set the parent published at the last check, and
+	// DelegationSource the parent zone or server that answered.
+	DelegatedTo      []*string `locationNameList:"NameServer" type:"list"`
+	DelegationSource *string   `type:"string"`
+	// Override is set when an operator activated or suspended the zone.
+	Override *string `type:"string"`
+}
+
+// GetHostedZoneActivationOutput reports a zone's activation.
+type GetHostedZoneActivationOutput struct {
+	_            struct{}    `type:"structure"`
+	HostedZoneId *string     `type:"string"`
+	Activation   *Activation `type:"structure"`
+}
+
+// VerifyHostedZoneDelegationOutput reports the activation the check left.
+type VerifyHostedZoneDelegationOutput struct {
+	_            struct{}    `type:"structure"`
+	HostedZoneId *string     `type:"string"`
+	Activation   *Activation `type:"structure"`
+}

spinifex/domains/route53/ids.go

@@ -0,0 +1,46 @@
+package route53
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/hex"
+	"strings"
+)
+
+const (
+	hostedZonePrefix = "/hostedzone/"
+	changePrefix     = "/change/"
+	idAlphabet       = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+	idRandomLength   = 20
+)
+
+// newZoneID returns an ID in the shape AWS issues: Z and 20 uppercase
+// alphanumerics.
+func newZoneID() string { return newID('Z') }
+
+// newChangeID returns a change ID: C and 20 uppercase alphanumerics.
+func newChangeID() string { return newID('C') }
+
+func newID(prefix byte) string {
+	buf := make([]byte, idRandomLength)
+	_, _ = rand.Read(buf)
+	out := make([]byte, 0, idRandomLength+1)
+	out = append(out, prefix)
+	for _, b := range buf {
+		out = append(out, idAlphabet[int(b)%len(idAlphabet)])
+	}
+	return string(out)
+}
+
+// bareZoneID strips the "/hostedzone/" prefix, which AWS accepts on input and
+// returns on output.
+func bareZoneID(id string) string { return strings.TrimPrefix(strings.TrimSpace(id), hostedZonePrefix) }
+
+func bareChangeID(id string) string { return strings.TrimPrefix(strings.TrimSpace(id), changePrefix) }
+
+// refDigest keys a caller reference, which may hold characters a KV key
+// cannot.
+func refDigest(ref string) string {
+	sum := sha256.Sum256([]byte(ref))
+	return hex.EncodeToString(sum[:16])
+}

spinifex/domains/route53/names.go

@@ -0,0 +1,97 @@
+package route53
+
+import (
+	"slices"
+	"strings"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// Route 53 accepts names up to 1024 characters with escapes; the decoded DNS
+// limits below are the ones that matter to a resolver.
+const (
+	maxDomainLength = 253
+	maxLabelLength  = 63
+)
+
+// normalizeName lowercases a DNS name and gives it a trailing dot, the form
+// Route 53 returns names in. An escaped "\052" leading label is "*".
+func normalizeName(name string) string {
+	n := strings.ToLower(strings.TrimSpace(name))
+	if strings.HasPrefix(n, `\052.`) {
+		n = "*" + n[len(`\052`):]
+	}
+	if n != "" && !strings.HasSuffix(n, ".") {
+		n += "."
+	}
+	return n
+}
+
+// validateZoneName checks a normalized hosted-zone name. A zone needs at least
+// two labels: a bare TLD is not a domain a customer can delegate to us.
+func validateZoneName(name string) error {
+	labels, err := splitLabels(name, false)
+	if err != nil {
+		return err
+	}
+	if len(labels) < 2 {
+		return awserrors.Errorf(awserrors.ErrorInvalidDomainName, "%s is a top-level domain and cannot be a hosted zone", name)
+	}
+	return nil
+}
+
+// validateRecordName checks a normalized record name and that it lies within
+// zone. A wildcard is allowed only as the whole leftmost label.
+func validateRecordName(name, zone string) error {
+	if _, err := splitLabels(name, true); err != nil {
+		return err
+	}
+	if !withinZone(name, zone) {
+		return awserrors.Errorf(awserrors.ErrorInvalidChangeBatch, "RRSet with DNS name %s is not permitted in zone %s", name, zone)
+	}
+	return nil
+}
+
+func splitLabels(name string, allowWildcard bool) ([]string, error) {
+	trimmed := strings.TrimSuffix(name, ".")
+	if trimmed == "" || len(trimmed) > maxDomainLength {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidDomainName, "%q is not a valid domain name", name)
+	}
+	labels := strings.Split(trimmed, ".")
+	for i, l := range labels {
+		if l == "*" && allowWildcard && i == 0 {
+			continue
+		}
+		if !validLabel(l) {
+			return nil, awserrors.Errorf(awserrors.ErrorInvalidDomainName, "%q is not a valid domain name", name)
+		}
+	}
+	return labels, nil
+}
+
+// validLabel accepts letters, digits, hyphens and underscores, the last for
+// service labels such as _acme-challenge and _sip._tcp.
+func validLabel(l string) bool {
+	if l == "" || len(l) > maxLabelLength || l[0] == '-' || l[len(l)-1] == '-' {
+		return false
+	}
+	for _, c := range l {
+		if (c < 'a' || c > 'z') && (c < '0' || c > '9') && c != '-' && c != '_' {
+			return false
+		}
+	}
+	return true
+}
+
+// withinZone reports whether name is zone or a name below it.
+func withinZone(name, zone string) bool {
+	return name == zone || strings.HasSuffix(name, "."+zone)
+}
+
+// reversedName orders names the way Route 53 lists them: by label, from the
+// rightmost, so a zone's apex sorts before everything under it.
+func reversedName(name string) string {
+	labels := strings.Split(strings.TrimSuffix(name, "."), ".")
+	slices.Reverse(labels)
+	return strings.Join(labels, ".")
+}

spinifex/domains/route53/realizer.go

@@ -0,0 +1,255 @@
+package route53
+
+import (
+	"context"
+	"crypto/md5" //nolint:gosec // matches the S3 ETag northstar reports, not a security boundary
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"log/slog"
+	"math"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/reconciler"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+)
+
+// Publisher is where realized zones go: the northstar zone store and the
+// servers that load from it.
+type Publisher interface {
+	PutZone(ctx context.Context, apex string, body []byte) error
+	DeleteZone(ctx context.Context, apex string) error
+	// Reload asks every northstar server to load apex. served holds the version
+	// each answering server reports, "" for none; complete is whether every
+	// server answered.
+	Reload(ctx context.Context, apex string) (served []string, complete bool, err error)
+}
+
+const (
+	defaultRealizeResync = time.Minute
+	// realizeRetry is how soon a zone some server has not yet loaded is retried.
+	realizeRetry = 5 * time.Second
+)
+
+// Realizer publishes every ACTIVE zone to northstar, withdraws every zone that
+// stops being ACTIVE, and moves changes to INSYNC once every northstar server
+// serves them. One node realizes at a time, through Elect.
+type Realizer struct {
+	store *Store
+	pub   Publisher
+
+	// Elect wins this pass for the calling node; nil means always.
+	Elect func(ctx context.Context) (release func(), ok bool)
+	// Reserved reports platform zone names, which are never written or deleted.
+	Reserved func(name string) bool
+	// Now is the clock; tests replace it.
+	Now func() time.Time
+	// Resync is the drift backstop; zero means one minute.
+	Resync time.Duration
+}
+
+// NewRealizer returns a Realizer over store that publishes through pub.
+func NewRealizer(store *Store, pub Publisher) *Realizer {
+	return &Realizer{store: store, pub: pub, Now: time.Now}
+}
+
+// Run realizes once, then on every zone or record-set change.
+func (r *Realizer) Run(ctx context.Context) {
+	resync := r.Resync
+	if resync <= 0 {
+		resync = defaultRealizeResync
+	}
+	reconciler.Run(ctx, reconciler.Config{
+		Name: "route53",
+		Sources: []reconciler.Source{
+			reconciler.Fixed(r.store.bucket, keyPrefixZone+">"),
+			reconciler.Fixed(r.store.bucket, keyPrefixRRSets+">"),
+		},
+		Reconcile: r.Reconcile,
+		Resync:    resync,
+	})
+}
+
+// Reconcile is one whole-set pass. A zone a server has not yet loaded asks for
+// a prompt revisit, since nothing in KV changes when it does.
+func (r *Realizer) Reconcile(ctx context.Context) (time.Duration, error) {
+	if r.Elect != nil {
+		release, ok := r.Elect(ctx)
+		if !ok {
+			return 0, nil
+		}
+		defer release()
+	}
+	zones, err := r.store.ListAllZones(ctx)
+	if err != nil {
+		return 0, fmt.Errorf("list hosted zones: %w", err)
+	}
+	published, err := r.store.ListPublished(ctx)
+	if err != nil {
+		return 0, fmt.Errorf("list published zones: %w", err)
+	}
+
+	var revisit time.Duration
+	var errs []error
+	served := map[string]bool{}
+	for apex, zone := range r.servedZones(zones) {
+		doc, _, err := r.store.GetRRSets(ctx, zone.AccountID, zone.ID)
+		if errors.Is(err, kvstore.ErrNotFound) {
+			continue // being deleted; withdrawn below
+		}
+		if err != nil {
+			errs = append(errs, fmt.Errorf("read record sets for %s: %w", apex, err))
+			served[apex] = true // unknown, so leave what is published alone
+			continue
+		}
+		served[apex] = true
+		prev, had := published[apex]
+		pending, err := r.realize(ctx, apex, zone, doc, prev, had)
+		if err != nil {
+			errs = append(errs, err)
+		}
+		if pending || err != nil {
+			revisit = realizeRetry
+		}
+	}
+	for apex, p := range published {
+		if served[apex] {
+			continue
+		}
+		pending, err := r.withdraw(ctx, p)
+		if err != nil {
+			errs = append(errs, err)
+		}
+		if pending || err != nil {
+			revisit = realizeRetry
+		}
+	}
+	if err := r.pruneOrphanChanges(ctx, zones); err != nil {
+		errs = append(errs, err)
+	}
+	return revisit, errors.Join(errs...)
+}
+
+// servedZones picks, for each zone name, the ACTIVE zone northstar serves. Only
+// one should be ACTIVE per name; if more are, the oldest is served, so the
+// answer does not flip between them.
+func (r *Realizer) servedZones(zones []ZoneRecord) map[string]*ZoneRecord {
+	out := map[string]*ZoneRecord{}
+	for i := range zones {
+		z := &zones[i]
+		if z.State != StateActive || (r.Reserved != nil && r.Reserved(z.Name)) {
+			continue
+		}
+		apex := strings.TrimSuffix(z.Name, ".")
+		if cur, ok := out[apex]; ok && (cur.CreatedAt.Before(z.CreatedAt) || (cur.CreatedAt.Equal(z.CreatedAt) && cur.ID < z.ID)) {
+			continue
+		}
+		out[apex] = z
+	}
+	return out
+}
+
+// realize writes the zone if northstar does not already hold this version,
+// then reloads until every server serves it.
+func (r *Realizer) realize(ctx context.Context, apex string, zone *ZoneRecord, doc *RRSetsDoc, prev PublishedZone, had bool) (pending bool, err error) {
+	body, err := renderZone(zone, doc)
+	if err != nil {
+		return false, fmt.Errorf("render %s: %w", apex, err)
+	}
+	sum := md5.Sum(body) //nolint:gosec // see import
+	digest := hex.EncodeToString(sum[:])
+	if had && prev.ZoneID == zone.ID && prev.Digest == digest && prev.InSync {
+		return false, nil
+	}
+	next := PublishedZone{Apex: apex, AccountID: zone.AccountID, ZoneID: zone.ID, Version: doc.Version, Digest: digest}
+	if !had || prev.Digest != digest || prev.ZoneID != zone.ID {
+		if err := r.pub.PutZone(ctx, apex, body); err != nil {
+			return false, fmt.Errorf("write zone %s: %w", apex, err)
+		}
+		if err := r.store.SetPublished(ctx, &next); err != nil {
+			return false, fmt.Errorf("record published zone %s: %w", apex, err)
+		}
+		slog.Info("route53: published zone", "zone", apex, "zone_id", zone.ID, "version", doc.Version)
+	}
+	if !r.reloaded(ctx, apex, digest) {
+		return true, nil
+	}
+	if err := r.store.MarkChangesInSync(ctx, zone.AccountID, zone.ID, doc.Version); err != nil {
+		return false, fmt.Errorf("mark changes in sync for %s: %w", apex, err)
+	}
+	if had && prev.ZoneID != zone.ID {
+		// The name moved to another zone; the one it left is no longer served.
+		if err := r.store.MarkChangesInSync(ctx, prev.AccountID, prev.ZoneID, math.MaxUint64); err != nil {
+			return false, fmt.Errorf("mark changes in sync for %s: %w", apex, err)
+		}
+	}
+	next.InSync = true
+	if err := r.store.SetPublished(ctx, &next); err != nil {
+		return false, fmt.Errorf("record published zone %s: %w", apex, err)
+	}
+	return false, nil
+}
+
+// withdraw deletes a zone northstar should no longer serve.
+func (r *Realizer) withdraw(ctx context.Context, p PublishedZone) (pending bool, err error) {
+	if r.Reserved != nil && r.Reserved(p.Apex+".") {
+		slog.Error("route53: refusing to withdraw a platform zone", "zone", p.Apex)
+		return false, nil
+	}
+	if err := r.pub.DeleteZone(ctx, p.Apex); err != nil {
+		return false, fmt.Errorf("delete zone %s: %w", p.Apex, err)
+	}
+	if !r.reloaded(ctx, p.Apex, "") {
+		return true, nil
+	}
+	if err := r.store.MarkChangesInSync(ctx, p.AccountID, p.ZoneID, math.MaxUint64); err != nil {
+		return false, fmt.Errorf("mark changes in sync for %s: %w", p.Apex, err)
+	}
+	if err := r.store.DeletePublished(ctx, p.Apex); err != nil {
+		return false, fmt.Errorf("forget withdrawn zone %s: %w", p.Apex, err)
+	}
+	slog.Info("route53: withdrew zone", "zone", p.Apex, "zone_id", p.ZoneID)
+	return false, nil
+}
+
+// reloaded reports whether every northstar server now serves digest.
+func (r *Realizer) reloaded(ctx context.Context, apex, digest string) bool {
+	served, complete, err := r.pub.Reload(ctx, apex)
+	if err != nil {
+		slog.Warn("route53: zone reload failed", "zone", apex, "error", err)
+		return false
+	}
+	if !complete {
+		return false
+	}
+	for _, v := range served {
+		if v != digest {
+			return false
+		}
+	}
+	return true
+}
+
+// pruneOrphanChanges deletes the INSYNC changes of deleted zones once they are
+// past the retention GetChange honours.
+func (r *Realizer) pruneOrphanChanges(ctx context.Context, zones []ZoneRecord) error {
+	live := make(map[string]bool, len(zones))
+	for _, z := range zones {
+		live[z.AccountID+"/"+z.ID] = true
+	}
+	changes, err := r.store.ListAllChanges(ctx)
+	if err != nil {
+		return fmt.Errorf("list changes: %w", err)
+	}
+	cutoff := r.Now().Add(-changeRetention)
+	for _, c := range changes {
+		if live[c.AccountID+"/"+c.ZoneID] || c.Status != ChangeStatusInSync || !c.SubmittedAt.Before(cutoff) {
+			continue
+		}
+		if err := r.store.DeleteChange(ctx, c.AccountID, c.ID); err != nil {
+			return fmt.Errorf("delete change %s: %w", c.ID, err)
+		}
+	}
+	return nil
+}

spinifex/domains/route53/realizer_test.go

@@ -0,0 +1,318 @@
+package route53_test
+
+import (
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	r53 "github.com/aws/aws-sdk-go/service/route53"
+	nsconfig "github.com/mulgadc/northstar/pkg/config"
+	toml "github.com/pelletier/go-toml/v2"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/route53"
+)
+
+// fakeNorthstar is the zone store plus a set of servers, each of which loads
+// what the store holds only when it is not lagging.
+type fakeNorthstar struct {
+	mu      sync.Mutex
+	objects map[string][]byte
+	served  []map[string]string
+	lagging map[int]bool
+	puts    int
+	deletes int
+}
+
+func newFakeNorthstar(servers int) *fakeNorthstar {
+	f := &fakeNorthstar{objects: map[string][]byte{}, lagging: map[int]bool{}}
+	for range servers {
+		f.served = append(f.served, map[string]string{})
+	}
+	return f
+}
+
+func (f *fakeNorthstar) PutZone(_ context.Context, apex string, body []byte) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.objects[apex] = body
+	f.puts++
+	return nil
+}
+
+func (f *fakeNorthstar) DeleteZone(_ context.Context, apex string) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	delete(f.objects, apex)
+	f.deletes++
+	return nil
+}
+
+func (f *fakeNorthstar) Reload(_ context.Context, apex string) ([]string, bool, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	var out []string
+	for i, s := range f.served {
+		if f.lagging[i] {
+			continue
+		}
+		if body, ok := f.objects[apex]; ok {
+			sum := md5.Sum(body)
+			s[apex] = hex.EncodeToString(sum[:])
+		} else {
+			delete(s, apex)
+		}
+		out = append(out, s[apex])
+	}
+	return out, len(out) == len(f.served), nil
+}
+
+func (f *fakeNorthstar) zone(t *testing.T, apex string) nsconfig.ConfigArr {
+	t.Helper()
+	f.mu.Lock()
+	body, ok := f.objects[apex]
+	f.mu.Unlock()
+	require.True(t, ok, "%s is not published", apex)
+	var cfg nsconfig.ConfigArr
+	require.NoError(t, toml.Unmarshal(body, &cfg))
+	return cfg
+}
+
+func (f *fakeNorthstar) has(apex string) bool {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	_, ok := f.objects[apex]
+	return ok
+}
+
+func setState(t *testing.T, svc *route53.Service, account, zoneID, state string) {
+	t.Helper()
+	require.NoError(t, svc.Store().MutateZone(ctx, account, strings.TrimPrefix(zoneID, "/hostedzone/"), func(z *route53.ZoneRecord) (bool, error) {
+		z.State = state
+		return true, nil
+	}))
+}
+
+func changeStatus(t *testing.T, svc *route53.Service, account, changeID string) string {
+	t.Helper()
+	out, err := svc.GetChange(ctx, &r53.GetChangeInput{Id: aws.String(changeID)}, account)
+	require.NoError(t, err)
+	return aws.StringValue(out.ChangeInfo.Status)
+}
+
+func newRealizer(svc *route53.Service, ns *fakeNorthstar) *route53.Realizer {
+	r := route53.NewRealizer(svc.Store(), ns)
+	r.Reserved = svc.Reserved
+	return r
+}
+
+func TestRealizerPublishesOnlyActiveZones(t *testing.T) {
+	svc := newService(t)
+	ns := newFakeNorthstar(2)
+	r := newRealizer(svc, ns)
+	id := createZone(t, svc, acctA, "example.com")
+
+	_, err := r.Reconcile(ctx)
+	require.NoError(t, err)
+	assert.False(t, ns.has("example.com"), "a zone pending its nameservers must not be served")
+
+	setState(t, svc, acctA, id, route53.StateActive)
+	out, err := apply(svc, acctA, id, change("CREATE", "www.example.com", "A", 60, "192.0.2.1"))
+	require.NoError(t, err)
+	assert.Equal(t, "PENDING", aws.StringValue(out.ChangeInfo.Status))
+
+	revisit, err := r.Reconcile(ctx)
+	require.NoError(t, err)
+	assert.Zero(t, revisit)
+	zone := ns.zone(t, "example.com")
+	assert.Equal(t, "example.com", zone.Domain.Domain)
+	assert.Equal(t, "ns1.spx3.net.", zone.Domain.SOA)
+	assert.Contains(t, zone.Records, nsconfig.Records{Domain: "www.", Type: 1, Class: 1, TTL: 60, Address: "192.0.2.1"})
+	assert.Contains(t, zone.Records, nsconfig.Records{Domain: "", Type: 2, Class: 1, TTL: 172800, Address: "ns1.spx3.net."})
+	assert.Equal(t, "INSYNC", changeStatus(t, svc, acctA, aws.StringValue(out.ChangeInfo.Id)))
+
+	puts := ns.puts
+	_, err = r.Reconcile(ctx)
+	require.NoError(t, err)
+	assert.Equal(t, puts, ns.puts, "an unchanged zone is not rewritten")
+}
+
+// A change is INSYNC only once every northstar server serves it, so a server
+// that has not loaded the new version holds it PENDING and asks for a retry.
+func TestRealizerHoldsChangesPendingUntilEveryServerLoads(t *testing.T) {
+	svc := newService(t)
+	ns := newFakeNorthstar(3)
+	r := newRealizer(svc, ns)
+	id := createZone(t, svc, acctA, "example.com")
+	setState(t, svc, acctA, id, route53.StateActive)
+	out, err := apply(svc, acctA, id, change("CREATE", "www.example.com", "A", 60, "192.0.2.1"))
+	require.NoError(t, err)
+
+	ns.lagging[2] = true
+	revisit, err := r.Reconcile(ctx)
+	require.NoError(t, err)
+	assert.Positive(t, revisit)
+	assert.Equal(t, "PENDING", changeStatus(t, svc, acctA, aws.StringValue(out.ChangeInfo.Id)))
+
+	ns.lagging[2] = false
+	puts := ns.puts
+	revisit, err = r.Reconcile(ctx)
+	require.NoError(t, err)
+	assert.Zero(t, revisit)
+	assert.Equal(t, puts, ns.puts, "the retry reloads without rewriting")
+	assert.Equal(t, "INSYNC", changeStatus(t, svc, acctA, aws.StringValue(out.ChangeInfo.Id)))
+}
+
+func TestRealizerWithdrawsDeletedAndDeactivatedZones(t *testing.T) {
+	svc := newService(t)
+	ns := newFakeNorthstar(2)
+	r := newRealizer(svc, ns)
+	gone := createZone(t, svc, acctA, "gone.com")
+	lost := createZone(t, svc, acctA, "lost.com")
+	setState(t, svc, acctA, gone, route53.StateActive)
+	setState(t, svc, acctA, lost, route53.StateActive)
+	_, err := r.Reconcile(ctx)
+	require.NoError(t, err)
+	require.True(t, ns.has("gone.com"))
+	require.True(t, ns.has("lost.com"))
+
+	del, err := svc.DeleteHostedZone(ctx, &r53.DeleteHostedZoneInput{Id: aws.String(gone)}, acctA)
+	require.NoError(t, err)
+	assert.Equal(t, "PENDING", aws.StringValue(del.ChangeInfo.Status))
+	setState(t, svc, acctA, lost, route53.StateNameserversLost)
+
+	_, err = r.Reconcile(ctx)
+	require.NoError(t, err)
+	assert.False(t, ns.has("gone.com"))
+	assert.False(t, ns.has("lost.com"))
+	assert.Equal(t, "INSYNC", changeStatus(t, svc, acctA, aws.StringValue(del.ChangeInfo.Id)))
+	published, err := svc.Store().ListPublished(ctx)
+	require.NoError(t, err)
+	assert.Empty(t, published)
+}
+
+func TestRealizerNeverWritesAReservedZone(t *testing.T) {
+	svc := newService(t)
+	ns := newFakeNorthstar(1)
+	r := newRealizer(svc, ns)
+	id := createZone(t, svc, acctA, "example.com")
+	setState(t, svc, acctA, id, route53.StateActive)
+	r.Reserved = func(name string) bool { return name == "example.com." }
+
+	_, err := r.Reconcile(ctx)
+	require.NoError(t, err)
+	assert.False(t, ns.has("example.com"))
+}
+
+// Two accounts may hold zones of the same name; if both are ACTIVE, the older
+// one is served and the answer does not flip between them.
+func TestRealizerServesTheOldestOfTwoActiveZones(t *testing.T) {
+	svc := newService(t)
+	clock := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
+	svc.Now = func() time.Time { clock = clock.Add(time.Second); return clock }
+	ns := newFakeNorthstar(1)
+	r := newRealizer(svc, ns)
+	first := createZone(t, svc, acctA, "example.com")
+	second := createZone(t, svc, acctB, "example.com")
+	_, err := apply(svc, acctB, second, change("CREATE", "www.example.com", "A", 60, "198.51.100.1"))
+	require.NoError(t, err)
+	setState(t, svc, acctB, second, route53.StateActive)
+	setState(t, svc, acctA, first, route53.StateActive)
+
+	for range 2 {
+		_, err = r.Reconcile(ctx)
+		require.NoError(t, err)
+		for _, rec := range ns.zone(t, "example.com").Records {
+			assert.NotEqual(t, "198.51.100.1", rec.Address, "the newer zone must not be served")
+		}
+	}
+}
+
+func TestRealizerDoesNothingWhenNotElected(t *testing.T) {
+	svc := newService(t)
+	ns := newFakeNorthstar(1)
+	r := newRealizer(svc, ns)
+	r.Elect = func(context.Context) (func(), bool) { return nil, false }
+	id := createZone(t, svc, acctA, "example.com")
+	setState(t, svc, acctA, id, route53.StateActive)
+
+	_, err := r.Reconcile(ctx)
+	require.NoError(t, err)
+	assert.False(t, ns.has("example.com"))
+}
+
+func TestRealizerRendersEveryRecordType(t *testing.T) {
+	svc := newService(t)
+	ns := newFakeNorthstar(1)
+	r := newRealizer(svc, ns)
+	id := createZone(t, svc, acctA, "example.com")
+	setState(t, svc, acctA, id, route53.StateActive)
+	long := strings.Repeat("k", 255)
+	_, err := apply(svc, acctA, id,
+		change("CREATE", "example.com", "MX", 300, "10 mail.example.com."),
+		change("CREATE", "_xmpp._tcp.example.com", "SRV", 300, "1 10 5269 xmpp.example.com."),
+		change("CREATE", "example.com", "CAA", 300, `0 issue "letsencrypt.org"`),
+		change("CREATE", "dkim.example.com", "TXT", 300, `"`+long+`" "tail"`, `"say \"hi\""`),
+		change("CREATE", "*.example.com", "CNAME", 300, "www.example.com."),
+		change("CREATE", "v6.example.com", "AAAA", 300, "2001:db8::1"),
+	)
+	require.NoError(t, err)
+
+	_, err = r.Reconcile(ctx)
+	require.NoError(t, err)
+	records := ns.zone(t, "example.com").Records
+	assert.Contains(t, records, nsconfig.Records{Domain: "", Type: 15, Class: 1, TTL: 300, Preference: 10, Address: "mail.example.com."})
+	assert.Contains(t, records, nsconfig.Records{Domain: "_xmpp._tcp.", Type: 33, Class: 1, TTL: 300, Priority: 1, Weight: 10, Port: 5269, Address: "xmpp.example.com."})
+	assert.Contains(t, records, nsconfig.Records{Domain: "", Type: 257, Class: 1, TTL: 300, CAATag: "issue", Address: "letsencrypt.org"})
+	assert.Contains(t, records, nsconfig.Records{Domain: "dkim.", Type: 16, Class: 1, TTL: 300, Strings: []string{long, "tail"}})
+	assert.Contains(t, records, nsconfig.Records{Domain: "dkim.", Type: 16, Class: 1, TTL: 300, Strings: []string{`say \"hi\"`}})
+	assert.Contains(t, records, nsconfig.Records{Domain: "*.", Type: 5, Class: 1, TTL: 300, Address: "www.example.com."})
+	assert.Contains(t, records, nsconfig.Records{Domain: "v6.", Type: 28, Class: 1, TTL: 300, Address: "2001:db8::1"})
+}
+
+func TestTXTValuesAreValidatedAsCharacterStrings(t *testing.T) {
+	svc := newService(t)
+	id := createZone(t, svc, acctA, "example.com")
+	for _, v := range []string{
+		`"` + strings.Repeat("a", 256) + `"`,
+		`"unterminated`,
+		`"ok" trailing`,
+		`"bad escape \999"`,
+		strings.Repeat("b", 256),
+	} {
+		_, err := apply(svc, acctA, id, change("UPSERT", "t.example.com", "TXT", 60, v))
+		requireCode(t, err, "InvalidChangeBatch")
+	}
+	for _, v := range []string{`"` + strings.Repeat("a", 255) + `"`, `"\255\"x"`, "unquoted", `"a" "b"`} {
+		_, err := apply(svc, acctA, id, change("UPSERT", "t.example.com", "TXT", 60, v))
+		require.NoError(t, err, v)
+	}
+}
+
+func TestRealizerPrunesTheChangesOfDeletedZones(t *testing.T) {
+	svc := newService(t)
+	clock := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
+	svc.Now = func() time.Time { return clock }
+	r := newRealizer(svc, newFakeNorthstar(1))
+	r.Now = func() time.Time { return clock }
+	id := createZone(t, svc, acctA, "example.com")
+	del, err := svc.DeleteHostedZone(ctx, &r53.DeleteHostedZoneInput{Id: aws.String(id)}, acctA)
+	require.NoError(t, err)
+	changeID := aws.StringValue(del.ChangeInfo.Id)
+
+	_, err = r.Reconcile(ctx)
+	require.NoError(t, err)
+	assert.Equal(t, "INSYNC", changeStatus(t, svc, acctA, changeID), "kept while GetChange may still ask")
+
+	clock = clock.Add(8 * 24 * time.Hour)
+	_, err = r.Reconcile(ctx)
+	require.NoError(t, err)
+	_, err = svc.GetChange(ctx, &r53.GetChangeInput{Id: aws.String(changeID)}, acctA)
+	requireCode(t, err, "NoSuchChange")
+}

spinifex/domains/route53/render.go

@@ -0,0 +1,110 @@
+package route53
+
+import (
+	"cmp"
+	"fmt"
+	"slices"
+	"strconv"
+	"strings"
+
+	"github.com/miekg/dns"
+	nsconfig "github.com/mulgadc/northstar/pkg/config"
+)
+
+// renderZone turns a zone's record sets into the northstar zone document. The
+// output depends only on its inputs, so the same version always renders to
+// the same bytes and its digest identifies what northstar should be serving.
+//
+// Northstar synthesizes the SOA from the first field of the stored SOA value,
+// so the record set itself is not emitted.
+func renderZone(zone *ZoneRecord, doc *RRSetsDoc) ([]byte, error) {
+	apex := strings.TrimSuffix(zone.Name, ".")
+	cfg := nsconfig.ConfigArr{
+		Version: 1.0,
+		Domain: nsconfig.Domain{
+			Domain:   apex,
+			Modified: doc.UpdatedAt.UTC(),
+			Active:   true,
+		},
+		Defaults: nsconfig.Defaults{TTL: 300, Type: dns.TypeA, Class: dns.ClassINET},
+	}
+	sets := slices.Clone(doc.Sets)
+	slices.SortFunc(sets, func(a, b RRSet) int {
+		return cmp.Or(cmp.Compare(reversedName(a.Name), reversedName(b.Name)), cmp.Compare(a.Type, b.Type))
+	})
+	for _, set := range sets {
+		if set.Type == "SOA" {
+			if f := strings.Fields(set.Values[0]); len(f) > 0 {
+				cfg.Domain.SOA = f[0]
+			}
+			continue
+		}
+		rtype, ok := dns.StringToType[set.Type]
+		if !ok {
+			return nil, fmt.Errorf("record set %s has unknown type %s", set.Name, set.Type)
+		}
+		label := relativeLabel(set.Name, zone.Name)
+		for _, v := range set.Values {
+			rec, err := renderRecord(label, rtype, uint32(set.TTL), v) //nolint:gosec // TTL is validated to at most maxTTL
+			if err != nil {
+				return nil, fmt.Errorf("record set %s %s: %w", set.Name, set.Type, err)
+			}
+			cfg.Records = append(cfg.Records, rec)
+		}
+	}
+	return nsconfig.RenderZone(cfg)
+}
+
+func renderRecord(label string, rtype uint16, ttl uint32, v string) (nsconfig.Records, error) {
+	rec := nsconfig.Records{Domain: label, Type: rtype, Class: dns.ClassINET, TTL: ttl}
+	f := strings.Fields(v)
+	switch rtype {
+	case dns.TypeMX:
+		rec.Preference = parseUint16(f[0])
+		rec.Address = f[1]
+	case dns.TypeSRV:
+		rec.Priority, rec.Weight, rec.Port = parseUint16(f[0]), parseUint16(f[1]), parseUint16(f[2])
+		rec.Address = f[3]
+	case dns.TypeCAA:
+		flag, _ := strconv.ParseUint(f[0], 10, 8)
+		rec.CAAFlag = uint8(flag)
+		rec.CAATag = f[1]
+		rec.Address = strings.Trim(afterFields(v, 2), `"`)
+	case dns.TypeTXT, dns.TypeSPF:
+		strs, err := characterStrings(v)
+		if err != nil {
+			return rec, err
+		}
+		rec.Strings = strs
+	default:
+		rec.Address = v
+	}
+	return rec, nil
+}
+
+// afterFields returns v with its first n whitespace-separated fields removed.
+func afterFields(v string, n int) string {
+	rest := strings.TrimSpace(v)
+	for range n {
+		i := strings.IndexAny(rest, " \t")
+		if i < 0 {
+			return ""
+		}
+		rest = strings.TrimSpace(rest[i:])
+	}
+	return rest
+}
+
+func parseUint16(s string) uint16 {
+	n, _ := strconv.ParseUint(s, 10, 16)
+	return uint16(n)
+}
+
+// relativeLabel is name relative to zone in northstar's form: "" for the apex
+// and "www." for www.<zone>.
+func relativeLabel(name, zone string) string {
+	if name == zone {
+		return ""
+	}
+	return strings.TrimSuffix(name, zone)
+}

spinifex/domains/route53/service.go

@@ -0,0 +1,634 @@
+// Package route53 is the daemon-side Route 53 control plane: public hosted
+// zones and their record sets in JetStream KV, change tracking for GetChange,
+// and the activation state that decides whether northstar serves a zone.
+package route53
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/route53"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+)
+
+const (
+	defaultMaxZonesPerAccount = 500
+	maxCallerReferenceLength  = 128
+	maxCommentLength          = 256
+	maxListZones              = 100
+	maxListRecordSets         = 300
+	maxTagsPerResource        = 50
+	maxTagResources           = 10
+	changeRetention           = 7 * 24 * time.Hour
+	commitAttempts            = 3
+)
+
+// Service implements the Route 53 operations the gateway forwards over NATS.
+// Methods take the account from the request, never from the input.
+type Service struct {
+	store *Store
+
+	// Nameservers returns the published nameserver host names, without a
+	// trailing dot, that a new zone is delegated to.
+	Nameservers func() []string
+	// Reserved reports whether a zone name belongs to the platform and may not
+	// be created by a tenant.
+	Reserved func(name string) bool
+	// Now is the clock; tests replace it.
+	Now func() time.Time
+	// MaxZonesPerAccount caps CreateHostedZone; zero means the AWS default.
+	MaxZonesPerAccount int
+	// Verifier checks a delegation on demand; nil leaves zones to the schedule.
+	Verifier *Verifier
+}
+
+// NewService returns a Service over store.
+func NewService(store *Store) *Service {
+	return &Service{store: store, Now: time.Now}
+}
+
+// Store returns the service's store, for the realizer and verifier.
+func (s *Service) Store() *Store { return s.store }
+
+func (s *Service) CreateHostedZone(ctx context.Context, in *route53.CreateHostedZoneInput, account string) (*route53.CreateHostedZoneOutput, error) {
+	ref := aws.StringValue(in.CallerReference)
+	if ref == "" || len(ref) > maxCallerReferenceLength {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "CallerReference must be 1 to %d characters", maxCallerReferenceLength)
+	}
+	if in.VPC != nil || (in.HostedZoneConfig != nil && aws.BoolValue(in.HostedZoneConfig.PrivateZone)) {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "private hosted zones are not supported")
+	}
+	if in.DelegationSetId != nil {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "reusable delegation sets are not supported")
+	}
+	comment := ""
+	if in.HostedZoneConfig != nil {
+		comment = aws.StringValue(in.HostedZoneConfig.Comment)
+	}
+	if len(comment) > maxCommentLength {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "Comment must be at most %d characters", maxCommentLength)
+	}
+	name := normalizeName(aws.StringValue(in.Name))
+	if err := validateZoneName(name); err != nil {
+		return nil, err
+	}
+	if s.Reserved != nil && s.Reserved(name) {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidDomainName, "%s is reserved by this cluster and cannot be a hosted zone", name)
+	}
+	var nameservers []string
+	if s.Nameservers != nil {
+		nameservers = s.Nameservers()
+	}
+	if len(nameservers) == 0 {
+		return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "no nameservers are published for this cluster")
+	}
+
+	existing, err := s.store.ListAccountZones(ctx, account)
+	if err != nil {
+		return nil, internal("list hosted zones", err)
+	}
+	limit := s.MaxZonesPerAccount
+	if limit == 0 {
+		limit = defaultMaxZonesPerAccount
+	}
+	if len(existing) >= limit {
+		return nil, awserrors.Errorf(awserrors.ErrorTooManyHostedZones, "account %s already has %d hosted zones", account, len(existing))
+	}
+
+	id := newZoneID()
+	held, err := s.store.ClaimCallerReference(ctx, account, ref, id)
+	if err != nil {
+		return nil, internal("claim caller reference", err)
+	}
+	if held != "" {
+		if _, err := s.store.GetZone(ctx, account, held); !errors.Is(err, kvstore.ErrNotFound) {
+			return nil, awserrors.Errorf(awserrors.ErrorHostedZoneAlreadyExists, "A hosted zone has already been created with the specified caller reference.")
+		}
+		// The earlier claim's zone was never written: take the reference over.
+		if err := s.store.ReleaseCallerReference(ctx, account, ref); err != nil {
+			return nil, internal("release caller reference", err)
+		}
+		if held, err = s.store.ClaimCallerReference(ctx, account, ref, id); err != nil || held != "" {
+			return nil, awserrors.Errorf(awserrors.ErrorPriorRequestNotComplete, "another request with this caller reference is in progress")
+		}
+	}
+
+	now := s.Now().UTC()
+	zone := &ZoneRecord{
+		ID: id, AccountID: account, Name: name, CallerReference: ref, Comment: comment,
+		CreatedAt: now, Nameservers: nameservers, State: StatePendingNameservers,
+	}
+	doc := &RRSetsDoc{ZoneID: id, Version: 1, UpdatedAt: now, Sets: defaultRecordSets(name, nameservers)}
+	if err := s.store.CreateZone(ctx, zone, doc); err != nil {
+		_ = s.store.ReleaseCallerReference(ctx, account, ref)
+		return nil, internal("create hosted zone", err)
+	}
+	change, err := s.recordChange(ctx, zone, doc.Version, "")
+	if err != nil {
+		return nil, err
+	}
+	return &route53.CreateHostedZoneOutput{
+		HostedZone:    hostedZone(zone, len(doc.Sets)),
+		ChangeInfo:    change,
+		DelegationSet: delegationSet(zone),
+		Location:      aws.String("/2013-04-01" + hostedZonePrefix + id),
+	}, nil
+}
+
+func (s *Service) GetHostedZone(ctx context.Context, in *route53.GetHostedZoneInput, account string) (*route53.GetHostedZoneOutput, error) {
+	zone, doc, _, err := s.zoneWithSets(ctx, account, aws.StringValue(in.Id))
+	if err != nil {
+		return nil, err
+	}
+	return &route53.GetHostedZoneOutput{HostedZone: hostedZone(zone, len(doc.Sets)), DelegationSet: delegationSet(zone)}, nil
+}
+
+func (s *Service) GetHostedZoneCount(ctx context.Context, _ *route53.GetHostedZoneCountInput, account string) (*route53.GetHostedZoneCountOutput, error) {
+	zones, err := s.store.ListAccountZones(ctx, account)
+	if err != nil {
+		return nil, internal("list hosted zones", err)
+	}
+	return &route53.GetHostedZoneCountOutput{HostedZoneCount: aws.Int64(int64(len(zones)))}, nil
+}
+
+func (s *Service) ListHostedZones(ctx context.Context, in *route53.ListHostedZonesInput, account string) (*route53.ListHostedZonesOutput, error) {
+	limit, err := maxItems(in.MaxItems, maxListZones)
+	if err != nil {
+		return nil, err
+	}
+	if in.DelegationSetId != nil {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "reusable delegation sets are not supported")
+	}
+	zones, err := s.sortedZones(ctx, account, func(a, b ZoneRecord) int { return strings.Compare(a.ID, b.ID) })
+	if err != nil {
+		return nil, err
+	}
+	if aws.StringValue(in.HostedZoneType) == route53.HostedZoneTypePrivateHostedZone {
+		zones = nil
+	}
+	marker := bareZoneID(aws.StringValue(in.Marker))
+	start := 0
+	if marker != "" {
+		start = slices.IndexFunc(zones, func(z ZoneRecord) bool { return z.ID >= marker })
+		if start < 0 {
+			start = len(zones)
+		}
+	}
+	page, truncated := pageOf(zones, start, limit)
+	out := &route53.ListHostedZonesOutput{
+		HostedZones: s.hostedZones(ctx, page),
+		IsTruncated: aws.Bool(truncated),
+		MaxItems:    aws.String(strconv.Itoa(limit)),
+		Marker:      in.Marker,
+	}
+	if out.Marker == nil {
+		out.Marker = aws.String("")
+	}
+	if truncated {
+		out.NextMarker = aws.String(zones[start+limit].ID)
+	}
+	return out, nil
+}
+
+func (s *Service) ListHostedZonesByName(ctx context.Context, in *route53.ListHostedZonesByNameInput, account string) (*route53.ListHostedZonesByNameOutput, error) {
+	limit, err := maxItems(in.MaxItems, maxListZones)
+	if err != nil {
+		return nil, err
+	}
+	dnsName := normalizeName(aws.StringValue(in.DNSName))
+	startID := bareZoneID(aws.StringValue(in.HostedZoneId))
+	if startID != "" && dnsName == "" {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "HostedZoneId requires DNSName")
+	}
+	zones, err := s.sortedZones(ctx, account, compareByName)
+	if err != nil {
+		return nil, err
+	}
+	start := 0
+	if dnsName != "" {
+		from := ZoneRecord{Name: dnsName, ID: startID}
+		start = slices.IndexFunc(zones, func(z ZoneRecord) bool { return compareByName(z, from) >= 0 })
+		if start < 0 {
+			start = len(zones)
+		}
+	}
+	page, truncated := pageOf(zones, start, limit)
+	out := &route53.ListHostedZonesByNameOutput{
+		HostedZones:  s.hostedZones(ctx, page),
+		IsTruncated:  aws.Bool(truncated),
+		MaxItems:     aws.String(strconv.Itoa(limit)),
+		DNSName:      in.DNSName,
+		HostedZoneId: in.HostedZoneId,
+	}
+	if truncated {
+		next := zones[start+limit]
+		out.NextDNSName = aws.String(next.Name)
+		out.NextHostedZoneId = aws.String(next.ID)
+	}
+	return out, nil
+}
+
+func (s *Service) UpdateHostedZoneComment(ctx context.Context, in *route53.UpdateHostedZoneCommentInput, account string) (*route53.UpdateHostedZoneCommentOutput, error) {
+	comment := aws.StringValue(in.Comment)
+	if len(comment) > maxCommentLength {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "Comment must be at most %d characters", maxCommentLength)
+	}
+	id := bareZoneID(aws.StringValue(in.Id))
+	err := s.store.MutateZone(ctx, account, id, func(z *ZoneRecord) (bool, error) {
+		if z.Comment == comment {
+			return false, nil
+		}
+		z.Comment = comment
+		return true, nil
+	})
+	if err != nil {
+		return nil, s.zoneErr(id, err)
+	}
+	return s.updatedZone(ctx, account, id)
+}
+
+func (s *Service) updatedZone(ctx context.Context, account, id string) (*route53.UpdateHostedZoneCommentOutput, error) {
+	zone, doc, _, err := s.zoneWithSets(ctx, account, id)
+	if err != nil {
+		return nil, err
+	}
+	return &route53.UpdateHostedZoneCommentOutput{HostedZone: hostedZone(zone, len(doc.Sets))}, nil
+}
+
+func (s *Service) DeleteHostedZone(ctx context.Context, in *route53.DeleteHostedZoneInput, account string) (*route53.DeleteHostedZoneOutput, error) {
+	zone, doc, rev, err := s.zoneWithSets(ctx, account, aws.StringValue(in.Id))
+	if err != nil {
+		return nil, err
+	}
+	for _, set := range doc.Sets {
+		if set.Name != zone.Name || (set.Type != "SOA" && set.Type != "NS") {
+			return nil, awserrors.Errorf(awserrors.ErrorHostedZoneNotEmpty, "The specified hosted zone contains non-required resource record sets and so cannot be deleted.")
+		}
+	}
+	if err := s.store.DeleteZone(ctx, zone, rev); err != nil {
+		if errors.Is(err, kvstore.ErrConflict) {
+			return nil, awserrors.Errorf(awserrors.ErrorPriorRequestNotComplete, "the hosted zone changed while it was being deleted")
+		}
+		return nil, internal("delete hosted zone", err)
+	}
+	// The deletion's change outlives the zone, so GetChange can report it.
+	change, err := s.recordChange(ctx, zone, doc.Version+1, "")
+	if err != nil {
+		return nil, err
+	}
+	return &route53.DeleteHostedZoneOutput{ChangeInfo: change}, nil
+}
+
+func (s *Service) ChangeResourceRecordSets(ctx context.Context, in *route53.ChangeResourceRecordSetsInput, account string) (*route53.ChangeResourceRecordSetsOutput, error) {
+	var comment string
+	if in.ChangeBatch != nil {
+		comment = aws.StringValue(in.ChangeBatch.Comment)
+	}
+	if len(comment) > maxCommentLength {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "Comment must be at most %d characters", maxCommentLength)
+	}
+	for range commitAttempts {
+		zone, doc, rev, err := s.zoneWithSets(ctx, account, aws.StringValue(in.HostedZoneId))
+		if err != nil {
+			return nil, err
+		}
+		sets, err := applyChangeBatch(zone.Name, doc.Sets, in.ChangeBatch)
+		if err != nil {
+			return nil, err
+		}
+		next := &RRSetsDoc{ZoneID: zone.ID, Version: doc.Version + 1, UpdatedAt: s.Now().UTC(), Sets: sets}
+		if err := s.store.CommitRRSets(ctx, account, next, rev); err != nil {
+			if errors.Is(err, kvstore.ErrConflict) {
+				continue
+			}
+			return nil, internal("commit record sets", err)
+		}
+		change, err := s.recordChange(ctx, zone, next.Version, comment)
+		if err != nil {
+			return nil, err
+		}
+		return &route53.ChangeResourceRecordSetsOutput{ChangeInfo: change}, nil
+	}
+	return nil, awserrors.Errorf(awserrors.ErrorPriorRequestNotComplete, "the hosted zone is being changed by another request")
+}
+
+func (s *Service) ListResourceRecordSets(ctx context.Context, in *route53.ListResourceRecordSetsInput, account string) (*route53.ListResourceRecordSetsOutput, error) {
+	limit, err := maxItems(in.MaxItems, maxListRecordSets)
+	if err != nil {
+		return nil, err
+	}
+	if in.StartRecordIdentifier != nil {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "StartRecordIdentifier is not supported: no record set has a SetIdentifier")
+	}
+	startType := strings.ToUpper(aws.StringValue(in.StartRecordType))
+	startName := normalizeName(aws.StringValue(in.StartRecordName))
+	if startType != "" && startName == "" {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "The input is not valid: StartRecordType requires StartRecordName")
+	}
+	_, doc, _, err := s.zoneWithSets(ctx, account, aws.StringValue(in.HostedZoneId))
+	if err != nil {
+		return nil, err
+	}
+	sets := slices.Clone(doc.Sets)
+	slices.SortFunc(sets, compareRRSets)
+	start := 0
+	if startName != "" {
+		from := RRSet{Name: startName, Type: startType}
+		start = slices.IndexFunc(sets, func(r RRSet) bool { return compareRRSets(r, from) >= 0 })
+		if start < 0 {
+			start = len(sets)
+		}
+	}
+	page, truncated := pageOf(sets, start, limit)
+	out := &route53.ListResourceRecordSetsOutput{
+		ResourceRecordSets: make([]*route53.ResourceRecordSet, 0, len(page)),
+		IsTruncated:        aws.Bool(truncated),
+		MaxItems:           aws.String(strconv.Itoa(limit)),
+	}
+	for _, r := range page {
+		out.ResourceRecordSets = append(out.ResourceRecordSets, wireRRSet(r))
+	}
+	if truncated {
+		next := sets[start+limit]
+		out.NextRecordName = aws.String(escapeName(next.Name))
+		out.NextRecordType = aws.String(next.Type)
+	}
+	return out, nil
+}
+
+func (s *Service) GetChange(ctx context.Context, in *route53.GetChangeInput, account string) (*route53.GetChangeOutput, error) {
+	id := bareChangeID(aws.StringValue(in.Id))
+	c, err := s.store.GetChange(ctx, account, id)
+	if errors.Is(err, kvstore.ErrNotFound) {
+		return nil, awserrors.Errorf(awserrors.ErrorNoSuchChange, "A change with the specified change ID does not exist: %s", id)
+	}
+	if err != nil {
+		return nil, internal("get change", err)
+	}
+	return &route53.GetChangeOutput{ChangeInfo: changeInfo(c)}, nil
+}
+
+func (s *Service) ChangeTagsForResource(ctx context.Context, in *route53.ChangeTagsForResourceInput, account string) (*route53.ChangeTagsForResourceOutput, error) {
+	id, err := taggedZoneID(aws.StringValue(in.ResourceType), aws.StringValue(in.ResourceId))
+	if err != nil {
+		return nil, err
+	}
+	for _, t := range in.AddTags {
+		key := aws.StringValue(t.Key)
+		if key == "" || len(key) > 128 || len(aws.StringValue(t.Value)) > 256 {
+			return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "tag keys must be 1 to 128 characters and values at most 256")
+		}
+	}
+	err = s.store.MutateZone(ctx, account, id, func(z *ZoneRecord) (bool, error) {
+		if z.Tags == nil {
+			z.Tags = map[string]string{}
+		}
+		for _, k := range in.RemoveTagKeys {
+			delete(z.Tags, aws.StringValue(k))
+		}
+		for _, t := range in.AddTags {
+			z.Tags[aws.StringValue(t.Key)] = aws.StringValue(t.Value)
+		}
+		if len(z.Tags) > maxTagsPerResource {
+			return false, awserrors.Errorf(awserrors.ErrorRoute53TooManyTagsForZone, "a hosted zone can have at most %d tags", maxTagsPerResource)
+		}
+		return true, nil
+	})
+	if err != nil {
+		return nil, s.zoneErr(id, err)
+	}
+	return &route53.ChangeTagsForResourceOutput{}, nil
+}
+
+func (s *Service) ListTagsForResource(ctx context.Context, in *route53.ListTagsForResourceInput, account string) (*route53.ListTagsForResourceOutput, error) {
+	id, err := taggedZoneID(aws.StringValue(in.ResourceType), aws.StringValue(in.ResourceId))
+	if err != nil {
+		return nil, err
+	}
+	zone, err := s.store.GetZone(ctx, account, id)
+	if err != nil {
+		return nil, s.zoneErr(id, err)
+	}
+	return &route53.ListTagsForResourceOutput{ResourceTagSet: tagSet(zone)}, nil
+}
+
+func (s *Service) ListTagsForResources(ctx context.Context, in *route53.ListTagsForResourcesInput, account string) (*route53.ListTagsForResourcesOutput, error) {
+	if len(in.ResourceIds) == 0 || len(in.ResourceIds) > maxTagResources {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "ResourceIds must name 1 to %d resources", maxTagResources)
+	}
+	out := &route53.ListTagsForResourcesOutput{ResourceTagSets: []*route53.ResourceTagSet{}}
+	for _, rid := range in.ResourceIds {
+		id, err := taggedZoneID(aws.StringValue(in.ResourceType), aws.StringValue(rid))
+		if err != nil {
+			return nil, err
+		}
+		zone, err := s.store.GetZone(ctx, account, id)
+		if err != nil {
+			return nil, s.zoneErr(id, err)
+		}
+		out.ResourceTagSets = append(out.ResourceTagSets, tagSet(zone))
+	}
+	return out, nil
+}
+
+// zoneWithSets loads an account's zone and its record-set document.
+func (s *Service) zoneWithSets(ctx context.Context, account, rawID string) (*ZoneRecord, *RRSetsDoc, uint64, error) {
+	id := bareZoneID(rawID)
+	if id == "" {
+		return nil, nil, 0, awserrors.Errorf(awserrors.ErrorInvalidInput, "a hosted zone ID is required")
+	}
+	zone, err := s.store.GetZone(ctx, account, id)
+	if err != nil {
+		return nil, nil, 0, s.zoneErr(id, err)
+	}
+	doc, rev, err := s.store.GetRRSets(ctx, account, id)
+	if err != nil {
+		return nil, nil, 0, s.zoneErr(id, err)
+	}
+	return zone, doc, rev, nil
+}
+
+func (s *Service) zoneErr(id string, err error) error {
+	if errors.Is(err, kvstore.ErrNotFound) {
+		return awserrors.Errorf(awserrors.ErrorNoSuchHostedZone, "No hosted zone found with ID: %s", id)
+	}
+	if _, _, ok := awserrors.ResolveErrorDetail(err); ok {
+		return err
+	}
+	return internal("read hosted zone", err)
+}
+
+// recordChange stores the change a write produced. A zone that is not being
+// served is in sync as soon as the write is durable; an active one waits for
+// every northstar server to load the version.
+func (s *Service) recordChange(ctx context.Context, zone *ZoneRecord, version uint64, comment string) (*route53.ChangeInfo, error) {
+	now := s.Now().UTC()
+	c := &ChangeRecord{
+		ID: newChangeID(), AccountID: zone.AccountID, ZoneID: zone.ID, Version: version,
+		Status: ChangeStatusInSync, Comment: comment, SubmittedAt: now,
+	}
+	if zone.State == StateActive {
+		c.Status = ChangeStatusPending
+	}
+	if err := s.store.PutChange(ctx, c); err != nil {
+		return nil, internal("record change", err)
+	}
+	_ = s.store.PruneChanges(ctx, zone.AccountID, zone.ID, now.Add(-changeRetention), false)
+	return changeInfo(c), nil
+}
+
+func (s *Service) sortedZones(ctx context.Context, account string, cmp func(a, b ZoneRecord) int) ([]ZoneRecord, error) {
+	zones, err := s.store.ListAccountZones(ctx, account)
+	if err != nil {
+		return nil, internal("list hosted zones", err)
+	}
+	slices.SortFunc(zones, cmp)
+	return zones, nil
+}
+
+func (s *Service) hostedZones(ctx context.Context, zones []ZoneRecord) []*route53.HostedZone {
+	out := make([]*route53.HostedZone, 0, len(zones))
+	for i := range zones {
+		count := 0
+		if doc, _, err := s.store.GetRRSets(ctx, zones[i].AccountID, zones[i].ID); err == nil {
+			count = len(doc.Sets)
+		}
+		out = append(out, hostedZone(&zones[i], count))
+	}
+	return out
+}
+
+func compareByName(a, b ZoneRecord) int {
+	if c := strings.Compare(reversedName(a.Name), reversedName(b.Name)); c != 0 {
+		return c
+	}
+	return strings.Compare(a.ID, b.ID)
+}
+
+func compareRRSets(a, b RRSet) int {
+	if c := strings.Compare(reversedName(a.Name), reversedName(b.Name)); c != 0 {
+		return c
+	}
+	return strings.Compare(a.Type, b.Type)
+}
+
+func pageOf[T any](items []T, start, limit int) ([]T, bool) {
+	end := min(start+limit, len(items))
+	return items[start:end], end < len(items)
+}
+
+func maxItems(raw *string, ceiling int) (int, error) {
+	if raw == nil || *raw == "" {
+		return ceiling, nil
+	}
+	n, err := strconv.Atoi(*raw)
+	if err != nil || n < 1 {
+		return 0, awserrors.Errorf(awserrors.ErrorInvalidInput, "MaxItems must be a positive integer")
+	}
+	return min(n, ceiling), nil
+}
+
+func taggedZoneID(resourceType, id string) (string, error) {
+	switch resourceType {
+	case route53.TagResourceTypeHostedzone:
+		return bareZoneID(id), nil
+	case route53.TagResourceTypeHealthcheck:
+		return "", awserrors.Errorf(awserrors.ErrorRoute53NoSuchHealthCheck, "No health check exists with the ID %s", id)
+	default:
+		return "", awserrors.Errorf(awserrors.ErrorInvalidInput, "ResourceType must be hostedzone or healthcheck")
+	}
+}
+
+func defaultRecordSets(zone string, nameservers []string) []RRSet {
+	ns := make([]string, 0, len(nameservers))
+	for _, n := range nameservers {
+		ns = append(ns, n+".")
+	}
+	hostmaster := "hostmaster." + zone
+	if _, parent, ok := strings.Cut(nameservers[0], "."); ok {
+		hostmaster = "hostmaster." + parent + "."
+	}
+	soa := fmt.Sprintf("%s %s 1 7200 900 1209600 86400", ns[0], hostmaster)
+	return []RRSet{
+		{Name: zone, Type: "NS", TTL: defaultNSTTL, Values: ns},
+		{Name: zone, Type: "SOA", TTL: defaultSOATTL, Values: []string{soa}},
+	}
+}
+
+func hostedZone(z *ZoneRecord, count int) *route53.HostedZone {
+	return &route53.HostedZone{
+		Id:              aws.String(hostedZonePrefix + z.ID),
+		Name:            aws.String(z.Name),
+		CallerReference: aws.String(z.CallerReference),
+		Config: &route53.HostedZoneConfig{
+			Comment:     nonEmpty(z.Comment),
+			PrivateZone: aws.Bool(false),
+		},
+		ResourceRecordSetCount: aws.Int64(int64(count)),
+	}
+}
+
+func delegationSet(z *ZoneRecord) *route53.DelegationSet {
+	return &route53.DelegationSet{NameServers: aws.StringSlice(z.Nameservers)}
+}
+
+func changeInfo(c *ChangeRecord) *route53.ChangeInfo {
+	submitted := c.SubmittedAt
+	return &route53.ChangeInfo{
+		Id:          aws.String(changePrefix + c.ID),
+		Status:      aws.String(c.Status),
+		SubmittedAt: &submitted,
+		Comment:     nonEmpty(c.Comment),
+	}
+}
+
+func tagSet(z *ZoneRecord) *route53.ResourceTagSet {
+	keys := make([]string, 0, len(z.Tags))
+	for k := range z.Tags {
+		keys = append(keys, k)
+	}
+	slices.Sort(keys)
+	tags := make([]*route53.Tag, 0, len(keys))
+	for _, k := range keys {
+		tags = append(tags, &route53.Tag{Key: aws.String(k), Value: aws.String(z.Tags[k])})
+	}
+	return &route53.ResourceTagSet{
+		ResourceType: aws.String(route53.TagResourceTypeHostedzone),
+		ResourceId:   aws.String(z.ID),
+		Tags:         tags,
+	}
+}
+
+func wireRRSet(r RRSet) *route53.ResourceRecordSet {
+	out := &route53.ResourceRecordSet{Name: aws.String(escapeName(r.Name)), Type: aws.String(r.Type), TTL: aws.Int64(r.TTL)}
+	for _, v := range r.Values {
+		out.ResourceRecords = append(out.ResourceRecords, &route53.ResourceRecord{Value: aws.String(v)})
+	}
+	return out
+}
+
+// escapeName renders a wildcard label the way Route 53 returns it, as \052.
+func escapeName(name string) string {
+	if strings.HasPrefix(name, "*.") {
+		return `\052` + name[1:]
+	}
+	return name
+}
+
+func nonEmpty(s string) *string {
+	if s == "" {
+		return nil
+	}
+	return aws.String(s)
+}
+
+func internal(what string, err error) error {
+	return awserrors.Errorf(awserrors.ErrorServerInternal, "%s: %v", what, err)
+}

spinifex/domains/route53/service_test.go

@@ -0,0 +1,510 @@
+package route53_test
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	r53 "github.com/aws/aws-sdk-go/service/route53"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/route53"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+const (
+	acctA = "000000000011"
+	acctB = "000000000012"
+)
+
+var ctx = context.Background()
+
+func newService(t *testing.T) *route53.Service {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	svc := route53.NewService(route53.NewStore(js))
+	svc.Nameservers = func() []string { return []string{"ns1.spx3.net", "ns2.spx3.net"} }
+	svc.Reserved = func(name string) bool { return name == "spx3.net." || strings.HasSuffix(name, ".compute.internal.") }
+	return svc
+}
+
+func createZone(t *testing.T, svc *route53.Service, account, name string) string {
+	t.Helper()
+	out, err := svc.CreateHostedZone(ctx, &r53.CreateHostedZoneInput{
+		Name: aws.String(name), CallerReference: aws.String("ref-" + name),
+	}, account)
+	require.NoError(t, err)
+	return aws.StringValue(out.HostedZone.Id)
+}
+
+func requireCode(t *testing.T, err error, code string) string {
+	t.Helper()
+	require.Error(t, err)
+	got, msg, ok := awserrors.ResolveErrorDetail(err)
+	require.True(t, ok, "error carries no AWS code: %v", err)
+	require.Equal(t, code, got, "message: %s", msg)
+	return msg
+}
+
+func change(action, name, rrType string, ttl int64, values ...string) *r53.Change {
+	set := &r53.ResourceRecordSet{Name: aws.String(name), Type: aws.String(rrType), TTL: aws.Int64(ttl)}
+	for _, v := range values {
+		set.ResourceRecords = append(set.ResourceRecords, &r53.ResourceRecord{Value: aws.String(v)})
+	}
+	return &r53.Change{Action: aws.String(action), ResourceRecordSet: set}
+}
+
+func apply(svc *route53.Service, account, zoneID string, changes ...*r53.Change) (*r53.ChangeResourceRecordSetsOutput, error) {
+	return svc.ChangeResourceRecordSets(ctx, &r53.ChangeResourceRecordSetsInput{
+		HostedZoneId: aws.String(zoneID), ChangeBatch: &r53.ChangeBatch{Changes: changes},
+	}, account)
+}
+
+func listSets(t *testing.T, svc *route53.Service, account, zoneID string) []string {
+	t.Helper()
+	out, err := svc.ListResourceRecordSets(ctx, &r53.ListResourceRecordSetsInput{HostedZoneId: aws.String(zoneID)}, account)
+	require.NoError(t, err)
+	var got []string
+	for _, s := range out.ResourceRecordSets {
+		var vals []string
+		for _, r := range s.ResourceRecords {
+			vals = append(vals, aws.StringValue(r.Value))
+		}
+		got = append(got, fmt.Sprintf("%s %s %d %s", aws.StringValue(s.Name), aws.StringValue(s.Type), aws.Int64Value(s.TTL), strings.Join(vals, ",")))
+	}
+	return got
+}
+
+func TestCreateHostedZoneReturnsAWSShape(t *testing.T) {
+	svc := newService(t)
+	out, err := svc.CreateHostedZone(ctx, &r53.CreateHostedZoneInput{
+		Name:             aws.String("Example.COM"),
+		CallerReference:  aws.String("terraform-1"),
+		HostedZoneConfig: &r53.HostedZoneConfig{Comment: aws.String("Managed by Terraform")},
+	}, acctA)
+	require.NoError(t, err)
+
+	id := aws.StringValue(out.HostedZone.Id)
+	assert.Regexp(t, `^/hostedzone/Z[A-Z0-9]{20}$`, id)
+	assert.Equal(t, "example.com.", aws.StringValue(out.HostedZone.Name))
+	assert.Equal(t, "Managed by Terraform", aws.StringValue(out.HostedZone.Config.Comment))
+	assert.False(t, aws.BoolValue(out.HostedZone.Config.PrivateZone))
+	assert.Equal(t, int64(2), aws.Int64Value(out.HostedZone.ResourceRecordSetCount))
+	assert.Equal(t, []string{"ns1.spx3.net", "ns2.spx3.net"}, aws.StringValueSlice(out.DelegationSet.NameServers))
+	assert.Equal(t, "/2013-04-01"+id, aws.StringValue(out.Location))
+	assert.Regexp(t, `^/change/C[A-Z0-9]{20}$`, aws.StringValue(out.ChangeInfo.Id))
+	assert.Equal(t, route53.ChangeStatusInSync, aws.StringValue(out.ChangeInfo.Status), "a zone not yet served is in sync once durable")
+
+	assert.Equal(t, []string{
+		"example.com. NS 172800 ns1.spx3.net.,ns2.spx3.net.",
+		"example.com. SOA 900 ns1.spx3.net. hostmaster.spx3.net. 1 7200 900 1209600 86400",
+	}, listSets(t, svc, acctA, id))
+
+	got, err := svc.GetHostedZone(ctx, &r53.GetHostedZoneInput{Id: aws.String(strings.TrimPrefix(id, "/hostedzone/"))}, acctA)
+	require.NoError(t, err)
+	assert.Equal(t, id, aws.StringValue(got.HostedZone.Id), "a bare ID and a prefixed one name the same zone")
+	assert.Equal(t, []string{"ns1.spx3.net", "ns2.spx3.net"}, aws.StringValueSlice(got.DelegationSet.NameServers))
+}
+
+func TestCreateHostedZoneRejections(t *testing.T) {
+	svc := newService(t)
+	createZone(t, svc, acctA, "example.com")
+
+	cases := []struct {
+		name string
+		in   *r53.CreateHostedZoneInput
+		code string
+	}{
+		{"repeated caller reference", &r53.CreateHostedZoneInput{Name: aws.String("other.com"), CallerReference: aws.String("ref-example.com")}, awserrors.ErrorHostedZoneAlreadyExists},
+		{"bare TLD", &r53.CreateHostedZoneInput{Name: aws.String("com"), CallerReference: aws.String("r1")}, awserrors.ErrorInvalidDomainName},
+		{"bad label", &r53.CreateHostedZoneInput{Name: aws.String("-bad.com"), CallerReference: aws.String("r2")}, awserrors.ErrorInvalidDomainName},
+		{"platform zone", &r53.CreateHostedZoneInput{Name: aws.String("spx3.net"), CallerReference: aws.String("r3")}, awserrors.ErrorInvalidDomainName},
+		{"private zone", &r53.CreateHostedZoneInput{Name: aws.String("p.com"), CallerReference: aws.String("r4"), VPC: &r53.VPC{VPCId: aws.String("vpc-1")}}, awserrors.ErrorInvalidInput},
+		{"delegation set", &r53.CreateHostedZoneInput{Name: aws.String("d.com"), CallerReference: aws.String("r5"), DelegationSetId: aws.String("N1")}, awserrors.ErrorInvalidInput},
+		{"no caller reference", &r53.CreateHostedZoneInput{Name: aws.String("n.com")}, awserrors.ErrorInvalidInput},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			_, err := svc.CreateHostedZone(ctx, tc.in, acctA)
+			requireCode(t, err, tc.code)
+		})
+	}
+
+	// The same reference in another account is a different request.
+	_, err := svc.CreateHostedZone(ctx, &r53.CreateHostedZoneInput{Name: aws.String("example.com"), CallerReference: aws.String("ref-example.com")}, acctB)
+	require.NoError(t, err)
+}
+
+func TestCreateHostedZoneEnforcesAccountLimit(t *testing.T) {
+	svc := newService(t)
+	svc.MaxZonesPerAccount = 2
+	createZone(t, svc, acctA, "a.com")
+	createZone(t, svc, acctA, "b.com")
+	_, err := svc.CreateHostedZone(ctx, &r53.CreateHostedZoneInput{Name: aws.String("c.com"), CallerReference: aws.String("c")}, acctA)
+	requireCode(t, err, awserrors.ErrorTooManyHostedZones)
+	createZone(t, svc, acctB, "c.com")
+}
+
+func TestZonesAreInvisibleToOtherAccounts(t *testing.T) {
+	svc := newService(t)
+	id := createZone(t, svc, acctA, "example.com")
+
+	_, err := svc.GetHostedZone(ctx, &r53.GetHostedZoneInput{Id: aws.String(id)}, acctB)
+	requireCode(t, err, awserrors.ErrorNoSuchHostedZone)
+	_, err = apply(svc, acctB, id, change("CREATE", "www.example.com", "A", 60, "192.0.2.1"))
+	requireCode(t, err, awserrors.ErrorNoSuchHostedZone)
+	_, err = svc.DeleteHostedZone(ctx, &r53.DeleteHostedZoneInput{Id: aws.String(id)}, acctB)
+	requireCode(t, err, awserrors.ErrorNoSuchHostedZone)
+
+	list, err := svc.ListHostedZones(ctx, &r53.ListHostedZonesInput{}, acctB)
+	require.NoError(t, err)
+	assert.Empty(t, list.HostedZones)
+}
+
+func TestChangeResourceRecordSetsCreateUpsertDelete(t *testing.T) {
+	svc := newService(t)
+	id := createZone(t, svc, acctA, "example.com")
+
+	_, err := apply(svc, acctA, id,
+		change("CREATE", "www.example.com", "A", 300, "192.0.2.10"),
+		change("CREATE", "*.example.com", "TXT", 60, `"hello"`),
+		change("CREATE", "example.com", "MX", 300, "10 mail.example.com"),
+	)
+	require.NoError(t, err)
+
+	_, err = apply(svc, acctA, id, change("UPSERT", "WWW.example.com.", "A", 120, "192.0.2.11", "192.0.2.12"))
+	require.NoError(t, err)
+
+	assert.Equal(t, []string{
+		"example.com. MX 300 10 mail.example.com",
+		"example.com. NS 172800 ns1.spx3.net.,ns2.spx3.net.",
+		"example.com. SOA 900 ns1.spx3.net. hostmaster.spx3.net. 1 7200 900 1209600 86400",
+		`\052.example.com. TXT 60 "hello"`,
+		"www.example.com. A 120 192.0.2.11,192.0.2.12",
+	}, listSets(t, svc, acctA, id))
+
+	msg := requireCode(t, func() error {
+		_, err := apply(svc, acctA, id, change("DELETE", "www.example.com", "A", 120, "192.0.2.11"))
+		return err
+	}(), awserrors.ErrorInvalidChangeBatch)
+	assert.Contains(t, msg, "Tried to delete resource record set [name='www.example.com.', type='A'] but the values provided do not match the current values")
+
+	_, err = apply(svc, acctA, id, change("DELETE", "www.example.com.", "A", 120, "192.0.2.12", "192.0.2.11"))
+	require.NoError(t, err, "a DELETE matches values regardless of order")
+	assert.Len(t, listSets(t, svc, acctA, id), 4)
+}
+
+func TestChangeBatchIsAllOrNothing(t *testing.T) {
+	svc := newService(t)
+	id := createZone(t, svc, acctA, "example.com")
+	_, err := apply(svc, acctA, id, change("CREATE", "www.example.com", "A", 60, "192.0.2.1"))
+	require.NoError(t, err)
+
+	msg := requireCode(t, func() error {
+		_, err := apply(svc, acctA, id,
+			change("CREATE", "api.example.com", "A", 60, "192.0.2.2"),
+			change("CREATE", "www.example.com", "A", 60, "192.0.2.3"),
+			change("DELETE", "gone.example.com", "A", 60, "192.0.2.4"),
+		)
+		return err
+	}(), awserrors.ErrorInvalidChangeBatch)
+	assert.Contains(t, msg, "Tried to create resource record set [name='www.example.com.', type='A'] but it already exists")
+	assert.Contains(t, msg, "Tried to delete resource record set [name='gone.example.com.', type='A'] but it was not found")
+	assert.Len(t, listSets(t, svc, acctA, id), 3, "the valid change in a failed batch must not land")
+
+	_, err = apply(svc, acctA, id,
+		change("DELETE", "www.example.com", "A", 60, "192.0.2.1"),
+		change("CREATE", "www.example.com", "CNAME", 60, "lb.example.net"),
+	)
+	require.NoError(t, err, "changes apply in order, so a delete frees the name for a CNAME in the same batch")
+}
+
+func TestChangeBatchValidation(t *testing.T) {
+	svc := newService(t)
+	id := createZone(t, svc, acctA, "example.com")
+	_, err := apply(svc, acctA, id, change("CREATE", "www.example.com", "A", 60, "192.0.2.1"))
+	require.NoError(t, err)
+
+	alias := change("CREATE", "lb.example.com", "A", 60)
+	alias.ResourceRecordSet.TTL = nil
+	alias.ResourceRecordSet.AliasTarget = &r53.AliasTarget{DNSName: aws.String("x.elb.spx3.net"), HostedZoneId: aws.String("Z1"), EvaluateTargetHealth: aws.Bool(false)}
+	weighted := change("CREATE", "w.example.com", "A", 60, "192.0.2.1")
+	weighted.ResourceRecordSet.SetIdentifier = aws.String("a")
+	weighted.ResourceRecordSet.Weight = aws.Int64(10)
+
+	cases := []struct {
+		name   string
+		change *r53.Change
+		code   string
+		msg    string
+	}{
+		{"CNAME beside A", change("CREATE", "www.example.com", "CNAME", 60, "x.example.net"), awserrors.ErrorInvalidChangeBatch, "conflicting RRSet of type A"},
+		{"CNAME at apex", change("CREATE", "example.com", "CNAME", 60, "x.example.net"), awserrors.ErrorInvalidChangeBatch, "not permitted at apex"},
+		{"name outside zone", change("CREATE", "www.example.org", "A", 60, "192.0.2.1"), awserrors.ErrorInvalidChangeBatch, "is not permitted in zone example.com."},
+		{"apex NS delete", change("DELETE", "example.com", "NS", 172800, "ns1.spx3.net.", "ns2.spx3.net."), awserrors.ErrorInvalidChangeBatch, "must contain at least one NS record"},
+		{"SOA below apex", change("CREATE", "sub.example.com", "SOA", 60, "a. b. 1 2 3 4 5"), awserrors.ErrorInvalidChangeBatch, "only permitted at the zone apex"},
+		{"bad IPv4", change("CREATE", "v4.example.com", "A", 60, "2001:db8::1"), awserrors.ErrorInvalidChangeBatch, "not a valid IPv4 address"},
+		{"bad IPv6", change("CREATE", "v6.example.com", "AAAA", 60, "192.0.2.1"), awserrors.ErrorInvalidChangeBatch, "not a valid IPv6 address"},
+		{"bad MX", change("CREATE", "example.com", "MX", 60, "mail.example.com"), awserrors.ErrorInvalidChangeBatch, "priority host"},
+		{"two CNAME values", change("CREATE", "c.example.com", "CNAME", 60, "a.example.net", "b.example.net"), awserrors.ErrorInvalidChangeBatch, "more than one value"},
+		{"bad action", change("REPLACE", "r.example.com", "A", 60, "192.0.2.1"), awserrors.ErrorInvalidChangeBatch, "Invalid action"},
+		{"alias", alias, awserrors.ErrorInvalidInput, "alias"},
+		{"weighted", weighted, awserrors.ErrorInvalidInput, "simple routing"},
+		{"unsupported type", change("CREATE", "ds.example.com", "DS", 60, "1 2 3 abc"), awserrors.ErrorInvalidInput, "not supported"},
+		{"negative TTL", change("CREATE", "t.example.com", "A", -1, "192.0.2.1"), awserrors.ErrorInvalidInput, "out of range"},
+		{"no values", change("CREATE", "e.example.com", "A", 60), awserrors.ErrorInvalidInput, "at least one record"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			_, err := apply(svc, acctA, id, tc.change)
+			assert.Contains(t, requireCode(t, err, tc.code), tc.msg)
+		})
+	}
+
+	_, err = apply(svc, acctA, id)
+	requireCode(t, err, awserrors.ErrorInvalidInput)
+}
+
+func TestChangeBatchSizeLimits(t *testing.T) {
+	svc := newService(t)
+	id := createZone(t, svc, acctA, "example.com")
+	values := make([]string, 0, 401)
+	for i := range 401 {
+		values = append(values, fmt.Sprintf("192.0.%d.%d", i/250, i%250+1))
+	}
+	_, err := apply(svc, acctA, id, change("CREATE", "many.example.com", "A", 60, values...))
+	assert.Contains(t, requireCode(t, err, awserrors.ErrorInvalidChangeBatch), "more than 400 values")
+
+	var batch []*r53.Change
+	for i := range 3 {
+		batch = append(batch, change("UPSERT", fmt.Sprintf("h%d.example.com", i), "A", 60, values[:200]...))
+	}
+	_, err = apply(svc, acctA, id, batch...)
+	assert.Contains(t, requireCode(t, err, awserrors.ErrorInvalidChangeBatch), "Number of records limit of 1000 exceeded", "an UPSERT counts each record twice")
+}
+
+func TestConcurrentChangeBatchesBothLand(t *testing.T) {
+	svc := newService(t)
+	id := createZone(t, svc, acctA, "example.com")
+	var wg sync.WaitGroup
+	errs := make([]error, 2)
+	for i := range 2 {
+		wg.Go(func() {
+			_, errs[i] = apply(svc, acctA, id, change("CREATE", fmt.Sprintf("h%d.example.com", i), "A", 60, "192.0.2.1"))
+		})
+	}
+	wg.Wait()
+	require.NoError(t, errs[0])
+	require.NoError(t, errs[1])
+	assert.Len(t, listSets(t, svc, acctA, id), 4)
+}
+
+func TestListResourceRecordSetsPagesInRoute53Order(t *testing.T) {
+	svc := newService(t)
+	id := createZone(t, svc, acctA, "example.com")
+	_, err := apply(svc, acctA, id,
+		change("CREATE", "www.example.com", "A", 300, "192.0.2.10"),
+		change("CREATE", "www.example.com", "AAAA", 300, "2001:db8::10"),
+		change("CREATE", "a.b.example.com", "A", 300, "192.0.2.11"),
+		change("CREATE", "z.example.com", "A", 300, "192.0.2.12"),
+	)
+	require.NoError(t, err)
+
+	// The read Terraform makes for one aws_route53_record.
+	out, err := svc.ListResourceRecordSets(ctx, &r53.ListResourceRecordSetsInput{
+		HostedZoneId: aws.String(id), MaxItems: aws.String("1"),
+		StartRecordName: aws.String("www.example.com."), StartRecordType: aws.String("A"),
+	}, acctA)
+	require.NoError(t, err)
+	require.Len(t, out.ResourceRecordSets, 1)
+	assert.Equal(t, "www.example.com.", aws.StringValue(out.ResourceRecordSets[0].Name))
+	assert.Equal(t, "A", aws.StringValue(out.ResourceRecordSets[0].Type))
+	assert.True(t, aws.BoolValue(out.IsTruncated))
+	assert.Equal(t, "www.example.com.", aws.StringValue(out.NextRecordName))
+	assert.Equal(t, "AAAA", aws.StringValue(out.NextRecordType))
+
+	var names []string
+	in := &r53.ListResourceRecordSetsInput{HostedZoneId: aws.String(id), MaxItems: aws.String("2")}
+	for {
+		page, err := svc.ListResourceRecordSets(ctx, in, acctA)
+		require.NoError(t, err)
+		for _, s := range page.ResourceRecordSets {
+			names = append(names, aws.StringValue(s.Name)+" "+aws.StringValue(s.Type))
+		}
+		if !aws.BoolValue(page.IsTruncated) {
+			break
+		}
+		in.StartRecordName, in.StartRecordType = page.NextRecordName, page.NextRecordType
+	}
+	assert.Equal(t, []string{
+		"example.com. NS", "example.com. SOA", "a.b.example.com. A",
+		"www.example.com. A", "www.example.com. AAAA", "z.example.com. A",
+	}, names, "names sort by reversed label, so b.example.com sorts before www")
+
+	_, err = svc.ListResourceRecordSets(ctx, &r53.ListResourceRecordSetsInput{HostedZoneId: aws.String(id), StartRecordType: aws.String("A")}, acctA)
+	requireCode(t, err, awserrors.ErrorInvalidInput)
+}
+
+func TestDeleteHostedZoneRequiresOnlyDefaultRecords(t *testing.T) {
+	svc := newService(t)
+	id := createZone(t, svc, acctA, "example.com")
+	_, err := apply(svc, acctA, id, change("CREATE", "sub.example.com", "NS", 300, "ns.elsewhere.net"))
+	require.NoError(t, err)
+
+	_, err = svc.DeleteHostedZone(ctx, &r53.DeleteHostedZoneInput{Id: aws.String(id)}, acctA)
+	requireCode(t, err, awserrors.ErrorHostedZoneNotEmpty)
+
+	_, err = apply(svc, acctA, id, change("DELETE", "sub.example.com", "NS", 300, "ns.elsewhere.net"))
+	require.NoError(t, err)
+	out, err := svc.DeleteHostedZone(ctx, &r53.DeleteHostedZoneInput{Id: aws.String(id)}, acctA)
+	require.NoError(t, err)
+
+	got, err := svc.GetChange(ctx, &r53.GetChangeInput{Id: out.ChangeInfo.Id}, acctA)
+	require.NoError(t, err, "Terraform polls GetChange after DeleteHostedZone")
+	assert.Equal(t, route53.ChangeStatusInSync, aws.StringValue(got.ChangeInfo.Status))
+
+	_, err = svc.GetHostedZone(ctx, &r53.GetHostedZoneInput{Id: aws.String(id)}, acctA)
+	requireCode(t, err, awserrors.ErrorNoSuchHostedZone)
+
+	// The caller reference is released with the zone.
+	_, err = svc.CreateHostedZone(ctx, &r53.CreateHostedZoneInput{Name: aws.String("example.com"), CallerReference: aws.String("ref-example.com")}, acctA)
+	require.NoError(t, err)
+}
+
+func TestListHostedZonesPaging(t *testing.T) {
+	svc := newService(t)
+	for _, n := range []string{"b.example.com", "example.org", "a.example.com", "example.com"} {
+		createZone(t, svc, acctA, n)
+	}
+	createZone(t, svc, acctB, "other.com")
+
+	var ids []string
+	in := &r53.ListHostedZonesInput{MaxItems: aws.String("3")}
+	for {
+		out, err := svc.ListHostedZones(ctx, in, acctA)
+		require.NoError(t, err)
+		for _, z := range out.HostedZones {
+			ids = append(ids, aws.StringValue(z.Id))
+		}
+		if !aws.BoolValue(out.IsTruncated) {
+			break
+		}
+		in.Marker = out.NextMarker
+	}
+	assert.Len(t, ids, 4)
+
+	byName, err := svc.ListHostedZonesByName(ctx, &r53.ListHostedZonesByNameInput{}, acctA)
+	require.NoError(t, err)
+	var names []string
+	for _, z := range byName.HostedZones {
+		names = append(names, aws.StringValue(z.Name))
+	}
+	assert.Equal(t, []string{"example.com.", "a.example.com.", "b.example.com.", "example.org."}, names)
+
+	from, err := svc.ListHostedZonesByName(ctx, &r53.ListHostedZonesByNameInput{DNSName: aws.String("b.example.com"), MaxItems: aws.String("1")}, acctA)
+	require.NoError(t, err)
+	require.Len(t, from.HostedZones, 1)
+	assert.Equal(t, "b.example.com.", aws.StringValue(from.HostedZones[0].Name))
+	assert.Equal(t, "example.org.", aws.StringValue(from.NextDNSName))
+
+	count, err := svc.GetHostedZoneCount(ctx, &r53.GetHostedZoneCountInput{}, acctA)
+	require.NoError(t, err)
+	assert.Equal(t, int64(4), aws.Int64Value(count.HostedZoneCount))
+
+	private, err := svc.ListHostedZones(ctx, &r53.ListHostedZonesInput{HostedZoneType: aws.String("PrivateHostedZone")}, acctA)
+	require.NoError(t, err)
+	assert.Empty(t, private.HostedZones)
+}
+
+func TestHostedZoneTags(t *testing.T) {
+	svc := newService(t)
+	id := createZone(t, svc, acctA, "example.com")
+	bare := strings.TrimPrefix(id, "/hostedzone/")
+
+	_, err := svc.ChangeTagsForResource(ctx, &r53.ChangeTagsForResourceInput{
+		ResourceType: aws.String("hostedzone"), ResourceId: aws.String(bare),
+		AddTags: []*r53.Tag{{Key: aws.String("env"), Value: aws.String("prod")}, {Key: aws.String("team"), Value: aws.String("dns")}},
+	}, acctA)
+	require.NoError(t, err)
+	_, err = svc.ChangeTagsForResource(ctx, &r53.ChangeTagsForResourceInput{
+		ResourceType: aws.String("hostedzone"), ResourceId: aws.String(bare), RemoveTagKeys: aws.StringSlice([]string{"team"}),
+	}, acctA)
+	require.NoError(t, err)
+
+	out, err := svc.ListTagsForResource(ctx, &r53.ListTagsForResourceInput{ResourceType: aws.String("hostedzone"), ResourceId: aws.String(bare)}, acctA)
+	require.NoError(t, err)
+	assert.Equal(t, bare, aws.StringValue(out.ResourceTagSet.ResourceId))
+	require.Len(t, out.ResourceTagSet.Tags, 1)
+	assert.Equal(t, "env", aws.StringValue(out.ResourceTagSet.Tags[0].Key))
+
+	many, err := svc.ListTagsForResources(ctx, &r53.ListTagsForResourcesInput{ResourceType: aws.String("hostedzone"), ResourceIds: aws.StringSlice([]string{bare})}, acctA)
+	require.NoError(t, err)
+	assert.Len(t, many.ResourceTagSets, 1)
+
+	_, err = svc.ListTagsForResource(ctx, &r53.ListTagsForResourceInput{ResourceType: aws.String("hostedzone"), ResourceId: aws.String(bare)}, acctB)
+	requireCode(t, err, awserrors.ErrorNoSuchHostedZone)
+	_, err = svc.ListTagsForResource(ctx, &r53.ListTagsForResourceInput{ResourceType: aws.String("healthcheck"), ResourceId: aws.String("hc")}, acctA)
+	requireCode(t, err, awserrors.ErrorRoute53NoSuchHealthCheck)
+
+	var tags []*r53.Tag
+	for i := range 51 {
+		tags = append(tags, &r53.Tag{Key: aws.String(fmt.Sprintf("k%d", i)), Value: aws.String("v")})
+	}
+	_, err = svc.ChangeTagsForResource(ctx, &r53.ChangeTagsForResourceInput{ResourceType: aws.String("hostedzone"), ResourceId: aws.String(bare), AddTags: tags}, acctA)
+	requireCode(t, err, awserrors.ErrorRoute53TooManyTagsForZone)
+}
+
+func TestChangesOnAnActiveZoneWaitForSync(t *testing.T) {
+	svc := newService(t)
+	id := createZone(t, svc, acctA, "example.com")
+	bare := strings.TrimPrefix(id, "/hostedzone/")
+	require.NoError(t, svc.Store().MutateZone(ctx, acctA, bare, func(z *route53.ZoneRecord) (bool, error) {
+		z.State = route53.StateActive
+		return true, nil
+	}))
+
+	out, err := apply(svc, acctA, id, change("CREATE", "www.example.com", "A", 60, "192.0.2.1"))
+	require.NoError(t, err)
+	assert.Equal(t, route53.ChangeStatusPending, aws.StringValue(out.ChangeInfo.Status))
+
+	doc, _, err := svc.Store().GetRRSets(ctx, acctA, bare)
+	require.NoError(t, err)
+	require.NoError(t, svc.Store().MarkChangesInSync(ctx, acctA, bare, doc.Version))
+	got, err := svc.GetChange(ctx, &r53.GetChangeInput{Id: out.ChangeInfo.Id}, acctA)
+	require.NoError(t, err)
+	assert.Equal(t, route53.ChangeStatusInSync, aws.StringValue(got.ChangeInfo.Status))
+
+	_, err = svc.GetChange(ctx, &r53.GetChangeInput{Id: out.ChangeInfo.Id}, acctB)
+	requireCode(t, err, awserrors.ErrorNoSuchChange)
+}
+
+func TestUpdateHostedZoneComment(t *testing.T) {
+	svc := newService(t)
+	id := createZone(t, svc, acctA, "example.com")
+	out, err := svc.UpdateHostedZoneComment(ctx, &r53.UpdateHostedZoneCommentInput{Id: aws.String(id), Comment: aws.String("moved")}, acctA)
+	require.NoError(t, err)
+	assert.Equal(t, "moved", aws.StringValue(out.HostedZone.Config.Comment))
+	_, err = svc.UpdateHostedZoneComment(ctx, &r53.UpdateHostedZoneCommentInput{Id: aws.String(id), Comment: aws.String(strings.Repeat("x", 257))}, acctA)
+	requireCode(t, err, awserrors.ErrorInvalidInput)
+}
+
+func TestChangeRecordsArePrunedAfterRetention(t *testing.T) {
+	svc := newService(t)
+	now := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
+	svc.Now = func() time.Time { return now }
+	id := createZone(t, svc, acctA, "example.com")
+	first, err := apply(svc, acctA, id, change("CREATE", "a.example.com", "A", 60, "192.0.2.1"))
+	require.NoError(t, err)
+
+	now = now.Add(8 * 24 * time.Hour)
+	_, err = apply(svc, acctA, id, change("CREATE", "b.example.com", "A", 60, "192.0.2.1"))
+	require.NoError(t, err)
+	_, err = svc.GetChange(ctx, &r53.GetChangeInput{Id: first.ChangeInfo.Id}, acctA)
+	requireCode(t, err, awserrors.ErrorNoSuchChange)
+}

spinifex/domains/route53/store.go

@@ -0,0 +1,340 @@
+package route53
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// KVBucketRoute53 holds every hosted zone, its record sets and its changes.
+const (
+	KVBucketRoute53        = "spinifex-route53"
+	KVBucketRoute53Version = 1
+	// KVBucketRoute53Realize elects the one node that publishes zones.
+	KVBucketRoute53Realize = "spinifex-route53-realize"
+	// KVBucketRoute53Verify elects the one node that checks delegations.
+	KVBucketRoute53Verify = "spinifex-route53-verify"
+
+	keyPrefixZone     = "zone."
+	keyPrefixRRSets   = "rrsets."
+	keyPrefixChange   = "change."
+	keyPrefixCallerRf = "ref."
+	keyPrefixPublish  = "published."
+	casAttempts       = 20
+)
+
+// Activation states. A zone is served by northstar only while ACTIVE.
+const (
+	StatePendingNameservers = "PENDING_NAMESERVERS"
+	StateActive             = "ACTIVE"
+	StateNameserversLost    = "NAMESERVERS_LOST"
+	// StateSuspended is an operator's deactivation; only the operator lifts it.
+	StateSuspended = "SUSPENDED"
+)
+
+// Change statuses, as GetChange reports them.
+const (
+	ChangeStatusPending = "PENDING"
+	ChangeStatusInSync  = "INSYNC"
+)
+
+// ZoneRecord is a hosted zone's identity and configuration. Its record sets
+// live in a separate RRSetsDoc so a change batch is one CAS on one document.
+type ZoneRecord struct {
+	ID              string            `json:"id"`
+	AccountID       string            `json:"account_id"`
+	Name            string            `json:"name"`
+	CallerReference string            `json:"caller_reference"`
+	Comment         string            `json:"comment,omitempty"`
+	CreatedAt       time.Time         `json:"created_at"`
+	Nameservers     []string          `json:"nameservers"`
+	Tags            map[string]string `json:"tags,omitempty"`
+
+	State       string    `json:"state"`
+	StateReason string    `json:"state_reason,omitempty"`
+	CheckedAt   time.Time `json:"checked_at"`
+	// Delegation is the NS set the parent published at the last check, and
+	// DelegationSource the parent zone or server that answered.
+	Delegation       []string  `json:"delegation,omitempty"`
+	DelegationSource string    `json:"delegation_source,omitempty"`
+	FailingSince     time.Time `json:"failing_since,omitzero"`
+	// Override is set when an operator activated or suspended the zone; the
+	// verifier leaves such a zone alone.
+	Override       string `json:"override,omitempty"`
+	OverrideReason string `json:"override_reason,omitempty"`
+}
+
+// RRSetsDoc is a zone's complete record-set document. Version increases by one
+// on every committed change batch and is what a realized zone is stamped with.
+type RRSetsDoc struct {
+	ZoneID    string    `json:"zone_id"`
+	Version   uint64    `json:"version"`
+	UpdatedAt time.Time `json:"updated_at"`
+	Sets      []RRSet   `json:"sets"`
+}
+
+// RRSet is one simple-routing record set. Name is normalized.
+type RRSet struct {
+	Name   string   `json:"name"`
+	Type   string   `json:"type"`
+	TTL    int64    `json:"ttl"`
+	Values []string `json:"values"`
+}
+
+// ChangeRecord tracks one submitted change for GetChange.
+type ChangeRecord struct {
+	ID          string    `json:"id"`
+	AccountID   string    `json:"account_id"`
+	ZoneID      string    `json:"zone_id"`
+	Version     uint64    `json:"version"`
+	Status      string    `json:"status"`
+	Comment     string    `json:"comment,omitempty"`
+	SubmittedAt time.Time `json:"submitted_at"`
+}
+
+// PublishedZone records what the realizer last wrote to northstar for a zone
+// name. It is how a withdrawn zone is found, since its own records may be gone.
+type PublishedZone struct {
+	Apex      string `json:"apex"`
+	AccountID string `json:"account_id"`
+	ZoneID    string `json:"zone_id"`
+	Version   uint64 `json:"version"`
+	Digest    string `json:"digest"`
+	InSync    bool   `json:"in_sync"`
+}
+
+type callerRef struct {
+	ZoneID string `json:"zone_id"`
+}
+
+// Store reads and writes the route53 bucket. Keys are account-scoped, so an
+// owner lookup never reads another account's records.
+type Store struct {
+	bucket  *kvstore.Bucket
+	zones   *kvstore.Store[ZoneRecord]
+	rrsets  *kvstore.Store[RRSetsDoc]
+	changes *kvstore.Store[ChangeRecord]
+	refs    *kvstore.Store[callerRef]
+	publish *kvstore.Store[PublishedZone]
+}
+
+// NewStore returns a Store over js. The bucket is created on first use.
+func NewStore(js jetstream.JetStream) *Store {
+	b := kvstore.NewBucket(js, kvstore.Config{
+		Name:        KVBucketRoute53,
+		History:     KVBucketRoute53Version,
+		Description: "Route 53 hosted zones, record sets and changes",
+		Missing:     "route53 store has no JetStream connection",
+		Attempts:    casAttempts,
+		Exhausted: func(key string, attempts int) error {
+			return fmt.Errorf("route53 store: exceeded %d CAS attempts updating %s", attempts, key)
+		},
+	})
+	return &Store{
+		bucket:  b,
+		zones:   kvstore.On[ZoneRecord](b),
+		rrsets:  kvstore.On[RRSetsDoc](b),
+		changes: kvstore.On[ChangeRecord](b),
+		refs:    kvstore.On[callerRef](b),
+		publish: kvstore.On[PublishedZone](b),
+	}
+}
+
+func zoneKey(account, id string) string   { return keyPrefixZone + kvutil.AccountKey(account, id) }
+func rrsetsKey(account, id string) string { return keyPrefixRRSets + kvutil.AccountKey(account, id) }
+func changeKey(account, id string) string { return keyPrefixChange + kvutil.AccountKey(account, id) }
+func refKey(account, ref string) string {
+	return keyPrefixCallerRf + kvutil.AccountKey(account, refDigest(ref))
+}
+
+// GetZone returns an account's zone, or kvstore.ErrNotFound.
+func (s *Store) GetZone(ctx context.Context, account, id string) (*ZoneRecord, error) {
+	z, _, err := s.zones.Get(ctx, zoneKey(account, id))
+	return z, err
+}
+
+// ListAccountZones returns every zone the account owns.
+func (s *Store) ListAccountZones(ctx context.Context, account string) ([]ZoneRecord, error) {
+	return s.zones.List(ctx, keyPrefixZone+account+".")
+}
+
+// ListAllZones returns every zone in every account, for the realizer and the
+// verifier, which act across accounts.
+func (s *Store) ListAllZones(ctx context.Context) ([]ZoneRecord, error) {
+	return s.zones.List(ctx, keyPrefixZone)
+}
+
+// MutateZone applies fn to a zone under CAS.
+func (s *Store) MutateZone(ctx context.Context, account, id string, fn func(*ZoneRecord) (bool, error)) error {
+	return s.zones.Mutate(ctx, zoneKey(account, id), fn)
+}
+
+// GetRRSets returns a zone's record sets and the revision to CAS against.
+func (s *Store) GetRRSets(ctx context.Context, account, id string) (*RRSetsDoc, uint64, error) {
+	return s.rrsets.Get(ctx, rrsetsKey(account, id))
+}
+
+// CommitRRSets writes doc if the document is still at rev.
+func (s *Store) CommitRRSets(ctx context.Context, account string, doc *RRSetsDoc, rev uint64) error {
+	_, err := s.rrsets.CompareAndSet(ctx, rrsetsKey(account, doc.ZoneID), doc, rev)
+	return err
+}
+
+// GetChange returns an account's change, or kvstore.ErrNotFound.
+func (s *Store) GetChange(ctx context.Context, account, id string) (*ChangeRecord, error) {
+	c, _, err := s.changes.Get(ctx, changeKey(account, id))
+	return c, err
+}
+
+// PutChange stores a new change.
+func (s *Store) PutChange(ctx context.Context, c *ChangeRecord) error {
+	_, err := s.changes.Create(ctx, changeKey(c.AccountID, c.ID), c)
+	return err
+}
+
+// ListZoneChanges returns every change recorded for one zone.
+func (s *Store) ListZoneChanges(ctx context.Context, account, zoneID string) ([]ChangeRecord, error) {
+	all, err := s.changes.List(ctx, keyPrefixChange+account+".")
+	if err != nil {
+		return nil, err
+	}
+	out := all[:0]
+	for _, c := range all {
+		if c.ZoneID == zoneID {
+			out = append(out, c)
+		}
+	}
+	return out, nil
+}
+
+// MarkChangesInSync moves every PENDING change for a zone at or below version
+// to INSYNC.
+func (s *Store) MarkChangesInSync(ctx context.Context, account, zoneID string, version uint64) error {
+	changes, err := s.ListZoneChanges(ctx, account, zoneID)
+	if err != nil {
+		return err
+	}
+	for _, c := range changes {
+		if c.Status != ChangeStatusPending || c.Version > version {
+			continue
+		}
+		err := s.changes.Mutate(ctx, changeKey(account, c.ID), func(rec *ChangeRecord) (bool, error) {
+			if rec.Status == ChangeStatusInSync {
+				return false, nil
+			}
+			rec.Status = ChangeStatusInSync
+			return true, nil
+		})
+		if err != nil && !errors.Is(err, kvstore.ErrNotFound) {
+			return err
+		}
+	}
+	return nil
+}
+
+// ClaimCallerReference records that ref created zoneID. It returns the zone
+// already holding ref when the claim loses.
+func (s *Store) ClaimCallerReference(ctx context.Context, account, ref, zoneID string) (existing string, err error) {
+	if _, err := s.refs.Create(ctx, refKey(account, ref), &callerRef{ZoneID: zoneID}); err != nil {
+		if !errors.Is(err, kvstore.ErrExists) {
+			return "", err
+		}
+		held, _, getErr := s.refs.Get(ctx, refKey(account, ref))
+		if getErr != nil {
+			return "", getErr
+		}
+		return held.ZoneID, nil
+	}
+	return "", nil
+}
+
+// CreateZone writes a new zone and its initial record sets.
+func (s *Store) CreateZone(ctx context.Context, z *ZoneRecord, doc *RRSetsDoc) error {
+	if _, err := s.rrsets.Create(ctx, rrsetsKey(z.AccountID, z.ID), doc); err != nil {
+		return err
+	}
+	if _, err := s.zones.Create(ctx, zoneKey(z.AccountID, z.ID), z); err != nil {
+		_ = s.bucket.Delete(ctx, rrsetsKey(z.AccountID, z.ID))
+		return err
+	}
+	return nil
+}
+
+// DeleteZone removes a zone whose record sets are still at rev, returning
+// kvstore.ErrConflict if a change landed since they were read. The record-set
+// document goes first, so a concurrent change batch cannot commit into a zone
+// that is being deleted; the zone's earlier changes go with it.
+func (s *Store) DeleteZone(ctx context.Context, z *ZoneRecord, rev uint64) error {
+	if err := s.bucket.CompareAndDelete(ctx, rrsetsKey(z.AccountID, z.ID), rev); err != nil {
+		return err
+	}
+	if err := s.bucket.Delete(ctx, zoneKey(z.AccountID, z.ID)); err != nil {
+		return err
+	}
+	if err := s.bucket.Delete(ctx, refKey(z.AccountID, z.CallerReference)); err != nil {
+		return err
+	}
+	return s.PruneChanges(ctx, z.AccountID, z.ID, time.Time{}, true)
+}
+
+// PruneChanges deletes a zone's INSYNC changes submitted before cutoff, or all
+// of them when all is set. A PENDING change is kept until it is in sync.
+func (s *Store) PruneChanges(ctx context.Context, account, zoneID string, cutoff time.Time, all bool) error {
+	changes, err := s.ListZoneChanges(ctx, account, zoneID)
+	if err != nil {
+		return err
+	}
+	for _, c := range changes {
+		if !all && (c.Status != ChangeStatusInSync || !c.SubmittedAt.Before(cutoff)) {
+			continue
+		}
+		if err := s.bucket.Delete(ctx, changeKey(account, c.ID)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// ReleaseCallerReference drops a claim whose zone was never created, so the
+// caller can retry with the same reference.
+func (s *Store) ReleaseCallerReference(ctx context.Context, account, ref string) error {
+	return s.bucket.Delete(ctx, refKey(account, ref))
+}
+
+// ListAllChanges returns every change in every account.
+func (s *Store) ListAllChanges(ctx context.Context) ([]ChangeRecord, error) {
+	return s.changes.List(ctx, keyPrefixChange)
+}
+
+// DeleteChange removes one change.
+func (s *Store) DeleteChange(ctx context.Context, account, id string) error {
+	return s.bucket.Delete(ctx, changeKey(account, id))
+}
+
+// ListPublished returns the published zones keyed by apex.
+func (s *Store) ListPublished(ctx context.Context) (map[string]PublishedZone, error) {
+	all, err := s.publish.List(ctx, keyPrefixPublish)
+	if err != nil {
+		return nil, err
+	}
+	out := make(map[string]PublishedZone, len(all))
+	for _, p := range all {
+		out[p.Apex] = p
+	}
+	return out, nil
+}
+
+// SetPublished records what was written for p.Apex.
+func (s *Store) SetPublished(ctx context.Context, p *PublishedZone) error {
+	return s.publish.Set(ctx, keyPrefixPublish+p.Apex, p)
+}
+
+// DeletePublished forgets a withdrawn zone.
+func (s *Store) DeletePublished(ctx context.Context, apex string) error {
+	return s.bucket.Delete(ctx, keyPrefixPublish+apex)
+}

spinifex/domains/route53/txt.go

@@ -0,0 +1,91 @@
+package route53
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+)
+
+const maxCharacterString = 255
+
+// characterStrings splits a TXT or SPF value into its character-strings, kept
+// in presentation format so their escapes reach the wire unchanged. A value
+// with no opening quote is one string, as Route 53 treats it.
+func characterStrings(v string) ([]string, error) {
+	v = strings.TrimSpace(v)
+	if !strings.HasPrefix(v, `"`) {
+		n, err := decodedLength(v)
+		if err != nil {
+			return nil, err
+		}
+		if n > maxCharacterString {
+			return nil, errTXTTooLong
+		}
+		return []string{v}, nil
+	}
+	var out []string
+	for v != "" {
+		if v[0] != '"' {
+			return nil, errUnquoted
+		}
+		end := closingQuote(v)
+		if end < 0 {
+			return nil, errUnquoted
+		}
+		s := v[1:end]
+		n, err := decodedLength(s)
+		if err != nil {
+			return nil, err
+		}
+		if n > maxCharacterString {
+			return nil, errTXTTooLong
+		}
+		out = append(out, s)
+		v = strings.TrimLeft(v[end+1:], " \t")
+	}
+	return out, nil
+}
+
+var (
+	errTXTTooLong = errors.New("TXTRDATATooLong (TXT RDATA string too long)")
+	errUnquoted   = errors.New("InvalidCharacterString (Value should be enclosed in quotation marks)")
+)
+
+// closingQuote returns the index of the quote closing the string opened at
+// v[0], skipping escaped characters, or -1.
+func closingQuote(v string) int {
+	for i := 1; i < len(v); i++ {
+		switch v[i] {
+		case '\\':
+			i++
+		case '"':
+			return i
+		}
+	}
+	return -1
+}
+
+// decodedLength is the wire length of a presentation-format string, in which
+// \ddd is one byte and \c is the character c.
+func decodedLength(s string) (int, error) {
+	n := 0
+	for i := 0; i < len(s); i++ {
+		if s[i] == '\\' {
+			switch {
+			case i+3 < len(s) && isDigit(s[i+1]) && isDigit(s[i+2]) && isDigit(s[i+3]):
+				if int(s[i+1]-'0')*100+int(s[i+2]-'0')*10+int(s[i+3]-'0') > 255 {
+					return 0, fmt.Errorf("escape \\%s is not a byte", s[i+1:i+4])
+				}
+				i += 3
+			case i+1 < len(s):
+				i++
+			default:
+				return 0, errors.New("value ends in a bare backslash")
+			}
+		}
+		n++
+	}
+	return n, nil
+}
+
+func isDigit(c byte) bool { return c >= '0' && c <= '9' }

spinifex/domains/route53/verifier.go

@@ -0,0 +1,296 @@
+package route53
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/reconciler"
+)
+
+const (
+	defaultVerifyInterval = time.Minute
+	// A new zone is checked every pass for this long, while its owner is most
+	// likely changing the registrar, and hourly after it.
+	newZoneWindow = 48 * time.Hour
+	slowRecheck   = time.Hour
+	activeRecheck = 24 * time.Hour
+	// lostAfter is how long an active zone's delegation may fail before the
+	// zone stops being served.
+	lostAfter = 72 * time.Hour
+
+	reasonContested = "the domain is active in another account"
+)
+
+// Delegation is the NS set a parent zone publishes for a name.
+type Delegation struct {
+	Source  string
+	Servers []string
+}
+
+// Verifier activates a zone once its parent delegates it to this cluster's
+// name servers, and stops serving one whose delegation has moved away.
+type Verifier struct {
+	store *Store
+
+	// Nameservers are this cluster's published name servers.
+	Nameservers func() []string
+	// Platform reads a delegation from a platform zone this cluster serves; ok
+	// is false when the name lies under none of them.
+	Platform func(ctx context.Context, name string) (d Delegation, ok bool, err error)
+	// Public asks the parent's authoritative servers, without recursion.
+	Public func(ctx context.Context, name string) (Delegation, error)
+	// Elect wins this pass for the calling node; nil means always.
+	Elect func(ctx context.Context) (release func(), ok bool)
+	Now   func() time.Time
+	// Interval is the pass period; zero means one minute.
+	Interval time.Duration
+}
+
+// NewVerifier returns a Verifier over store.
+func NewVerifier(store *Store) *Verifier {
+	return &Verifier{store: store, Now: time.Now}
+}
+
+// Run verifies on every interval. Nothing in KV changes when a registrar does,
+// so the loop is driven by time alone.
+func (v *Verifier) Run(ctx context.Context) {
+	interval := v.Interval
+	if interval <= 0 {
+		interval = defaultVerifyInterval
+	}
+	reconciler.Run(ctx, reconciler.Config{Name: "route53-verify", Reconcile: v.Reconcile, Resync: interval})
+}
+
+// Reconcile checks every zone that is due.
+func (v *Verifier) Reconcile(ctx context.Context) (time.Duration, error) {
+	if v.Elect != nil {
+		release, ok := v.Elect(ctx)
+		if !ok {
+			return 0, nil
+		}
+		defer release()
+	}
+	zones, err := v.store.ListAllZones(ctx)
+	if err != nil {
+		return 0, fmt.Errorf("list hosted zones: %w", err)
+	}
+	now := v.Now().UTC()
+	var due []*ZoneRecord
+	for i := range zones {
+		if checkDue(&zones[i], now) {
+			due = append(due, &zones[i])
+		}
+	}
+	return 0, v.verify(ctx, zones, due, now)
+}
+
+// Verify checks one zone now, whatever its schedule, and returns it updated.
+func (v *Verifier) Verify(ctx context.Context, account, id string) (*ZoneRecord, error) {
+	zones, err := v.store.ListAllZones(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("list hosted zones: %w", err)
+	}
+	i := slices.IndexFunc(zones, func(z ZoneRecord) bool { return z.AccountID == account && z.ID == id })
+	if i < 0 {
+		return v.store.GetZone(ctx, account, id) // reports not-found the usual way
+	}
+	if zones[i].Override == "" {
+		if err := v.verify(ctx, zones, []*ZoneRecord{&zones[i]}, v.Now().UTC()); err != nil {
+			return nil, err
+		}
+	}
+	return v.store.GetZone(ctx, account, id)
+}
+
+// checkDue reports whether a zone's next check has come round.
+func checkDue(z *ZoneRecord, now time.Time) bool {
+	if z.Override != "" || z.State == StateSuspended {
+		return false
+	}
+	var every time.Duration
+	switch {
+	case z.State == StateActive && z.FailingSince.IsZero():
+		every = activeRecheck
+	case z.State == StateActive:
+		every = slowRecheck
+	case now.Sub(z.CreatedAt) >= newZoneWindow:
+		every = slowRecheck
+	}
+	return now.Sub(z.CheckedAt) >= every
+}
+
+type outcome struct {
+	zone   *ZoneRecord
+	d      Delegation
+	reason string // empty when the delegation passes
+	err    error  // the check itself could not be made
+}
+
+func (v *Verifier) verify(ctx context.Context, all []ZoneRecord, due []*ZoneRecord, now time.Time) error {
+	ours := v.ourNameservers()
+	results := make([]outcome, 0, len(due))
+	for _, z := range due {
+		d, err := v.delegation(ctx, all, z)
+		o := outcome{zone: z, d: d, err: err}
+		if err == nil {
+			o.reason = compareDelegation(z.Name, d, ours)
+		}
+		results = append(results, o)
+	}
+
+	// First match wins: the earliest-created zone of a name that passes is the
+	// one activated, and no zone activates while another of its name is active.
+	slices.SortStableFunc(results, func(a, b outcome) int { return olderFirst(a.zone, b.zone) })
+	active := map[string]string{}
+	for _, z := range all {
+		if z.State == StateActive {
+			active[z.Name] = z.AccountID + "/" + z.ID
+		}
+	}
+
+	var errs []error
+	for _, o := range results {
+		self := o.zone.AccountID + "/" + o.zone.ID
+		contested := false
+		if holder, ok := active[o.zone.Name]; ok && holder != self {
+			contested = true
+		}
+		err := v.store.MutateZone(ctx, o.zone.AccountID, o.zone.ID, func(z *ZoneRecord) (bool, error) {
+			if z.Override != "" || z.State == StateSuspended {
+				return false, nil
+			}
+			applyOutcome(z, o, contested, now)
+			return true, nil
+		})
+		if err != nil {
+			errs = append(errs, fmt.Errorf("record check of %s: %w", o.zone.Name, err))
+			continue
+		}
+		if o.err == nil && o.reason == "" && !contested {
+			if _, ok := active[o.zone.Name]; !ok {
+				active[o.zone.Name] = self
+				slog.Info("route53: zone activated", "zone", o.zone.Name, "zone_id", o.zone.ID, "account", o.zone.AccountID, "parent", o.d.Source)
+			}
+		}
+	}
+	return errors.Join(errs...)
+}
+
+// applyOutcome moves a zone's activation state for one check.
+func applyOutcome(z *ZoneRecord, o outcome, contested bool, now time.Time) {
+	z.CheckedAt = now
+	if o.err != nil {
+		// Not knowing is not failing: an unreachable parent moves nothing.
+		z.StateReason = "the delegation could not be checked: " + o.err.Error()
+		return
+	}
+	z.Delegation, z.DelegationSource = o.d.Servers, o.d.Source
+	switch {
+	case o.reason != "":
+		z.StateReason = o.reason
+		if z.State == StateActive {
+			if z.FailingSince.IsZero() {
+				z.FailingSince = now
+			}
+			if now.Sub(z.FailingSince) >= lostAfter {
+				z.State = StateNameserversLost
+				slog.Warn("route53: zone lost its delegation", "zone", z.Name, "zone_id", z.ID, "reason", o.reason)
+			}
+		}
+	case z.State == StateActive:
+		z.StateReason, z.FailingSince = "", time.Time{}
+	case contested:
+		z.StateReason = reasonContested
+	default:
+		z.State, z.StateReason, z.FailingSince = StateActive, "", time.Time{}
+	}
+}
+
+// delegation finds the parent of z and reads what it publishes. A parent this
+// cluster serves is read from its records, never through DNS.
+func (v *Verifier) delegation(ctx context.Context, all []ZoneRecord, z *ZoneRecord) (Delegation, error) {
+	if parent := enclosingZone(all, z); parent != nil {
+		doc, _, err := v.store.GetRRSets(ctx, parent.AccountID, parent.ID)
+		if err != nil {
+			return Delegation{}, fmt.Errorf("read the parent zone %s: %w", parent.Name, err)
+		}
+		d := Delegation{Source: "hosted zone " + strings.TrimSuffix(parent.Name, ".")}
+		for _, set := range doc.Sets {
+			if set.Name == z.Name && set.Type == "NS" {
+				d.Servers = set.Values
+			}
+		}
+		return d, nil
+	}
+	if v.Platform != nil {
+		d, ok, err := v.Platform(ctx, z.Name)
+		if err != nil || ok {
+			return d, err
+		}
+	}
+	if v.Public == nil {
+		return Delegation{}, errors.New("no public DNS lookup is configured")
+	}
+	return v.Public(ctx, z.Name)
+}
+
+// enclosingZone returns the most specific active zone strictly above z, from
+// any account: that zone, not the public parent, decides who holds z.
+func enclosingZone(all []ZoneRecord, z *ZoneRecord) *ZoneRecord {
+	var best *ZoneRecord
+	for i := range all {
+		p := &all[i]
+		if p.State != StateActive || p.Name == z.Name || !withinZone(z.Name, p.Name) {
+			continue
+		}
+		if best == nil || len(p.Name) > len(best.Name) || (p.Name == best.Name && olderFirst(p, best) < 0) {
+			best = p
+		}
+	}
+	return best
+}
+
+func (v *Verifier) ourNameservers() map[string]bool {
+	ours := map[string]bool{}
+	if v.Nameservers != nil {
+		for _, ns := range v.Nameservers() {
+			ours[normalizeName(ns)] = true
+		}
+	}
+	return ours
+}
+
+// compareDelegation returns why d does not delegate name to this cluster, or ""
+// when it does. Every listed server must be ours: with a foreign one in the
+// set, a share of resolvers would get answers we do not control.
+func compareDelegation(name string, d Delegation, ours map[string]bool) string {
+	if len(ours) == 0 {
+		return "this cluster publishes no name servers"
+	}
+	if len(d.Servers) == 0 {
+		return fmt.Sprintf("%s does not delegate %s", d.Source, strings.TrimSuffix(name, "."))
+	}
+	var foreign []string
+	for _, s := range d.Servers {
+		if n := normalizeName(s); !ours[n] {
+			foreign = append(foreign, strings.TrimSuffix(n, "."))
+		}
+	}
+	if len(foreign) > 0 {
+		slices.Sort(foreign)
+		return fmt.Sprintf("%s delegates %s to name servers that are not this cluster's: %s", d.Source, strings.TrimSuffix(name, "."), strings.Join(foreign, ", "))
+	}
+	return ""
+}
+
+func olderFirst(a, b *ZoneRecord) int {
+	if c := a.CreatedAt.Compare(b.CreatedAt); c != 0 {
+		return c
+	}
+	return strings.Compare(a.ID, b.ID)
+}

spinifex/domains/route53/verifier_test.go

@@ -0,0 +1,274 @@
+package route53_test
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	r53 "github.com/aws/aws-sdk-go/service/route53"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/route53"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+)
+
+// fakeDNS is the public DNS tree: what each name's parent publishes for it.
+type fakeDNS struct {
+	parents map[string][]string
+	lookups int
+	err     error
+}
+
+func (f *fakeDNS) lookup(_ context.Context, name string) (route53.Delegation, error) {
+	f.lookups++
+	if f.err != nil {
+		return route53.Delegation{}, f.err
+	}
+	return route53.Delegation{Source: "the parent's name servers", Servers: f.parents[strings.TrimSuffix(name, ".")]}, nil
+}
+
+type clock struct{ t time.Time }
+
+func (c *clock) now() time.Time          { return c.t }
+func (c *clock) advance(d time.Duration) { c.t = c.t.Add(d) }
+func newClock() *clock                   { return &clock{t: time.Date(2026, 10, 24, 0, 0, 0, 0, time.UTC)} }
+func ours() []string                     { return []string{"ns1.spx3.net", "ns2.spx3.net"} }
+func zoneID(id string) string            { return strings.TrimPrefix(id, "/hostedzone/") }
+
+func newVerifier(t *testing.T) (*route53.Service, *route53.Verifier, *fakeDNS, *clock) {
+	t.Helper()
+	svc := newService(t)
+	c := newClock()
+	svc.Now = c.now
+	dns := &fakeDNS{parents: map[string][]string{}}
+	v := route53.NewVerifier(svc.Store())
+	v.Nameservers = ours
+	v.Public = dns.lookup
+	v.Now = c.now
+	return svc, v, dns, c
+}
+
+func zone(t *testing.T, svc *route53.Service, account, id string) *route53.ZoneRecord {
+	t.Helper()
+	z, err := svc.Store().GetZone(ctx, account, zoneID(id))
+	require.NoError(t, err)
+	return z
+}
+
+func pass(t *testing.T, v *route53.Verifier) {
+	t.Helper()
+	_, err := v.Reconcile(ctx)
+	require.NoError(t, err)
+}
+
+func TestVerifierActivatesOnlyAFullDelegationToUs(t *testing.T) {
+	svc, v, dns, _ := newVerifier(t)
+	cases := map[string]struct {
+		parent []string
+		state  string
+		reason string
+	}{
+		"all.example":     {[]string{"NS1.spx3.net.", "ns2.spx3.net"}, route53.StateActive, ""},
+		"subset.example":  {[]string{"ns2.spx3.net."}, route53.StateActive, ""},
+		"mixed.example":   {[]string{"ns1.spx3.net.", "ns-1.awsdns-01.org."}, route53.StatePendingNameservers, "not this cluster's: ns-1.awsdns-01.org"},
+		"foreign.example": {[]string{"dana.ns.cloudflare.com."}, route53.StatePendingNameservers, "not this cluster's"},
+		"none.example":    {nil, route53.StatePendingNameservers, "does not delegate none.example"},
+	}
+	ids := map[string]string{}
+	for name, tc := range cases {
+		dns.parents[name] = tc.parent
+		ids[name] = createZone(t, svc, acctA, name)
+	}
+	pass(t, v)
+	for name, tc := range cases {
+		z := zone(t, svc, acctA, ids[name])
+		assert.Equal(t, tc.state, z.State, name)
+		assert.Contains(t, z.StateReason, tc.reason, name)
+		assert.Equal(t, tc.parent, z.Delegation, name)
+	}
+}
+
+func TestVerifierFirstMatchWins(t *testing.T) {
+	svc, v, dns, c := newVerifier(t)
+	dns.parents["example.com"] = ours()
+	first := createZone(t, svc, acctA, "example.com")
+	c.advance(time.Second)
+	second := createZone(t, svc, acctB, "example.com")
+	pass(t, v)
+
+	assert.Equal(t, route53.StateActive, zone(t, svc, acctA, first).State)
+	later := zone(t, svc, acctB, second)
+	assert.Equal(t, route53.StatePendingNameservers, later.State)
+	assert.Equal(t, "the domain is active in another account", later.StateReason)
+
+	// The claim is not released by the loser checking again.
+	c.advance(time.Minute)
+	pass(t, v)
+	assert.Equal(t, route53.StatePendingNameservers, zone(t, svc, acctB, second).State)
+}
+
+func TestVerifierScheduleBacksOffAfterTheFirstTwoDays(t *testing.T) {
+	svc, v, dns, c := newVerifier(t)
+	createZone(t, svc, acctA, "slow.example")
+	pass(t, v)
+	require.Equal(t, 1, dns.lookups)
+
+	c.advance(time.Minute)
+	pass(t, v)
+	assert.Equal(t, 2, dns.lookups, "a new zone is checked every pass")
+
+	c.advance(48 * time.Hour)
+	pass(t, v)
+	c.advance(30 * time.Minute)
+	pass(t, v)
+	assert.Equal(t, 3, dns.lookups, "an old pending zone is checked hourly")
+	c.advance(30 * time.Minute)
+	pass(t, v)
+	assert.Equal(t, 4, dns.lookups)
+}
+
+func TestVerifierLosesAZoneOnlyAfterThreeDaysOfFailure(t *testing.T) {
+	svc, v, dns, c := newVerifier(t)
+	dns.parents["moved.example"] = ours()
+	id := createZone(t, svc, acctA, "moved.example")
+	pass(t, v)
+	require.Equal(t, route53.StateActive, zone(t, svc, acctA, id).State)
+
+	// The owner moves the domain away; the daily re-check notices.
+	dns.parents["moved.example"] = []string{"ns-1.awsdns-01.org."}
+	c.advance(24 * time.Hour)
+	pass(t, v)
+	z := zone(t, svc, acctA, id)
+	assert.Equal(t, route53.StateActive, z.State, "one failed check is not a lost delegation")
+	assert.Equal(t, c.t, z.FailingSince)
+
+	c.advance(71 * time.Hour)
+	pass(t, v)
+	assert.Equal(t, route53.StateActive, zone(t, svc, acctA, id).State)
+	c.advance(time.Hour)
+	pass(t, v)
+	assert.Equal(t, route53.StateNameserversLost, zone(t, svc, acctA, id).State)
+
+	// It comes back when the delegation does.
+	dns.parents["moved.example"] = ours()
+	c.advance(time.Hour)
+	pass(t, v)
+	z = zone(t, svc, acctA, id)
+	assert.Equal(t, route53.StateActive, z.State)
+	assert.True(t, z.FailingSince.IsZero())
+}
+
+func TestVerifierTreatsAnUnreachableParentAsUnknown(t *testing.T) {
+	svc, v, dns, c := newVerifier(t)
+	dns.parents["flaky.example"] = ours()
+	id := createZone(t, svc, acctA, "flaky.example")
+	pass(t, v)
+
+	dns.err = errors.New("i/o timeout")
+	for range 5 {
+		c.advance(24 * time.Hour)
+		pass(t, v)
+	}
+	z := zone(t, svc, acctA, id)
+	assert.Equal(t, route53.StateActive, z.State)
+	assert.True(t, z.FailingSince.IsZero())
+	assert.Contains(t, z.StateReason, "could not be checked: i/o timeout")
+}
+
+// A subdomain of an active zone is decided by that zone's records, never by
+// public DNS, so account B cannot take a name under account A's domain.
+func TestVerifierReadsAnActiveParentZoneLocally(t *testing.T) {
+	svc, v, dns, c := newVerifier(t)
+	dns.parents["example.com"] = ours()
+	dns.parents["shop.example.com"] = ours() // what a lookup would wrongly accept
+	parent := createZone(t, svc, acctA, "example.com")
+	pass(t, v)
+	require.Equal(t, route53.StateActive, zone(t, svc, acctA, parent).State)
+
+	child := createZone(t, svc, acctB, "shop.example.com")
+	c.advance(time.Minute)
+	pass(t, v)
+	z := zone(t, svc, acctB, child)
+	assert.Equal(t, route53.StatePendingNameservers, z.State)
+	assert.Equal(t, "hosted zone example.com does not delegate shop.example.com", z.StateReason)
+
+	_, err := apply(svc, acctA, parent, change("CREATE", "shop.example.com", "NS", 300, "ns1.spx3.net.", "ns2.spx3.net."))
+	require.NoError(t, err)
+	c.advance(time.Minute)
+	pass(t, v)
+	z = zone(t, svc, acctB, child)
+	assert.Equal(t, route53.StateActive, z.State)
+	assert.Equal(t, "hosted zone example.com", z.DelegationSource)
+}
+
+func TestVerifierReadsAPlatformZoneLocally(t *testing.T) {
+	svc, v, dns, _ := newVerifier(t)
+	v.Platform = func(_ context.Context, name string) (route53.Delegation, bool, error) {
+		if !strings.HasSuffix(name, ".spx3.net.") {
+			return route53.Delegation{}, false, nil
+		}
+		return route53.Delegation{Source: "platform zone spx3.net", Servers: ours()}, true, nil
+	}
+	id := createZone(t, svc, acctA, "app.spx3.net")
+	pass(t, v)
+	assert.Equal(t, route53.StateActive, zone(t, svc, acctA, id).State)
+	assert.Zero(t, dns.lookups)
+}
+
+func TestVerifierLeavesOverriddenZonesAlone(t *testing.T) {
+	svc, v, dns, c := newVerifier(t)
+	dns.parents["held.example"] = ours()
+	id := createZone(t, svc, acctA, "held.example")
+	require.NoError(t, svc.Store().MutateZone(ctx, acctA, zoneID(id), func(z *route53.ZoneRecord) (bool, error) {
+		z.State, z.Override = route53.StateSuspended, route53.StateSuspended
+		return true, nil
+	}))
+	pass(t, v)
+	c.advance(time.Hour)
+	_, err := v.Verify(ctx, acctA, zoneID(id))
+	require.NoError(t, err)
+	assert.Equal(t, route53.StateSuspended, zone(t, svc, acctA, id).State)
+	assert.Zero(t, dns.lookups)
+}
+
+func TestVerifyChecksImmediately(t *testing.T) {
+	svc, v, dns, _ := newVerifier(t)
+	id := createZone(t, svc, acctA, "now.example")
+	pass(t, v)
+	dns.parents["now.example"] = ours()
+
+	z, err := v.Verify(ctx, acctA, zoneID(id))
+	require.NoError(t, err)
+	assert.Equal(t, route53.StateActive, z.State)
+
+	_, err = v.Verify(ctx, acctB, zoneID(id))
+	require.ErrorIs(t, err, kvstore.ErrNotFound, "another account's zone is not found")
+}
+
+func TestVerifierNeedsPublishedNameservers(t *testing.T) {
+	svc, v, dns, _ := newVerifier(t)
+	v.Nameservers = func() []string { return nil }
+	dns.parents["any.example"] = ours()
+	id := createZone(t, svc, acctA, "any.example")
+	pass(t, v)
+	z := zone(t, svc, acctA, id)
+	assert.Equal(t, route53.StatePendingNameservers, z.State)
+	assert.Equal(t, "this cluster publishes no name servers", z.StateReason)
+}
+
+func TestActivatedZoneChangesWaitForSync(t *testing.T) {
+	svc, v, dns, _ := newVerifier(t)
+	dns.parents["sync.example"] = ours()
+	id := createZone(t, svc, acctA, "sync.example")
+	pass(t, v)
+	out, err := svc.ChangeResourceRecordSets(ctx, &r53.ChangeResourceRecordSetsInput{
+		HostedZoneId: aws.String(id),
+		ChangeBatch:  &r53.ChangeBatch{Changes: []*r53.Change{change("CREATE", "www.sync.example", "A", 60, "192.0.2.1")}},
+	}, acctA)
+	require.NoError(t, err)
+	assert.Equal(t, "PENDING", aws.StringValue(out.ChangeInfo.Status))
+}

spinifex/foundation/aws/errors/awserrors.go

@@ -409,6 +409,19 @@ var (
 	// that exists but has not been issued yet (still PENDING_VALIDATION), so a
 	// client polling for the body can tell "not ready" apart from "not mine".
 	ErrorACMRequestInProgress = "RequestInProgressException"
+	// Route 53 REST-XML error codes, from the 2013-04-01 API reference.
+	ErrorNoSuchHostedZone          = "NoSuchHostedZone"
+	ErrorHostedZoneAlreadyExists   = "HostedZoneAlreadyExists"
+	ErrorHostedZoneNotEmpty        = "HostedZoneNotEmpty"
+	ErrorInvalidChangeBatch        = "InvalidChangeBatch"
+	ErrorNoSuchChange              = "NoSuchChange"
+	ErrorTooManyHostedZones        = "TooManyHostedZones"
+	ErrorInvalidDomainName         = "InvalidDomainName"
+	ErrorConflictingDomainExists   = "ConflictingDomainExists"
+	ErrorPriorRequestNotComplete   = "PriorRequestNotComplete"
+	ErrorRoute53InvalidArgument    = "InvalidArgument"
+	ErrorRoute53NoSuchHealthCheck  = "NoSuchHealthCheck"
+	ErrorRoute53TooManyTagsForZone = "TooManyTags"
 	// ECS JSON-1.1 exception codes. ECS clients key on the "Exception"-suffixed
 	// __type; do not cross-wire with the EC2/ACM not-found codes above.
 	ErrorECSClusterNotFound                             = "ClusterNotFoundException"
@@ -658,13 +671,22 @@ func HasDependencies(kind, id string) error {
 // retryableError wraps a registered code with a suggested Retry-After
 // duration, for a 503 that a client can expect to clear on its own within a
 // bounded window (e.g. a warm-up race) rather than an open-ended outage.
-// Error() returns only the code, so code resolution is unaffected.
+// Error() returns only the code, or the wrapped error's text, so code
+// resolution is unaffected.
 type retryableError struct {
 	code       string
+	err        error
 	retryAfter time.Duration
 }
 
-func (e *retryableError) Error() string { return e.code }
+func (e *retryableError) Error() string {
+	if e.err != nil {
+		return e.err.Error()
+	}
+	return e.code
+}
+
+func (e *retryableError) Unwrap() error { return e.err }
 
 // RetryAfter wraps code (a registered ErrorLookup key) with a suggested
 // Retry-After duration for ErrorHandler to surface as a response header.
@@ -672,6 +694,13 @@ func RetryAfter(code string, d time.Duration) error {
 	return &retryableError{code: code, retryAfter: d}
 }
 
+// WithRetryAfter attaches a suggested Retry-After duration to err without
+// changing its text, unlike joining err with RetryAfter, which appends the
+// bare code on a second line.
+func WithRetryAfter(err error, d time.Duration) error {
+	return &retryableError{err: err, retryAfter: d}
+}
+
 // ResolveRetryAfter returns the Retry-After duration attached to err via
 // RetryAfter, if any, walking the same unwrap tree ResolveErrorDetail does.
 func ResolveRetryAfter(err error) (time.Duration, bool) {
@@ -726,12 +755,16 @@ var errorLookupByService = withNeutralValidationMessages(map[string]map[string]E
 	"rds": {
 		ErrorOperationNotSupported: {HTTPCode: 400, Message: "The specified RDS action is not supported in the RDS v1 API."},
 	},
+	// Route 53 throttles with a 400, not the 503 the shared entry carries.
+	"route53": {
+		ErrorThrottling: {HTTPCode: 400, Message: "Rate exceeded."},
+	},
 })
 
 // nonEC2Services are the gateway's services other than EC2, whose validation
 // errors must not fall back to ErrorLookup's EC2 wording.
 var nonEC2Services = []string{
-	"iam", "sts", "elasticloadbalancing", "eks", "ecs", "ecr", "acm", "rds", "tagging", "spinifex",
+	"iam", "sts", "elasticloadbalancing", "eks", "ecs", "ecr", "acm", "rds", "route53", "tagging", "spinifex",
 	"bedrock", "bedrock-runtime", "bedrock-agent", "bedrock-agent-runtime",
 }
 
@@ -1333,4 +1366,18 @@ var ErrorLookup = map[string]ErrorMessage{
 	ErrorModelErrorException:           {HTTPCode: 424, Message: "The request failed because of an error while running the model."},
 	ErrorServiceQuotaExceededException: {HTTPCode: 400, Message: "The number of requests exceeds the service quota. Resubmit your request later."},
 	ErrorConflictException:             {HTTPCode: 409, Message: "There was a conflict performing an operation."},
+
+	// Route 53.
+	ErrorNoSuchHostedZone:          {HTTPCode: 404, Message: "No hosted zone exists with the ID that you specified."},
+	ErrorHostedZoneAlreadyExists:   {HTTPCode: 409, Message: "The hosted zone you're trying to create already exists."},
+	ErrorHostedZoneNotEmpty:        {HTTPCode: 400, Message: "The hosted zone contains resource records that are not SOA or NS records."},
+	ErrorInvalidChangeBatch:        {HTTPCode: 400, Message: "This exception contains a list of messages that might contain one or more error messages."},
+	ErrorNoSuchChange:              {HTTPCode: 404, Message: "A change with the specified change ID does not exist."},
+	ErrorTooManyHostedZones:        {HTTPCode: 400, Message: "This operation can't be completed because the current account has reached the limit on the number of hosted zones."},
+	ErrorInvalidDomainName:         {HTTPCode: 400, Message: "The specified domain name is not valid."},
+	ErrorConflictingDomainExists:   {HTTPCode: 400, Message: "The domain name is already in use by another hosted zone."},
+	ErrorPriorRequestNotComplete:   {HTTPCode: 400, Message: "Route 53 is still processing a previous request for this hosted zone. Try again later."},
+	ErrorRoute53InvalidArgument:    {HTTPCode: 400, Message: "Parameter name is not valid."},
+	ErrorRoute53NoSuchHealthCheck:  {HTTPCode: 404, Message: "No health check exists with the specified ID."},
+	ErrorRoute53TooManyTagsForZone: {HTTPCode: 400, Message: "The resource has too many tags."},
 }

spinifex/foundation/aws/errors/awserrors_test.go

@@ -176,6 +176,24 @@ func TestResolveRetryAfter_WrappedAndResolvable(t *testing.T) {
 	}
 }
 
+// WithRetryAfter keeps the coded error's text and message while still
+// carrying the hint: a joined RetryAfter would log the code twice.
+func TestWithRetryAfter_KeepsTextAndResolves(t *testing.T) {
+	inner := Errorf(ErrorServiceUnavailable, "probe incomplete")
+	err := WithRetryAfter(inner, 5*time.Second)
+
+	if err.Error() != inner.Error() {
+		t.Fatalf("Error() = %q, want %q", err.Error(), inner.Error())
+	}
+	if d, ok := ResolveRetryAfter(fmt.Errorf("wrapped: %w", err)); !ok || d != 5*time.Second {
+		t.Fatalf("ResolveRetryAfter() = (%v, %v), want (5s, true)", d, ok)
+	}
+	code, msg, ok := ResolveErrorDetail(err)
+	if !ok || code != ErrorServiceUnavailable || msg != "probe incomplete" {
+		t.Fatalf("ResolveErrorDetail() = (%q, %q, %v), want (%q, %q, true)", code, msg, ok, ErrorServiceUnavailable, "probe incomplete")
+	}
+}
+
 // TestResolveRetryAfter_PlainErrorNotFound is the compatibility case: an
 // ordinary errors.New(code) -- the overwhelming majority of call sites --
 // carries no Retry-After hint.

spinifex/gateway/admin_accountquota.go

@@ -43,6 +43,9 @@ func (gw *GatewayConfig) adminGetAccountQuota(ctx context.Context, body []byte)
 
 	over, limits, err := gw.Quota.GetAccountQuota(ctx, accountID)
 	if err != nil {
+		if awserrors.IsErrorCode(err, awserrors.ErrorBlocked) {
+			return nil, err
+		}
 		slog.ErrorContext(ctx, "GetAccountQuota: failed to read override", "accountID", accountID, "err", err)
 		return nil, errors.New(awserrors.ErrorInternalError)
 	}
@@ -61,6 +64,9 @@ func (gw *GatewayConfig) adminPutAccountQuota(ctx context.Context, body []byte)
 	// The caller's own identity, so a quota change names who made it.
 	updatedBy, _ := ctx.Value(ctxIdentity).(string)
 	if err := gw.Quota.PutAccountQuota(ctx, accountID, over, updatedBy); err != nil {
+		if awserrors.IsErrorCode(err, awserrors.ErrorBlocked) {
+			return nil, err
+		}
 		slog.ErrorContext(ctx, "PutAccountQuota: failed to store override", "accountID", accountID, "err", err)
 		return nil, errors.New(awserrors.ErrorInternalError)
 	}
@@ -69,6 +75,9 @@ func (gw *GatewayConfig) adminPutAccountQuota(ctx context.Context, body []byte)
 
 	stored, limits, err := gw.Quota.GetAccountQuota(ctx, accountID)
 	if err != nil {
+		if awserrors.IsErrorCode(err, awserrors.ErrorBlocked) {
+			return nil, err
+		}
 		slog.ErrorContext(ctx, "PutAccountQuota: failed to read back override", "accountID", accountID, "err", err)
 		return nil, errors.New(awserrors.ErrorInternalError)
 	}

spinifex/gateway/admin_accountquota_test.go

@@ -205,3 +205,56 @@ func TestAccountQuotaMethodsAreRegistered(t *testing.T) {
 	require.Contains(t, AdminMethodNames(), "GetAccountQuota")
 	require.Contains(t, AdminMethodNames(), "PutAccountQuota")
 }
+
+// quotaStubChecker answers AccountTerminating from a fixed set, so these
+// tests can seal a record without a real IAM bucket.
+type quotaStubChecker struct {
+	terminating map[string]bool
+}
+
+func (c *quotaStubChecker) AccountTerminating(_ context.Context, accountID string) (bool, error) {
+	return c.terminating[accountID], nil
+}
+
+// A request body cannot carry or alter the stored state: an attempted
+// "state":"sealed" in the overrides object is simply unknown to the wire
+// type, the stored record stays active, and the response never shows state.
+func TestAdminRequestCannotCarryState(t *testing.T) {
+	gw, account := quotaTestGateway(t)
+	ctx := context.WithValue(t.Context(), ctxIdentity, "operator")
+
+	body := []byte(`{"accountId":"` + account + `","overrides":{"vcpus":32,"state":"sealed"}}`)
+	output, err := gw.adminPutAccountQuota(ctx, body)
+	require.NoError(t, err)
+
+	resp, ok := output.(*AccountQuotaResponse)
+	require.True(t, ok)
+	require.Equal(t, 32, resp.Limits["vcpus"])
+
+	data, err := json.Marshal(resp.Overrides)
+	require.NoError(t, err)
+	require.NotContains(t, string(data), "state")
+
+	_, _, err = gw.Quota.GetAccountQuota(ctx, account)
+	require.NoError(t, err, "the record must still be active, not sealed")
+}
+
+// Admin get and the put read-back on a sealed record must return Blocked,
+// not the generic InternalError every other read failure maps to.
+func TestAdminQuotaReadsMapBlocked(t *testing.T) {
+	gw, account := quotaTestGateway(t)
+	gw.Quota.SetAccountTerminationChecker(&quotaStubChecker{terminating: map[string]bool{account: true}})
+	_, err := gw.Quota.SealAccountOverrides(t.Context(), account)
+	require.NoError(t, err)
+
+	_, err = gw.adminGetAccountQuota(t.Context(), quotaRequestBody(t, AccountQuotaRequest{AccountID: account}))
+	require.Error(t, err)
+	require.Equal(t, awserrors.ErrorBlocked, err.Error())
+
+	ctx := context.WithValue(t.Context(), ctxIdentity, "operator")
+	_, err = gw.adminPutAccountQuota(ctx, quotaRequestBody(t, AccountQuotaRequest{
+		AccountID: account, Overrides: admissionquota.Overrides{VCPUs: quotaPtr(8)},
+	}))
+	require.Error(t, err)
+	require.Equal(t, awserrors.ErrorBlocked, err.Error())
+}

spinifex/gateway/admin_deleteaccount_job_test.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go/service/s3"
+	ec2v1 "github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/accountteardown"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
@@ -31,6 +32,10 @@ const deleteClientToken = "8b9c1d2e3f4a5b6c7d8e9f0a1b2c3d4e"
 // to keep step with the reapers NewClusterEngine wires.
 var teardownSubjects = []string{"ec2.>", "ecs.>", "eks.>", "rds.>", "elbv2.>", "acm.>", "bedrock.>"}
 
+// sealCapabilitySubjectPrefix matches every per-node seal-capability subject,
+// which the generic ec2.> stub below must not also answer.
+const sealCapabilitySubjectPrefix = "ec2.v1.AccountSettingsSealCapability."
+
 // newDeleteAccountGateway wires a gateway against a real IAM service and
 // embedded JetStream, with a stub cluster answering the subjects the reapers
 // drive. The tenant has no resources, so every stage drains empty and the test
@@ -46,8 +51,12 @@ func newDeleteAccountGateway(t *testing.T) *GatewayConfig {
 
 	for _, subject := range teardownSubjects {
 		// Replies carry a node ID: the instance inventory only counts a frame
-		// it can attribute to a configured node.
+		// it can attribute to a configured node. The seal subjects have their
+		// own responders below, which a second reply here would race.
 		sub, err := nc.Subscribe(subject, func(msg *nats.Msg) {
+			if msg.Subject == ec2v1.SealAccountSettingsSubject || strings.HasPrefix(msg.Subject, sealCapabilitySubjectPrefix) {
+				return
+			}
 			reply := nats.NewMsg(msg.Reply)
 			reply.Data = []byte(`{}`)
 			reply.Header.Set(natsmsg.NodeIDHeader, "node-1")
@@ -56,6 +65,38 @@ func newDeleteAccountGateway(t *testing.T) *GatewayConfig {
 		require.NoError(t, err)
 		t.Cleanup(func() { _ = sub.Unsubscribe() })
 	}
+
+	// The seal reaper needs a real sealed disposition and revision, and every
+	// configured node answering capability version 1, or it never finalises:
+	// the generic `{}` stub above is neither.
+	sealSub, err := nc.Subscribe(ec2v1.SealAccountSettingsSubject, func(msg *nats.Msg) {
+		var req ec2v1.SealAccountSettingsRequest
+		if json.Unmarshal(msg.Data, &req) != nil {
+			return
+		}
+		data, _ := json.Marshal(ec2v1.SealAccountSettingsReply{
+			AccountID: req.AccountID, CorrelationRef: req.CorrelationRef,
+			Disposition: ec2v1.AccountSettingsDispositionSealed, Revision: 1,
+		})
+		reply := nats.NewMsg(msg.Reply)
+		reply.Data = data
+		reply.Header.Set(natsmsg.NodeIDHeader, "node-1")
+		_ = msg.RespondMsg(reply)
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sealSub.Unsubscribe() })
+
+	for _, node := range []string{"node-1"} {
+		capSub, err := nc.Subscribe(ec2v1.AccountSettingsSealCapabilitySubject(node), func(msg *nats.Msg) {
+			data, _ := json.Marshal(ec2v1.AccountSettingsSealCapabilityReply{Node: node, CapabilityVersion: ec2v1.AccountSettingsSealCapabilityVersion})
+			reply := nats.NewMsg(msg.Reply)
+			reply.Data = data
+			reply.Header.Set(natsmsg.NodeIDHeader, node)
+			_ = msg.RespondMsg(reply)
+		})
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = capSub.Unsubscribe() })
+	}
 	require.NoError(t, nc.Flush())
 
 	return &GatewayConfig{

spinifex/gateway/auth.go

@@ -158,9 +158,9 @@ func (gw *GatewayConfig) SigV4AuthMiddleware() func(http.Handler) http.Handler {
 				return
 			}
 
-			// Region is pinned to the gateway; service is the client-claimed scope,
-			// already gated against supportedServices above.
-			if _, err := sig.Verify(secret, gw.Region, sig.Credential.Service); err != nil {
+			// Region is the gateway's, or a global service's fixed signing region;
+			// service is the client-claimed scope, gated against supportedServices above.
+			if _, err := sig.Verify(secret, gw.expectedSigningRegion(sig.Credential.Service, sig.Credential.Region), sig.Credential.Service); err != nil {
 				// A mismatch means our canonical request differs from the one the client
 				// signed, so log ours to diff against the SDK's. Warn, not Debug: these
 				// failures are intermittent and never reproduce on demand.
@@ -558,3 +558,20 @@ func signedService(r *http.Request) string {
 	svc, _ := r.Context().Value(ctxService).(string)
 	return svc
 }
+
+// globalSigningRegions maps a global AWS service to the region its clients sign
+// with when they use the default endpoint. A client given an endpoint override
+// signs with its own configured region instead, so both are accepted.
+var globalSigningRegions = map[string]string{
+	"route53": "us-east-1",
+}
+
+// expectedSigningRegion returns the credential-scope region a request for
+// service must carry: the global service's signing region when the client
+// claimed it, else the gateway's own region.
+func (gw *GatewayConfig) expectedSigningRegion(service, claimed string) string {
+	if global, ok := globalSigningRegions[service]; ok && claimed == global {
+		return global
+	}
+	return gw.Region
+}

spinifex/gateway/auth_test.go

@@ -2580,14 +2580,14 @@ func TestRedactedCanonicalRequest_MasksSessionToken(t *testing.T) {
 // credentials, must carry InvalidAction (the same code an unimplemented
 // action on a served service already returns) rather than a signature
 // failure that never happened, and must render in the generic REST-XML
-// envelope rather than the EC2 query shape a REST-XML client like route53's
-// cannot deserialize.
+// envelope rather than the EC2 query shape a non-query client cannot
+// deserialize.
 func TestSigV4Auth_UnservedServiceNamesServiceNotCredentials(t *testing.T) {
 	handler := setupTestApp(testAccessKey, testSecretKey)
 
 	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Host = "localhost:9999"
-	signTestRequestAs(t, req, nil, testAccessKey, testSecretKey, "route53", time.Now().UTC())
+	signTestRequestAs(t, req, nil, testAccessKey, testSecretKey, "sqs", time.Now().UTC())
 
 	resp := doRequest(handler, req)
 	require.Equal(t, http.StatusBadRequest, resp.StatusCode)
@@ -2606,7 +2606,7 @@ func TestSigV4Auth_UnservedServiceNamesServiceNotCredentials(t *testing.T) {
 
 	assert.Equal(t, "ErrorResponse", xmlRootName(t, body))
 	assert.Equal(t, awserrors.ErrorInvalidAction, parsed.Error.Code)
-	assert.Contains(t, parsed.Error.Message, "route53")
+	assert.Contains(t, parsed.Error.Message, "sqs")
 	assert.NotContains(t, parsed.Error.Message, "credentials",
 		"nothing was wrong with the credentials, which were never checked")
 }
@@ -2803,3 +2803,20 @@ func TestSigV4Auth_ResolvesCtxActionForNonQueryServices(t *testing.T) {
 		})
 	}
 }
+
+// Route 53 is global: the AWS CLI and SDKs sign it for us-east-1 against the
+// default endpoint and for the configured region against an override. Every
+// other service stays pinned to the gateway's region.
+func TestExpectedSigningRegion(t *testing.T) {
+	gw := &GatewayConfig{Region: "ap-southeast-2"}
+	for _, tc := range []struct {
+		service, claimed, want string
+	}{
+		{"route53", "us-east-1", "us-east-1"},
+		{"route53", "ap-southeast-2", "ap-southeast-2"},
+		{"route53", "eu-west-1", "ap-southeast-2"},
+		{"ec2", "us-east-1", "ap-southeast-2"},
+	} {
+		assert.Equalf(t, tc.want, gw.expectedSigningRegion(tc.service, tc.claimed), "%s signed for %s", tc.service, tc.claimed)
+	}
+}

spinifex/gateway/dispatch_seam_test.go

@@ -385,7 +385,7 @@ func TestDispatchSeam_UnregisteredUnknownServiceStillInvalidAction(t *testing.T)
 
 	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Host = "localhost:9999"
-	signTestRequestAs(t, req, nil, testAccessKey, testSecretKey, "route53", time.Now().UTC())
+	signTestRequestAs(t, req, nil, testAccessKey, testSecretKey, "sqs", time.Now().UTC())
 
 	resp := doRequest(r, req)
 	require.Equal(t, http.StatusBadRequest, resp.StatusCode)
@@ -400,7 +400,7 @@ func TestDispatchSeam_UnregisteredUnknownServiceStillInvalidAction(t *testing.T)
 	}
 	require.NoError(t, xml.Unmarshal(body, &parsed))
 	assert.Equal(t, awserrors.ErrorInvalidAction, parsed.Error.Code)
-	assert.Contains(t, parsed.Error.Message, "route53")
+	assert.Contains(t, parsed.Error.Message, "sqs")
 }
 
 // --- h. ValidateServices overlap guard --------------------------------------

spinifex/gateway/elbv2.go

@@ -187,6 +187,9 @@ func accountLoadBalancerLimit(ctx context.Context, gw *GatewayConfig, accountID
 	}
 	_, limits, err := gw.Quota.GetAccountQuota(ctx, accountID)
 	if err != nil {
+		if awserrors.IsErrorCode(err, awserrors.ErrorBlocked) {
+			return 0, err
+		}
 		slog.ErrorContext(ctx, "DescribeAccountLimits: failed to read account quota", "accountID", accountID, "err", err)
 		return 0, errors.New(awserrors.ErrorInternalError)
 	}

spinifex/gateway/elbv2_accountlimits_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/service/elbv2"
 	"github.com/mulgadc/spinifex/spinifex/domains/admission/quota"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	gateway_elbv2 "github.com/mulgadc/spinifex/spinifex/gateway/elbv2"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -121,6 +122,20 @@ func TestELBv2AccountLoadBalancerLimitDefaultsForExemptAccount(t *testing.T) {
 	assert.Equal(t, gateway_elbv2.DefaultLoadBalancerLimit, limit)
 }
 
+// A sealed override must refuse the load balancer cap with Blocked rather
+// than the generic InternalError every other read failure maps to.
+func TestELBv2AccountLoadBalancerLimitBlockedOnSealedRecord(t *testing.T) {
+	gw, account := quotaTestGateway(t)
+	gw.Quota.SetAccountTerminationChecker(&quotaStubChecker{terminating: map[string]bool{account: true}})
+	_, err := gw.Quota.SealAccountOverrides(t.Context(), account)
+	require.NoError(t, err)
+
+	limit, err := accountLoadBalancerLimit(t.Context(), gw, account)
+	require.Error(t, err)
+	require.Equal(t, awserrors.ErrorBlocked, err.Error())
+	assert.Zero(t, limit)
+}
+
 // The dispatch entry must accept the paging parameters the model defines and
 // still answer with the whole list.
 func TestELBv2DescribeAccountLimitsAcceptsPaging(t *testing.T) {

spinifex/gateway/gateway.go

@@ -464,10 +464,10 @@ func (gw *GatewayConfig) writeThrottleError(w http.ResponseWriter, r *http.Reque
 	svc, _ := r.Context().Value(ctxService).(string)
 
 	errorCode := awserrors.ErrorRequestLimitExceeded
-	if svc == "iam" || svc == "sts" || svc == "eks" {
+	if svc == "iam" || svc == "sts" || svc == "eks" || svc == "route53" {
 		errorCode = awserrors.ErrorThrottling
 	}
-	errorMsg := awserrors.ErrorLookup[errorCode]
+	errorMsg := awserrors.LookupErrorMessage(svc, errorCode)
 
 	// AWS JSON 1.1 services (EKS/ECS/bedrock family, …) get a JSON body.
 	if gw.jsonErrorService(svc) {

spinifex/gateway/gateway_test.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	acmawsapi "github.com/mulgadc/spinifex/spinifex/domains/acm/awsapi"
 	awsapi "github.com/mulgadc/spinifex/spinifex/domains/ecr/awsapi"
+	route53awsapi "github.com/mulgadc/spinifex/spinifex/domains/route53/awsapi"
 	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"io"
 	"maps"
@@ -241,8 +242,9 @@ func TestErrorHandler_NonEC2ValidationDefaults_DropEC2Wording(t *testing.T) {
 	b := dispatch.NewBuilder()
 	require.NoError(t, b.Register(awsapi.NewRegistration(awsapi.Deps{})))
 	require.NoError(t, b.Register(acmawsapi.NewRegistration(acmawsapi.Deps{})))
+	require.NoError(t, b.Register(route53awsapi.NewRegistration(route53awsapi.Deps{})))
 	gw.Services = b.Build()
-	services := append(slices.Collect(maps.Keys(supportedServices)), awsapi.ServiceName, acmawsapi.ServiceName)
+	services := append(slices.Collect(maps.Keys(supportedServices)), awsapi.ServiceName, acmawsapi.ServiceName, route53awsapi.ServiceName)
 	for _, svc := range services {
 		if svc == "ec2" {
 			continue
@@ -1460,6 +1462,23 @@ func TestWriteThrottleError_IAM(t *testing.T) {
 	assert.Contains(t, string(body), "<ErrorResponse>")
 }
 
+// Route 53 throttles with a 400 carrying the Throttling code, which SDK
+// retryers classify by code rather than status.
+func TestWriteThrottleError_Route53(t *testing.T) {
+	gw := &GatewayConfig{DisableLogging: true}
+	req := httptest.NewRequest(http.MethodGet, "/2013-04-01/hostedzone", nil)
+	req = req.WithContext(context.WithValue(req.Context(), ctxService, "route53"))
+	w := httptest.NewRecorder()
+
+	gw.writeThrottleError(w, req)
+
+	resp := w.Result()
+	body, _ := io.ReadAll(resp.Body)
+	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
+	assert.Contains(t, string(body), "<Code>Throttling</Code>")
+	assert.Contains(t, string(body), "<ErrorResponse>")
+}
+
 func TestWriteThrottleError_Bedrock(t *testing.T) {
 	gw := &GatewayConfig{DisableLogging: true}
 

spinifex/handlers/iam/account_status_reader.go

@@ -0,0 +1,56 @@
+package handlers_iam
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// AccountStatusReader reads account status from spinifex-accounts and never
+// writes. It is the read-only capability EC2's settings seal and the quota
+// override seal both use to check their TERMINATING precondition.
+type AccountStatusReader struct {
+	accounts *kvstore.Bucket // AttachOnly: never creates the bucket
+}
+
+// NewAccountStatusReader attaches to the existing spinifex-accounts bucket.
+// It needs no master key and runs no migration: a caller that only reads
+// account status must not bootstrap IAM's bucket out from under it.
+func NewAccountStatusReader(js jetstream.JetStream) *AccountStatusReader {
+	return &AccountStatusReader{
+		accounts: kvstore.NewBucket(js, kvstore.Config{
+			Name:       KVBucketAccounts,
+			Missing:    "account status reader: nil jetstream client",
+			AttachOnly: true,
+		}),
+	}
+}
+
+// AccountTerminating is a direct read, which suffices because TERMINATING is
+// one-way until the record is deleted. A missing record is (false, nil); only
+// a failed read or decode is an error.
+func (r *AccountStatusReader) AccountTerminating(ctx context.Context, accountID string) (bool, error) {
+	kv, err := r.accounts.KV(ctx)
+	if err != nil {
+		return false, err
+	}
+
+	entry, err := kv.Get(ctx, accountID)
+	if err != nil {
+		if errors.Is(err, jetstream.ErrKeyNotFound) {
+			return false, nil
+		}
+		return false, err
+	}
+
+	var record struct {
+		Status string `json:"status"`
+	}
+	if err := json.Unmarshal(entry.Value(), &record); err != nil {
+		return false, err
+	}
+	return record.Status == AccountStatusTerminating, nil
+}

spinifex/handlers/iam/account_status_reader_test.go

@@ -0,0 +1,102 @@
+package handlers_iam_test
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/internal/testkit"
+	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// newStatusReaderFixture starts a real IAM service (which creates and
+// migrates spinifex-accounts) and a reader attached to the same JetStream.
+func newStatusReaderFixture(t *testing.T) (handlers_iam.IAMService, *handlers_iam.AccountStatusReader, jetstream.JetStream) {
+	t.Helper()
+	_, nc, js := testutil.StartTestJetStream(t)
+
+	masterKey, err := handlers_iam.GenerateMasterKey()
+	require.NoError(t, err)
+	svc, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, masterKey)
+	require.NoError(t, err)
+
+	return svc, handlers_iam.NewAccountStatusReader(js), js
+}
+
+func TestAccountStatusReader_TerminatingIsTrue(t *testing.T) {
+	t.Parallel()
+	svc, reader, _ := newStatusReaderFixture(t)
+	account, err := svc.CreateAccount("tenant@example.com")
+	require.NoError(t, err)
+	_, err = svc.SetAccountStatus(account.AccountID, handlers_iam.AccountStatusTerminating)
+	require.NoError(t, err)
+
+	terminating, err := reader.AccountTerminating(t.Context(), account.AccountID)
+	require.NoError(t, err)
+	assert.True(t, terminating)
+}
+
+func TestAccountStatusReader_ActiveAndSuspendedAreFalse(t *testing.T) {
+	t.Parallel()
+	svc, reader, _ := newStatusReaderFixture(t)
+	active, err := svc.CreateAccount("active@example.com")
+	require.NoError(t, err)
+	terminating, err := reader.AccountTerminating(t.Context(), active.AccountID)
+	require.NoError(t, err)
+	assert.False(t, terminating)
+
+	suspended, err := svc.CreateAccount("suspended@example.com")
+	require.NoError(t, err)
+	_, err = svc.SetAccountStatus(suspended.AccountID, handlers_iam.AccountStatusSuspended)
+	require.NoError(t, err)
+	terminating, err = reader.AccountTerminating(t.Context(), suspended.AccountID)
+	require.NoError(t, err)
+	assert.False(t, terminating)
+}
+
+func TestAccountStatusReader_MissingRecordIsFalseNotError(t *testing.T) {
+	t.Parallel()
+	_, reader, _ := newStatusReaderFixture(t)
+	terminating, err := reader.AccountTerminating(t.Context(), "999999999999")
+	require.NoError(t, err)
+	assert.False(t, terminating)
+}
+
+// A missing bucket is an error, and the reader never creates one: a caller
+// that only reads account status must not bootstrap IAM's bucket.
+func TestAccountStatusReader_MissingBucketIsErrorAndNeverCreated(t *testing.T) {
+	t.Parallel()
+	_, nc, js := testutil.StartTestJetStream(t)
+	reader := handlers_iam.NewAccountStatusReader(js)
+
+	_, err := reader.AccountTerminating(t.Context(), "111111111111")
+	require.Error(t, err)
+
+	_, bucketErr := js.KeyValue(t.Context(), handlers_iam.KVBucketAccounts)
+	assert.ErrorIs(t, bucketErr, jetstream.ErrBucketNotFound)
+	_ = nc
+}
+
+func TestAccountStatusReader_UndecodableValueIsError(t *testing.T) {
+	t.Parallel()
+	_, nc, js := testutil.StartTestJetStream(t)
+	_, err := handlers_iam.NewIAMServiceImpl(t.Context(), nc, mustMasterKey(t))
+	require.NoError(t, err)
+
+	kv, err := js.KeyValue(t.Context(), handlers_iam.KVBucketAccounts)
+	require.NoError(t, err)
+	_, err = kv.Put(t.Context(), "222222222222", []byte(`{"status":`))
+	require.NoError(t, err)
+
+	reader := handlers_iam.NewAccountStatusReader(js)
+	_, err = reader.AccountTerminating(t.Context(), "222222222222")
+	require.Error(t, err)
+}
+
+func mustMasterKey(t *testing.T) []byte {
+	t.Helper()
+	key, err := handlers_iam.GenerateMasterKey()
+	require.NoError(t, err)
+	return key
+}

spinifex/handlers/iam/managed_policies.go

@@ -258,6 +258,53 @@ var awsManagedPolicyDocs = map[string]string{
 		]
 	}`,
 
+	"arn:aws:iam::aws:policy/AmazonRoute53FullAccess": `{
+		"Version": "2012-10-17",
+		"Statement": [
+			{
+				"Effect": "Allow",
+				"Action": [
+					"route53:*",
+					"route53domains:*",
+					"cloudfront:ListDistributions",
+					"elasticloadbalancing:DescribeLoadBalancers",
+					"elasticbeanstalk:DescribeEnvironments",
+					"s3:ListBucket",
+					"s3:GetBucketLocation",
+					"s3:GetBucketWebsite",
+					"ec2:DescribeVpcs",
+					"ec2:DescribeVpcEndpoints",
+					"ec2:DescribeRegions",
+					"sns:ListTopics",
+					"sns:ListSubscriptionsByTopic",
+					"cloudwatch:DescribeAlarms",
+					"cloudwatch:GetMetricStatistics"
+				],
+				"Resource": "*"
+			},
+			{
+				"Effect": "Allow",
+				"Action": "apigateway:GET",
+				"Resource": "arn:aws:apigateway:*::/domainnames"
+			}
+		]
+	}`,
+
+	"arn:aws:iam::aws:policy/AmazonRoute53ReadOnlyAccess": `{
+		"Version": "2012-10-17",
+		"Statement": [
+			{
+				"Effect": "Allow",
+				"Action": [
+					"route53:Get*",
+					"route53:List*",
+					"route53:TestDNSAnswer"
+				],
+				"Resource": "*"
+			}
+		]
+	}`,
+
 	"arn:aws:iam::aws:policy/AmazonEKSServicePolicy": `{
 		"Version": "2012-10-17",
 		"Statement": [

spinifex/handlers/iam/managed_policies_test.go

@@ -159,3 +159,30 @@ func TestResolveAttachedPolicy_UnmodeledRDSManagedARNGrantsNothing(t *testing.T)
 		assert.Empty(t, doc.Statement)
 	}
 }
+
+// Terraform users commonly carry only a Route 53 managed policy, so both must
+// resolve, and the read-only one must not reach a mutation.
+func TestRoute53ManagedShims(t *testing.T) {
+	t.Parallel()
+	full, ok := builtinManagedPolicyDoc("arn:aws:iam::aws:policy/AmazonRoute53FullAccess")
+	require.True(t, ok)
+	readOnly, ok := builtinManagedPolicyDoc("arn:aws:iam::aws:policy/AmazonRoute53ReadOnlyAccess")
+	require.True(t, ok)
+
+	reads := []string{"route53:GetHostedZone", "route53:ListHostedZones", "route53:ListResourceRecordSets",
+		"route53:GetChange", "route53:ListTagsForResource"}
+	writes := []string{"route53:CreateHostedZone", "route53:DeleteHostedZone",
+		"route53:ChangeResourceRecordSets", "route53:ChangeTagsForResource"}
+	for _, action := range append(append([]string{}, reads...), writes...) {
+		assert.Equal(t, iampolicy.Allow, iampolicy.EvaluateWithKeys(action, "*", []PolicyDocument{full}, nil),
+			"AmazonRoute53FullAccess must grant %s", action)
+	}
+	for _, action := range reads {
+		assert.Equal(t, iampolicy.Allow, iampolicy.EvaluateWithKeys(action, "*", []PolicyDocument{readOnly}, nil),
+			"AmazonRoute53ReadOnlyAccess must grant %s", action)
+	}
+	for _, action := range writes {
+		assert.Equal(t, iampolicy.Deny, iampolicy.EvaluateWithKeys(action, "*", []PolicyDocument{readOnly}, nil),
+			"AmazonRoute53ReadOnlyAccess must not grant %s", action)
+	}
+}

spinifex/ingress/aws/dispatch/registry_test.go

@@ -139,12 +139,12 @@ func TestLookupServesEveryNameOfOneRegistration(t *testing.T) {
 func TestUnknownServiceKeepsTheEstablishedResult(t *testing.T) {
 	b := NewBuilder()
 	require.NoError(t, b.Register(validReg("ecr")))
-	_, ok := b.Build().Lookup("route53")
+	_, ok := b.Build().Lookup("sqs")
 	assert.False(t, ok)
 
-	code, msg := UnknownService("route53")
+	code, msg := UnknownService("sqs")
 	assert.Equal(t, awserrors.ErrorInvalidAction, code)
-	assert.Equal(t, `Service "route53" is not served by this gateway.`, msg)
+	assert.Equal(t, `Service "sqs" is not served by this gateway.`, msg)
 }
 
 func TestResolveActionIsAdvisory(t *testing.T) {

spinifex/ingress/aws/dispatch/selector_test.go

@@ -88,7 +88,7 @@ func TestSelectorLegacyFallback(t *testing.T) {
 
 	assert.True(t, s.Served("ec2"))
 	assert.True(t, s.Served("acm"))
-	assert.False(t, s.Served("route53"))
+	assert.False(t, s.Served("sqs"))
 	assert.Equal(t, "LegacyAction", s.ResolveAction(req, "ec2"))
 	assert.False(t, s.JSONErrors("ec2"))
 	assert.True(t, s.JSONErrors("acm"))