feat: activate Route 53 hosted zones once their parent delegates them to the cluster

A zone stays PENDING_NAMESERVERS until its parent's NS set is exactly the
cluster's name servers, read from an active enclosing zone, the platform zone,
or a non-recursive walk from the roots. The first account to verify a name wins.
Adds the activation extension (GET/POST /hostedzone/{Id}/activation), the
spx admin dns zone and delegate commands, and an e2e that delegates a cluster
subdomain and checks every node serves it.

.github/scripts/suite-sets.sh

@@ -28,6 +28,9 @@
 # that runs identically on either, and because a quota left switched on would
 # cap every suite sharing the environment — so it must be proved to restore
 # itself wherever it runs, not only on one node.
+# route53 belongs to both topologies because every node serves every hosted
+# zone from one shared bucket: on one node it proves the zone loads at all, on
+# three that each node's sync reaches the same set.
 # storagefault runs last in both lists and belongs to both topologies, for
 # different reasons. On one node it is RS(1,0), so freezing the only predastore
 # is a total outage and the guest-survival assertion is unambiguous. On three it
@@ -37,7 +40,7 @@
 # received. It is last because it SIGSTOPs a cluster-wide
 # service: anything sharing the environment would see the same outage and report
 # a failure that says nothing about itself.
-E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota storagefault"
+E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota route53 storagefault"
 
 # Suites runnable against a multi-node environment.
 #
@@ -57,7 +60,7 @@ E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota
 # replicated rather than sitting on one node, so it runs before the pair that
 # take a node away — a cluster whose buckets are already wrong should report
 # that, not report a recovery failing.
-E2E_SUITES_MULTI="multinode lb cert quota kvquorum lbrecovery instancerecovery storagefault"
+E2E_SUITES_MULTI="multinode lb cert quota route53 kvquorum lbrecovery instancerecovery storagefault"
 
 # grep -xE alternation form, for narrowing a requested set to the eligible one.
 E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')"

docs/COMMANDS.md

@@ -110,6 +110,7 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 |---------|-------|-------------|
 | `spx admin kv digest` | `--store-dir` (default `<spinifex-dir>/nats/jetstream`), `--work-dir` (holds the temporary copy, default the system temp dir), `--stream` (repeatable filter), `--seqs` (also print a hash per sequence), `--json` (print one JSON document, the input `spx admin kv compare` reads) | Read-only per-replica consistency check. Copies this node's `$G` streams tree (without consumer state) to a temporary directory, opens the copy in a private in-process NATS server with no listener, and prints one line per stream with first/last sequence, message count and a SHA-256 over every stored message (sequence, subject, timestamp, headers, data). Run it on every node with `sudo` and diff the output with the `#` lines removed: replicas of one stream must match. A direct get over NATS is not used because any replica may answer it. Writes landing during the copy can change a replica's tail, so gate API writes or re-run before calling a tail difference divergence. Prefer `--json --seqs` on every node followed by `spx admin kv compare`, which tells a diverged replica from one that is merely behind. |
 | `spx admin kv compare <digest.json>...` | — (`-` reads any number of digests from stdin) | Reads the `spx admin kv digest --json --seqs` output of every node and checks that each stream's replicas hold the same content. A replica that is merely behind is not divergence; these are: the same sequence holding different content on two replicas, a message missing from one replica that nothing on it explains (no later write to the same subject, no expiry under `max_age` or a message TTL, no stream limit dropping its head), a stream held by fewer or more nodes than its replica count, replicas disagreeing on that count, or no sequence range common to every replica. A stream with a `max_age` whose replicas were copied at different times and share no sequence range is reported as `skipped` (volatile) rather than diverged, since a short-lived bucket such as a leader lease turns over between copies; any sequence two of its replicas share must still match, and each message a replica lacks must be superseded, expired, or written after that replica was copied. Sequences past the slowest replica's last are not compared. Exits 0 when every stream is consistent, 1 on divergence, 2 on bad input. Digests are taken from live stores seconds apart, so re-run before trusting a failure that a write landing between two copies could explain. `install-node.sh` gates cluster formation on it. |
+| `spx admin dns nameservers` | `--check` (compare against the parent zone's delegation), `--timeout` (per DNS query, default 5s) | Prints the `nsN` nameservers this cluster publishes for its base domain, with the node and address behind each. At most four are published however many nodes run northstar; `[northstar] nameserver_count` (1–4) and `nameserver_nodes` (ordered pin) in `spinifex.toml` set the count and order, and every northstar node keeps serving every zone either way. `--check` asks a server of the parent zone (a `.net` server for `spx3.net`) for the delegation without recursion and reports each name as `ok`, `not delegated`, `no glue`, `glue mismatch` or `not published here`, exiting non-zero unless all are `ok`. |
 | `spx admin cluster shutdown` | `--force` (shutdown even if nodes don't respond), `--timeout` (max wait per phase, default 120s), `--dry-run` (print phase plan without executing) | Performs coordinated, phased shutdown of entire cluster. Phases execute in order: GATE (stop API/UI) → DRAIN (stop VMs) → STORAGE (stop viperblock) → PERSIST (stop predastore) → INFRA (stop NATS/daemon). Each phase waits for all nodes to ACK before proceeding. Uses JetStream state tracking. |
 | `spx admin cluster drain-dhcp` | `--timeout` (reply-collection window, default 30s) | Asks each vpcd to DHCPRELEASE every external-pool DHCP lease it currently holds, returning them to the upstream DHCP server. Run on teardown before stopping services — an env reset otherwise strands held leases upstream until their TTL expires, eventually exhausting the upstream scope. Best-effort: warns and exits 0 if the cluster is already down. |
 | `spx admin node drain --local` | `--local` (drain the local node only, required), `--timeout` (max wait per phase, default 120s), `--unless-restarting` (drain on a real host shutdown or a plain target stop, and skip only when a restart of the stack is already queued; unset, it always drains) | Runs the GATE and DRAIN phases against the local node only: powers down its guests via QMP and unmounts their volumes (flushing the viperblock WAL) while every service is still up. STORAGE/PERSIST/INFRA are left to systemd's ordered teardown. Wired as the `ExecStop` of `spinifex-shutdown.service`, so `systemctl stop spinifex.target` and host reboot/poweroff drain guests before any storage service stops. |

docs/service-interfaces.yaml

@@ -31,6 +31,7 @@ services:
       - "ec2.*"
       - "elbv2.*"
       - "acm.*"
+      - "route53.*"
       - "iam.*"
       - "rds.*"
       - "ecs.*"
@@ -39,6 +40,7 @@ services:
       - "ec2.*"
       - "elbv2.*"
       - "acm.*"
+      - "route53.*"
       - "iam.*"
       - "rds.*"
       - "ecs.*"
@@ -76,6 +78,7 @@ services:
       - spinifex/domains/ec2/vpc
       - spinifex/handlers/elbv2
       - spinifex/domains/acm
+      - spinifex/domains/route53
       - spinifex/handlers/iam
       - spinifex/handlers/ecs
       - spinifex/handlers/rds
@@ -154,6 +157,10 @@ services:
       - "acm.DescribeCertificate"
       - "acm.ListCertificates"
       - "acm.DeleteCertificate"
+      # Route 53 hosted zones (domains/route53; subscribed in daemon.go).
+      - "route53.*"
+      # Operator-only zone activation and delegation, sent by spx admin dns.
+      - "route53.admin.*"
       - "elbv2.AddListenerCertificates"
       - "elbv2.RemoveListenerCertificates"
       - "elbv2.DescribeListenerCertificates"
@@ -466,6 +473,12 @@ suites:
     # Exercises the full daemon stack under network failure injection;
     # any service touching the wire path is in scope.
 
+  e2e-route53:
+    path: tests/e2e/route53
+    covers: [awsgw, spinifex_daemon, predastore]
+    # Hosted zones end to end: delegation check, publish to the northstar zone
+    # store in predastore, and every node answering before a change is INSYNC.
+
   # Suites below are bash-only today. Entries activate as the Go ports
   # land (Bead 2 fixtures unblock new-shape ports for multinode etc.).
   #

docs/testing/e2e-coverage.md

@@ -6,8 +6,8 @@ Suites live in `tests/e2e/<name>/` and build to `tests/e2e/_bin/<name>.test`. Ev
 
 | Set | Suites |
 | --- | --- |
-| `E2E_SUITES_SINGLE` | `single iam cert eks ecs storagegrowth partialblock rds quota storagefault` |
-| `E2E_SUITES_MULTI` | `multinode lb cert quota kvquorum lbrecovery instancerecovery storagefault` |
+| `E2E_SUITES_SINGLE` | `single iam cert eks ecs storagegrowth partialblock rds quota route53 storagefault` |
+| `E2E_SUITES_MULTI` | `multinode lb cert quota route53 kvquorum lbrecovery instancerecovery storagefault` |
 | `E2E_SUITES_NIGHTLY_SINGLE` | `single cert iam` |
 | `E2E_SUITES_NIGHTLY_MULTI` | `multinode cert lb` |
 
@@ -120,6 +120,7 @@ The create/describe/connect path against a live cluster: a DB VM that actually b
 ### Other
 
 - **`quota`** — every enforced dimension refuses with the same `quotaExceeded` code; a launch is charged for every instance it asks for; one instance wider than the cap is refused.
+- **`route53`** — hosted zones as northstar serves them, on every node. `TestNorthstarServesZonesBeyondFirstListPage` writes 1,100 zone objects (more than one `ListObjectsV2` page), requires the last one to resolve on every node and the platform base zone to survive the sync, then requires the deleted zones to stop resolving.
 - **`gpu`** — device visible in the guest, driver access, launch.
 - **`bedrock`** — `ListFoundationModels`/`GetFoundationModel` against a running cluster. Wire-format parity is unit-tested, not here.
 - **`ddil`** — Denied, Disrupted, Intermittent, Limited network scenarios.

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/config v1.33.9
 	github.com/aws/aws-sdk-go-v2/credentials v1.20.9
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3
 	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/charmbracelet/bubbles v1.0.0
 	github.com/charmbracelet/bubbletea v1.3.10
@@ -24,7 +25,7 @@ require (
 	github.com/miekg/dns v1.1.73
 	github.com/minio/highwayhash v1.0.4
 	github.com/mulgadc/bluebottle v1.22.0
-	github.com/mulgadc/northstar v1.22.0
+	github.com/mulgadc/northstar v1.22.1-0.20261023224547-8c5555ef1d1d
 	github.com/mulgadc/predastore v1.22.0
 	github.com/mulgadc/viperblock v1.22.0
 	github.com/nats-io/nats-server/v2 v2.15.1
@@ -67,10 +68,9 @@ require (
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 // indirect
 	github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect

go.sum

@@ -40,14 +40,14 @@ github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 h1:e6YW5uME30TkrElR+4/xcY7NB6CCj2dcv7vtmGXhfhQ=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21/go.mod h1:obpHAMP693+f3/A8PkLhIqfK1C+NWWn8zXzqD3OH520=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 h1:/TYsZXdA8UTa+WCtCYSAJIr1vwl0+eho6TUgJGwFFO8=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5/go.mod h1:qPqp1Uwd/BqdhPufv6oem9j5J7HNsgc2V22dUiDPn+s=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7 h1:CwvFlKw0uUScfgOMflXSRZa+ayXgOf6wk7lhyMemQYU=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.7/go.mod h1:4SbD8BB+p+QEnG0aX6DoWwrtk9SufyYXRsUv0nve8LE=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 h1:LHHdobjo5le1sjJdolitV7uVvYg3wkaeM1Ju2fDgeZQ=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6/go.mod h1:CBGB0evAiaMD3Lk4wPO7LQCcCZLuhzL5Qf/wfEUHQ5w=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 h1:pPiWfgeNxqluKEph7hvU88kuGKBPOWzO+Dk9t2zqqNs=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4/go.mod h1:YlwGoIUDG/3kBQbdNOVs/xKZ9J01G8e/6D1mRBj9uTk=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 h1:VMAdYqr4Jn/8ATs9BHC5riwrs0d6m1Z2ohFriSwZwm0=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0/go.mod h1:9APRWGLFITKD+xzWSIyT9V7QV4bNlEuIieWlzXgGFlI=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6 h1:mYViDr2azcgxOpRrObZOl631bvavqILoE/L+Kxgbwys=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.6/go.mod h1:OXVdAo9jNhggB+hJRGOyy7WolGTKB6Rg7IIulDeSzPM=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3 h1:NyeuyXkZL476uoAowNActm4XJO1QPKf7aassgXHpiqA=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3/go.mod h1:GfoBeKMDJecgRUsGf/0RGdx0NJAh81ObTd922h8pQys=
 github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A=
 github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M=
 github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk=
@@ -295,8 +295,12 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc
 github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
 github.com/mulgadc/bluebottle v1.22.0 h1:pGZ5/Ly3CYem62pPu2zKBRpvMpAG0LHMIMw9Yt/lkIg=
 github.com/mulgadc/bluebottle v1.22.0/go.mod h1:DwT91bJyUsrpg1gCJmcCOTG+tgw5J124LloHYvGAQu0=
-github.com/mulgadc/northstar v1.22.0 h1:fTndlk47hK76fcNr2MqsyeKhBNVEDt8+BC6Z30ohokA=
-github.com/mulgadc/northstar v1.22.0/go.mod h1:yLH8waqfHZx177Ae6dGQGTDt/02rPE9alnUKyxDNhVs=
+github.com/mulgadc/northstar v1.22.1-0.20261023220040-11ff69dd571a h1:juPJ0jubDYy6Sdk14BGnYB4QF7N0PhAmV522FdBA2yI=
+github.com/mulgadc/northstar v1.22.1-0.20261023220040-11ff69dd571a/go.mod h1:Halhinpgo5rzUhYemGN8uYTWu1pJsL5e/7l1eEKc7nM=
+github.com/mulgadc/northstar v1.22.1-0.20261023224401-31cf7e48e4db h1:OrXatiBIUEv6InRzh0zHIPnfp+CrIxf2+rX4T3fo7cI=
+github.com/mulgadc/northstar v1.22.1-0.20261023224401-31cf7e48e4db/go.mod h1:Halhinpgo5rzUhYemGN8uYTWu1pJsL5e/7l1eEKc7nM=
+github.com/mulgadc/northstar v1.22.1-0.20261023224547-8c5555ef1d1d h1:FTch26kEKKv+c4X3L0cQKMXwBJ4D1SCFlV4Mfgh7nEU=
+github.com/mulgadc/northstar v1.22.1-0.20261023224547-8c5555ef1d1d/go.mod h1:Halhinpgo5rzUhYemGN8uYTWu1pJsL5e/7l1eEKc7nM=
 github.com/mulgadc/predastore v1.22.0 h1:mUQJMTxMiRDVvemMP+34BP/+r1U0iCptc9B++u4NJ8o=
 github.com/mulgadc/predastore v1.22.0/go.mod h1:/cMIoKtnQG/DWyeX/0matEQrG2/abY60tNHWcBKS9Kc=
 github.com/mulgadc/viperblock v1.22.0 h1:8vS2AtpIDkIhGNOazQPicKXV2y/8PAy8BNrVWIOCP6I=

spinifex/bootstrap/config/config.go

@@ -18,13 +18,29 @@ import (
 // ClusterConfig is the root of spinifex.toml as returned by LoadConfig: the
 // cluster-wide settings plus a [nodes.<name>] Config for every node.
 type ClusterConfig struct {
-	Epoch     uint64            `mapstructure:"epoch"`     // bump when leader commits changes
-	Node      string            `mapstructure:"node"`      // my node name
-	Version   string            `mapstructure:"version"`   // spinifex version
-	Network   NetworkConfig     `mapstructure:"network"`   // cluster-wide external network settings
-	Bootstrap BootstrapConfig   `mapstructure:"bootstrap"` // default VPC IDs for OVN reconciliation
-	AWS       AWSConfig         `mapstructure:"aws"`       // cluster-wide AWS-parity settings (region, endpoint suffix)
-	Nodes     map[string]Config `mapstructure:"nodes"`     // full config for every node
+	Epoch     uint64                 `mapstructure:"epoch"`     // bump when leader commits changes
+	Node      string                 `mapstructure:"node"`      // my node name
+	Version   string                 `mapstructure:"version"`   // spinifex version
+	Network   NetworkConfig          `mapstructure:"network"`   // cluster-wide external network settings
+	Bootstrap BootstrapConfig        `mapstructure:"bootstrap"` // default VPC IDs for OVN reconciliation
+	AWS       AWSConfig              `mapstructure:"aws"`       // cluster-wide AWS-parity settings (region, endpoint suffix)
+	Northstar NorthstarClusterConfig `mapstructure:"northstar"` // cluster-wide authoritative DNS settings
+	Nodes     map[string]Config      `mapstructure:"nodes"`     // full config for every node
+}
+
+// MaxNameservers caps the nsN names a cluster publishes for its zones. Every
+// northstar node still serves every zone; the cap bounds only the delegation a
+// registrar has to carry.
+const MaxNameservers = 4
+
+// NorthstarClusterConfig is the cluster-wide [northstar] block.
+type NorthstarClusterConfig struct {
+	// NameserverCount is how many nsN names to publish, 1 to MaxNameservers.
+	// Zero means one per northstar node, up to MaxNameservers.
+	NameserverCount int `mapstructure:"nameserver_count"`
+	// NameserverNodes pins ns1, ns2, ... to these nodes, in order. Unpinned
+	// slots are filled from the remaining northstar nodes in name order.
+	NameserverNodes []string `mapstructure:"nameserver_nodes"`
 }
 
 // Defaults for cluster-wide AWS-parity settings.
@@ -654,6 +670,32 @@ func LoadConfig(configPath string) (*ClusterConfig, error) {
 	return &config, nil
 }
 
+// validateNorthstarCluster checks the [northstar] nameserver settings against
+// the cap and the configured nodes.
+func validateNorthstarCluster(cc *ClusterConfig) error {
+	ns := cc.Northstar
+	if ns.NameserverCount < 0 || ns.NameserverCount > MaxNameservers {
+		return fmt.Errorf("config: [northstar] nameserver_count=%d out of range; use 1-%d, or remove the key for one per northstar node up to %d", ns.NameserverCount, MaxNameservers, MaxNameservers)
+	}
+	if len(ns.NameserverNodes) > MaxNameservers {
+		return fmt.Errorf("config: [northstar] nameserver_nodes lists %d nodes; at most %d are published", len(ns.NameserverNodes), MaxNameservers)
+	}
+	if ns.NameserverCount > 0 && len(ns.NameserverNodes) > ns.NameserverCount {
+		return fmt.Errorf("config: [northstar] nameserver_nodes lists %d nodes but nameserver_count is %d", len(ns.NameserverNodes), ns.NameserverCount)
+	}
+	seen := make(map[string]bool, len(ns.NameserverNodes))
+	for _, name := range ns.NameserverNodes {
+		if _, ok := cc.Nodes[name]; !ok {
+			return fmt.Errorf("config: [northstar] nameserver_nodes names %q, which is not a configured node", name)
+		}
+		if seen[name] {
+			return fmt.Errorf("config: [northstar] nameserver_nodes names %q twice", name)
+		}
+		seen[name] = true
+	}
+	return nil
+}
+
 // validateClusterConfig rejects legacy DHCP config keys and validates external pool ranges.
 func validateClusterConfig(cc *ClusterConfig) error {
 	if viper.IsSet("network.external_dhcp") {
@@ -684,6 +726,10 @@ func validateClusterConfig(cc *ClusterConfig) error {
 		}
 	}
 
+	if err := validateNorthstarCluster(cc); err != nil {
+		return err
+	}
+
 	if len(cc.Network.NATExemptCIDRs) > 0 && cc.Network.ExternalMode != "nat" {
 		return fmt.Errorf("config: [network] nat_exempt_cidrs requires external_mode = \"nat\"")
 	}

spinifex/bootstrap/config/config_test.go

@@ -1426,3 +1426,43 @@ exoscale_instance_id = "0f1e2d3c-0000-4000-8000-000000000001"
 		})
 	}
 }
+
+func TestLoadConfig_NorthstarNameservers(t *testing.T) {
+	load := func(t *testing.T, block string) (*ClusterConfig, error) {
+		t.Helper()
+		resetViper(t)
+		path := filepath.Join(t.TempDir(), "spinifex.toml")
+		body := "node = \"n1\"\n\n" + block + "\n[nodes.n1]\nregion = \"us-east-1\"\n\n[nodes.n2]\nregion = \"us-east-1\"\n"
+		require.NoError(t, os.WriteFile(path, []byte(body), 0600))
+		return LoadConfig(path)
+	}
+
+	t.Run("absent block means automatic", func(t *testing.T) {
+		cfg, err := load(t, "")
+		require.NoError(t, err)
+		assert.Zero(t, cfg.Northstar.NameserverCount)
+		assert.Empty(t, cfg.Northstar.NameserverNodes)
+	})
+
+	t.Run("count and pin load", func(t *testing.T) {
+		cfg, err := load(t, "[northstar]\nnameserver_count = 2\nnameserver_nodes = [\"n2\", \"n1\"]\n")
+		require.NoError(t, err)
+		assert.Equal(t, 2, cfg.Northstar.NameserverCount)
+		assert.Equal(t, []string{"n2", "n1"}, cfg.Northstar.NameserverNodes)
+	})
+
+	for name, tc := range map[string]struct{ block, want string }{
+		"count above the cap":       {"[northstar]\nnameserver_count = 5\n", "nameserver_count=5"},
+		"negative count":            {"[northstar]\nnameserver_count = -1\n", "nameserver_count=-1"},
+		"pin longer than count":     {"[northstar]\nnameserver_count = 1\nnameserver_nodes = [\"n1\", \"n2\"]\n", "nameserver_count is 1"},
+		"pin names an unknown node": {"[northstar]\nnameserver_nodes = [\"n9\"]\n", `"n9"`},
+		"pin repeats a node":        {"[northstar]\nnameserver_nodes = [\"n1\", \"n1\"]\n", "twice"},
+		"pin above the cap":         {"[northstar]\nnameserver_nodes = [\"n1\", \"n2\", \"n3\", \"n4\", \"n5\"]\n", "at most 4"},
+	} {
+		t.Run(name, func(t *testing.T) {
+			_, err := load(t, tc.block)
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), tc.want)
+		})
+	}
+}

spinifex/daemon/daemon.go

@@ -71,6 +71,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/domains/ochre"
 	ochrevector "github.com/mulgadc/spinifex/spinifex/domains/ochre/vector"
 	rdsengine "github.com/mulgadc/spinifex/spinifex/domains/rds/engine"
+	route53domain "github.com/mulgadc/spinifex/spinifex/domains/route53"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
@@ -175,6 +176,8 @@ type Daemon struct {
 	bedrockReaper         *ochre.Reaper
 	acmService            *acmdomain.ACMServiceImpl
 	acmRenewalWorker      *acmdomain.Worker
+	route53Service        *route53domain.Service
+	route53Verifier       *route53domain.Verifier
 	ochreVectorService    ochrevector.VectorService
 	ochreAppliance        *ochrevector.Appliance
 	ochreBackupService    *ochrevector.BackupService
@@ -1232,6 +1235,31 @@ func (d *Daemon) subscribeAll() error {
 		)
 	}
 
+	// Route 53 is global: any daemon serves any account's zones from shared KV.
+	if d.route53Service != nil {
+		subs = append(subs,
+			natsSub{"route53.CreateHostedZone", handleNATSRequest(d.node, d.route53Service.CreateHostedZone), "spinifex-workers"},
+			natsSub{"route53.GetHostedZone", handleNATSRequest(d.node, d.route53Service.GetHostedZone), "spinifex-workers"},
+			natsSub{"route53.GetHostedZoneCount", handleNATSRequest(d.node, d.route53Service.GetHostedZoneCount), "spinifex-workers"},
+			natsSub{"route53.ListHostedZones", handleNATSRequest(d.node, d.route53Service.ListHostedZones), "spinifex-workers"},
+			natsSub{"route53.ListHostedZonesByName", handleNATSRequest(d.node, d.route53Service.ListHostedZonesByName), "spinifex-workers"},
+			natsSub{"route53.UpdateHostedZoneComment", handleNATSRequest(d.node, d.route53Service.UpdateHostedZoneComment), "spinifex-workers"},
+			natsSub{"route53.DeleteHostedZone", handleNATSRequest(d.node, d.route53Service.DeleteHostedZone), "spinifex-workers"},
+			natsSub{"route53.ChangeResourceRecordSets", handleNATSRequest(d.node, d.route53Service.ChangeResourceRecordSets), "spinifex-workers"},
+			natsSub{"route53.ListResourceRecordSets", handleNATSRequest(d.node, d.route53Service.ListResourceRecordSets), "spinifex-workers"},
+			natsSub{"route53.GetChange", handleNATSRequest(d.node, d.route53Service.GetChange), "spinifex-workers"},
+			natsSub{"route53.ChangeTagsForResource", handleNATSRequest(d.node, d.route53Service.ChangeTagsForResource), "spinifex-workers"},
+			natsSub{"route53.ListTagsForResource", handleNATSRequest(d.node, d.route53Service.ListTagsForResource), "spinifex-workers"},
+			natsSub{"route53.ListTagsForResources", handleNATSRequest(d.node, d.route53Service.ListTagsForResources), "spinifex-workers"},
+			natsSub{"route53.GetHostedZoneActivation", handleNATSRequest(d.node, d.route53Service.GetHostedZoneActivation), "spinifex-workers"},
+			natsSub{"route53.VerifyHostedZoneDelegation", handleNATSRequest(d.node, d.route53Service.VerifyHostedZoneDelegation), "spinifex-workers"},
+			natsSub{"route53.admin.FindZones", handleNATSRequest(d.node, d.route53Service.AdminFindZones), "spinifex-workers"},
+			natsSub{"route53.admin.Verify", handleNATSRequest(d.node, d.route53Service.AdminVerify), "spinifex-workers"},
+			natsSub{"route53.admin.Override", handleNATSRequest(d.node, d.route53Service.AdminOverride), "spinifex-workers"},
+			natsSub{"route53.admin.Delegate", handleNATSRequest(d.node, d.route53Delegate), "spinifex-workers"},
+		)
+	}
+
 	// Ochre vector store tenant surface is deliberately NOT registered here:
 	// its subjects only exist once the platform appliance has finished
 	// launching (startOchreVector, run in the background after this method
@@ -2173,6 +2201,14 @@ func (d *Daemon) startCluster() error {
 		return dns.HostsZone(d.config, domain)
 	}
 
+	route53JS, err := jetstream.New(d.natsConn)
+	if err != nil {
+		return fmt.Errorf("failed to initialize Route 53 service: %w", err)
+	}
+	d.route53Service = d.newRoute53Service(route53JS)
+	d.route53Verifier = d.newRoute53Verifier()
+	d.route53Service.Verifier = d.route53Verifier
+
 	// The tenant private CA is optional, unlike the master key above: a
 	// deployment issuing only public (PROVIDER_API/MANUAL_TXT) certificates has
 	// no reason to have one, so its absence must not block daemon startup.
@@ -2315,6 +2351,10 @@ func (d *Daemon) startCluster() error {
 		go d.dnsReconciler.Run(d.ctx)
 		slog.Info("Started DNS reconcile backstop", "interval_ms", otelsetup.Millis(dns.DefaultReconcileInterval))
 	}
+	if d.route53Service != nil {
+		d.startRoute53Realizer()
+		go d.route53Verifier.Run(d.ctx)
+	}
 
 	// Initialize per-instance-type NATS subscriptions for capacity-aware routing.
 	d.resourceMgr.initSubscriptions(d.natsConn, d.handleEC2RunInstances, d.handleSystemLaunchInstance, d.node)

spinifex/daemon/route53.go

@@ -0,0 +1,92 @@
+package daemon
+
+import (
+	"context"
+	"log/slog"
+
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/mulgadc/spinifex/spinifex/domains/dns"
+	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
+	route53domain "github.com/mulgadc/spinifex/spinifex/domains/route53"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+var _ route53domain.Publisher = (*dns.ZonePublisher)(nil)
+
+// newRoute53Service builds the Route 53 control plane. The nameservers and the
+// reserved names are read from the cluster's DNS settings on every call, so a
+// node joining or a domain change needs no restart.
+func (d *Daemon) newRoute53Service(js jetstream.JetStream) *route53domain.Service {
+	svc := route53domain.NewService(route53domain.NewStore(js))
+	svc.Nameservers = func() []string {
+		base := dns.ResolveBaseDomain(d.config)
+		if base == "" || d.clusterConfig == nil {
+			return nil
+		}
+		published := dns.PublishedNameservers(d.clusterConfig)
+		hosts := make([]string, 0, len(published))
+		for _, ns := range published {
+			hosts = append(hosts, ns.Host+"."+base)
+		}
+		return hosts
+	}
+	svc.Reserved = func(name string) bool {
+		return dns.ReservedZoneName(name, dns.ResolveBaseDomain(d.config), dns.ResolveInternalDomain(d.config))
+	}
+	return svc
+}
+
+// startRoute53Realizer publishes ACTIVE hosted zones to northstar. Every node
+// runs it, and a per-pass election picks the one that writes. Without a zone
+// store the zones stay unpublished and their changes stay PENDING.
+func (d *Daemon) startRoute53Realizer() {
+	pub, ok := dns.NewZonePublisher(d.config, func() *config.ClusterConfig { return d.clusterConfig }, d.natsConn)
+	if !ok {
+		slog.Info("Route 53 realizer not started: no northstar zone store is configured")
+		return
+	}
+	r := route53domain.NewRealizer(d.route53Service.Store(), pub)
+	r.Elect = func(ctx context.Context) (func(), bool) {
+		return reconcilelock.AcquireLeader(ctx, d.natsConn, route53domain.KVBucketRoute53Realize, d.node)
+	}
+	r.Reserved = d.route53Service.Reserved
+	go r.Run(d.ctx)
+	slog.Info("Started Route 53 realizer")
+}
+
+// newRoute53Verifier checks each zone's delegation: from this cluster's own
+// records when the parent is a zone it serves, and from the parent's public
+// servers otherwise.
+func (d *Daemon) newRoute53Verifier() *route53domain.Verifier {
+	v := route53domain.NewVerifier(d.route53Service.Store())
+	v.Nameservers = d.route53Service.Nameservers
+	if d.dnsWriter.Enabled() {
+		v.Platform = func(_ context.Context, name string) (route53domain.Delegation, bool, error) {
+			servers, zone, ok, err := d.dnsWriter.PlatformDelegation(name)
+			return route53domain.Delegation{Source: "platform zone " + zone, Servers: servers}, ok, err
+		}
+	}
+	lookup := &dns.ParentLookup{}
+	v.Public = func(ctx context.Context, name string) (route53domain.Delegation, error) {
+		servers, parent, err := lookup.Delegation(ctx, name)
+		return route53domain.Delegation{Source: "the " + parent + " name servers", Servers: servers}, err
+	}
+	v.Elect = func(ctx context.Context) (func(), bool) {
+		return reconcilelock.AcquireLeader(ctx, d.natsConn, route53domain.KVBucketRoute53Verify, d.node)
+	}
+	return v
+}
+
+// route53Delegate writes or withdraws platform-zone delegations for the
+// operator's spx admin dns delegate.
+func (d *Daemon) route53Delegate(ctx context.Context, in *route53domain.AdminDelegateInput, _ string) (*route53domain.AdminDelegateOutput, error) {
+	if len(in.Names) == 0 {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "no names to delegate")
+	}
+	res, err := d.dnsWriter.Delegate(ctx, d.natsConn, d.node, in.Names, in.Remove)
+	if err != nil {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "%v", err)
+	}
+	return &route53domain.AdminDelegateOutput{Applied: res.Applied}, nil
+}

spinifex/daemon/route53_test.go

@@ -0,0 +1,50 @@
+//test:in-package — newRoute53Service is an unexported Daemon method and reads
+//the unexported config and clusterConfig fields.
+
+package daemon
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/route53"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRoute53ServiceDelegatesToThePublishedNameservers(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	ns := config.NorthstarConfig{ConfigPath: "/etc/spinifex/northstar.toml", DefaultDomain: "spx3.net"}
+	nodes := map[string]config.Config{
+		"node1": {AdvertiseIP: "10.0.0.1", Northstar: ns},
+		"node2": {AdvertiseIP: "10.0.0.2", Northstar: ns},
+	}
+	local := nodes["node1"]
+	d := &Daemon{config: &local, clusterConfig: &config.ClusterConfig{Node: "node1", Nodes: nodes}}
+	svc := d.newRoute53Service(js)
+
+	out, err := svc.CreateHostedZone(context.Background(), &route53.CreateHostedZoneInput{
+		Name: aws.String("example.com"), CallerReference: aws.String("ref-1"),
+	}, "000000000001")
+	require.NoError(t, err)
+	assert.Equal(t, []string{"ns1.spx3.net", "ns2.spx3.net"}, aws.StringValueSlice(out.DelegationSet.NameServers))
+
+	_, err = svc.CreateHostedZone(context.Background(), &route53.CreateHostedZoneInput{
+		Name: aws.String("lb.elb.spx3.net"), CallerReference: aws.String("ref-2"),
+	}, "000000000001")
+	assert.ErrorContains(t, err, "InvalidDomainName")
+}
+
+func TestRoute53ServiceRefusesZonesWithoutABaseDomain(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	local := config.Config{AdvertiseIP: "10.0.0.1"}
+	d := &Daemon{config: &local, clusterConfig: &config.ClusterConfig{Node: "node1", Nodes: map[string]config.Config{"node1": local}}}
+
+	_, err := d.newRoute53Service(js).CreateHostedZone(context.Background(), &route53.CreateHostedZoneInput{
+		Name: aws.String("example.com"), CallerReference: aws.String("ref-1"),
+	}, "000000000001")
+	assert.ErrorContains(t, err, "no nameservers")
+}

spinifex/domains/dns/delegation.go

@@ -0,0 +1,104 @@
+package dns
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	nsconfig "github.com/mulgadc/northstar/pkg/config"
+	"github.com/nats-io/nats.go"
+
+	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
+)
+
+// delegateLockPoll is how often Delegate retries the zone-writer election
+// while another node holds it.
+var delegateLockPoll = time.Second
+
+// PlatformDelegation returns the NS set the base zone publishes for name and
+// the zone it read. ok is false when name is not strictly below the base
+// domain, so the zone's parent is somewhere else.
+func (w *Writer) PlatformDelegation(name string) (servers []string, zone string, ok bool, err error) {
+	if !w.Enabled() {
+		return nil, "", false, nil
+	}
+	base := normalizeZone(w.baseDomain)
+	n := normalizeZone(name)
+	if base == "" || !strings.HasSuffix(n, "."+base) {
+		return nil, "", false, nil
+	}
+	cfg, exists, err := nsconfig.ReadZoneRaw(w.s3cfg, base)
+	if err != nil {
+		return nil, base, true, fmt.Errorf("read zone %s: %w", base, err)
+	}
+	if !exists {
+		return nil, base, true, nil
+	}
+	label := relativeLabel(n, base)
+	for _, r := range cfg.Records {
+		if r.Type == nsconfig.TypeNS && strings.EqualFold(r.Domain, label) {
+			servers = append(servers, r.Address)
+		}
+	}
+	return servers, base, true, nil
+}
+
+// Delegate writes, or with remove withdraws, an NS set in the base zone that
+// delegates each name to this cluster's nameservers. It takes the zone-writer
+// election first, so it never races the reconcile's own writes to the zone.
+func (w *Writer) Delegate(ctx context.Context, nc *nats.Conn, holder string, names []string, remove bool) (*ChangeResult, error) {
+	if !w.Enabled() {
+		return nil, errors.New("northstar is not configured on this node")
+	}
+	base := normalizeZone(w.baseDomain)
+	values := make([]string, 0, len(w.nameservers))
+	for _, ns := range w.nameservers {
+		values = append(values, ns.Host+"."+base+".")
+	}
+	batch := &ChangeBatch{}
+	for _, name := range names {
+		n := normalizeZone(name)
+		if !strings.HasSuffix(n, "."+base) {
+			return nil, fmt.Errorf("%s is not below the base domain %s", name, base)
+		}
+		if ReservedZoneName(n, base, w.internalDomain) {
+			return nil, fmt.Errorf("%s is a platform name and cannot be delegated", name)
+		}
+		if remove {
+			for _, v := range values {
+				batch.Changes = append(batch.Changes, Change{Action: ActionDelete, Zone: base, Name: n + ".", Type: "NS", Value: v})
+			}
+			continue
+		}
+		batch.Changes = append(batch.Changes, Change{Action: ActionUpsertSet, Zone: base, Name: n + ".", Type: "NS", Values: values})
+	}
+	if len(batch.Changes) == 0 {
+		return &ChangeResult{}, nil
+	}
+	release, err := w.awaitWriterElection(ctx, nc, holder)
+	if err != nil {
+		return nil, err
+	}
+	defer release()
+	return w.ApplyBatch(batch)
+}
+
+func (w *Writer) awaitWriterElection(ctx context.Context, nc *nats.Conn, holder string) (func(), error) {
+	if nc == nil {
+		return func() {}, nil
+	}
+	tick := time.NewTicker(delegateLockPoll)
+	defer tick.Stop()
+	for {
+		if release, ok := reconcilelock.AcquireLeader(ctx, nc, KVBucketDNSReconcile, holder); ok {
+			return release, nil
+		}
+		select {
+		case <-ctx.Done():
+			return nil, fmt.Errorf("another node is writing the zone: %w", ctx.Err())
+		case <-tick.C:
+		}
+	}
+}

spinifex/domains/dns/delegation_test.go

@@ -0,0 +1,82 @@
+package dns
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
+)
+
+func TestDelegateWritesAndWithdrawsPlatformDelegations(t *testing.T) {
+	w, _ := newMultiNodeTestWriter(t)
+	_, err := w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net", "B.r53e2e.spx3.net."}, false)
+	require.NoError(t, err)
+
+	for _, name := range []string{"a.r53e2e.spx3.net.", "b.r53e2e.spx3.net"} {
+		servers, zone, ok, err := w.PlatformDelegation(name)
+		require.NoError(t, err)
+		assert.True(t, ok)
+		assert.Equal(t, "spx3.net", zone)
+		assert.ElementsMatch(t, []string{"ns1.spx3.net.", "ns2.spx3.net."}, servers, name)
+	}
+	servers, _, ok, err := w.PlatformDelegation("c.r53e2e.spx3.net")
+	require.NoError(t, err)
+	assert.True(t, ok, "a name below the base domain is decided by the base zone")
+	assert.Empty(t, servers)
+
+	_, err = w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net"}, true)
+	require.NoError(t, err)
+	servers, _, _, err = w.PlatformDelegation("a.r53e2e.spx3.net")
+	require.NoError(t, err)
+	assert.Empty(t, servers)
+	servers, _, _, err = w.PlatformDelegation("b.r53e2e.spx3.net")
+	require.NoError(t, err)
+	assert.Len(t, servers, 2, "withdrawing one delegation leaves the others")
+}
+
+func TestPlatformDelegationIgnoresNamesOutsideTheBaseDomain(t *testing.T) {
+	w, _ := newMultiNodeTestWriter(t)
+	for _, name := range []string{"example.com", "spx3.net", "notspx3.net"} {
+		_, _, ok, err := w.PlatformDelegation(name)
+		require.NoError(t, err)
+		assert.False(t, ok, name)
+	}
+}
+
+func TestDelegateRefusesPlatformAndForeignNames(t *testing.T) {
+	w, _ := newMultiNodeTestWriter(t)
+	for _, name := range []string{"ns1.spx3.net", "x.elb.spx3.net", "compute.spx3.net", "example.com", "spx3.net"} {
+		_, err := w.Delegate(t.Context(), nil, "node1", []string{"ok.spx3.net", name}, false)
+		require.Error(t, err, name)
+	}
+	servers, _, _, err := w.PlatformDelegation("ok.spx3.net")
+	require.NoError(t, err)
+	assert.Empty(t, servers, "a refused batch writes nothing")
+}
+
+func TestDelegateWaitsForTheZoneWriterElection(t *testing.T) {
+	w, _ := newMultiNodeTestWriter(t)
+	_, nc, _ := testutil.StartTestJetStream(t)
+	prev := delegateLockPoll
+	delegateLockPoll = 10 * time.Millisecond
+	t.Cleanup(func() { delegateLockPoll = prev })
+
+	release, ok := reconcilelock.AcquireLeader(t.Context(), nc, KVBucketDNSReconcile, "node2")
+	require.True(t, ok)
+	ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
+	defer cancel()
+	_, err := w.Delegate(ctx, nc, "node1", []string{"a.spx3.net"}, false)
+	require.ErrorContains(t, err, "another node is writing the zone")
+
+	release()
+	_, err = w.Delegate(t.Context(), nc, "node1", []string{"a.spx3.net"}, false)
+	require.NoError(t, err)
+	servers, _, _, err := w.PlatformDelegation("a.spx3.net")
+	require.NoError(t, err)
+	assert.Len(t, servers, 2)
+}

spinifex/domains/dns/nameservers.go

@@ -10,17 +10,18 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 )
 
-// NameserverSeeds derives one nameserver (nsN → node IP) per cluster node that
-// runs northstar and is reachable by its peers, ordered deterministically so
-// every node computes the same set. Falls back to the local node when none
-// qualifies (single-node / dev).
+// NameserverSeeds derives the published nameservers (nsN → node IP) from the
+// cluster nodes that run northstar and are reachable by their peers: the
+// [northstar] pins first, then the rest in name order, capped at
+// nameserver_count or MaxNameservers. Every node computes the same set. Falls
+// back to the local node when none qualifies (single-node / dev).
 //
 // Both zone creators build from this: the northstar bootstrap seed and the
 // writer's on-demand materialisation. Both are create-if-absent and neither
 // overwrites, so they must agree — a set derived independently would be baked
 // into the zone permanently by whichever creator ran first.
 func NameserverSeeds(cluster *config.ClusterConfig) []nsconfig.NameserverSeed {
-	names := usableNorthstarNodeNames(cluster)
+	names := publishedNameserverNodes(usableNorthstarNodeNames(cluster), cluster.Northstar)
 	// A zone needs an SOA and at least one NS whatever the topology, so unlike the
 	// resolver list this falls back to the local node. On single-node dev that node
 	// is loopback, which is correct there: it is the zone's only reader.
@@ -29,17 +30,37 @@ func NameserverSeeds(cluster *config.ClusterConfig) []nsconfig.NameserverSeed {
 	}
 
 	seeds := make([]nsconfig.NameserverSeed, 0, len(names))
-	for i, name := range names {
-		seeds = append(seeds, nsconfig.NameserverSeed{
-			Host: fmt.Sprintf("ns%d", i+1),
-			IP:   nameserverIP(cluster.Nodes[name]),
-		})
+	for _, ns := range nameserverAssignments(cluster, names) {
+		seeds = append(seeds, nsconfig.NameserverSeed{Host: ns.Host, IP: ns.IP})
 	}
 	return seeds
 }
 
-// ResolverNameserverIPs returns the WAN IPs of cluster nodes running northstar,
-// in the same deterministic order as the seeded nameservers. vpcd's per-tap DNS
+// Nameserver is one published nameserver: its label under the base domain,
+// the node it is assigned to and that node's address.
+type Nameserver struct {
+	Host string
+	Node string
+	IP   string
+}
+
+// PublishedNameservers returns the nsN assignment NameserverSeeds publishes,
+// with the node behind each name. Unlike NameserverSeeds it has no local-node
+// fallback, so an empty result means no node qualifies.
+func PublishedNameservers(cluster *config.ClusterConfig) []Nameserver {
+	return nameserverAssignments(cluster, publishedNameserverNodes(usableNorthstarNodeNames(cluster), cluster.Northstar))
+}
+
+func nameserverAssignments(cluster *config.ClusterConfig, names []string) []Nameserver {
+	out := make([]Nameserver, 0, len(names))
+	for i, name := range names {
+		out = append(out, Nameserver{Host: fmt.Sprintf("ns%d", i+1), Node: name, IP: nameserverIP(cluster.Nodes[name])})
+	}
+	return out
+}
+
+// ResolverNameserverIPs returns the WAN IPs of every cluster node running
+// northstar in name order, not only the published nameservers. vpcd's per-tap DNS
 // shim uses these as forward targets (northstar's :5300 listener), so internal
 // names resolve authoritatively and external names via upstream forwarders.
 //
@@ -55,6 +76,36 @@ func ResolverNameserverIPs(cluster *config.ClusterConfig) []string {
 	return ips
 }
 
+// publishedNameserverNodes orders the usable northstar nodes for publication
+// and truncates them to the configured count. A pinned node that is not usable
+// is skipped rather than published with an address its peers cannot reach.
+func publishedNameserverNodes(usable []string, ns config.NorthstarClusterConfig) []string {
+	limit := ns.NameserverCount
+	if limit <= 0 || limit > config.MaxNameservers {
+		limit = config.MaxNameservers
+	}
+	ok := make(map[string]bool, len(usable))
+	for _, name := range usable {
+		ok[name] = true
+	}
+	ordered := make([]string, 0, len(usable))
+	for _, name := range ns.NameserverNodes {
+		if ok[name] {
+			ordered = append(ordered, name)
+			delete(ok, name)
+		}
+	}
+	for _, name := range usable {
+		if ok[name] {
+			ordered = append(ordered, name)
+		}
+	}
+	if len(ordered) > limit {
+		ordered = ordered[:limit]
+	}
+	return ordered
+}
+
 // usableNorthstarNodeNames returns the deterministically ordered nodes that run
 // northstar and can be reached by their peers.
 //

spinifex/domains/dns/nameservers_test.go

@@ -1,8 +1,10 @@
 package dns
 
 import (
+	"fmt"
 	"testing"
 
+	nsconfig "github.com/mulgadc/northstar/pkg/config"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -132,3 +134,62 @@ func TestResolverNameserverIPs(t *testing.T) {
 	}}
 	assert.Empty(t, ResolverNameserverIPs(dev))
 }
+
+func northstarNodes(n int) map[string]config.Config {
+	nodes := make(map[string]config.Config, n)
+	for i := 1; i <= n; i++ {
+		nodes[fmt.Sprintf("node%02d", i)] = config.Config{
+			AdvertiseIP: fmt.Sprintf("10.0.0.%d", i),
+			Northstar:   config.NorthstarConfig{ConfigPath: "/etc/n.toml"},
+		}
+	}
+	return nodes
+}
+
+func seedIPs(seeds []nsconfig.NameserverSeed) []string {
+	ips := make([]string, 0, len(seeds))
+	for i, s := range seeds {
+		if s.Host != fmt.Sprintf("ns%d", i+1) {
+			return []string{"misnumbered " + s.Host}
+		}
+		ips = append(ips, s.IP)
+	}
+	return ips
+}
+
+// A large cluster publishes at most four nameservers, while every northstar
+// node stays a resolver backend for the guest shim.
+func TestNameserverSeedsCappedAtFour(t *testing.T) {
+	cluster := &config.ClusterConfig{Node: "node01", Nodes: northstarNodes(12)}
+	assert.Equal(t, []string{"10.0.0.1", "10.0.0.2", "10.0.0.3", "10.0.0.4"}, seedIPs(NameserverSeeds(cluster)))
+	assert.Len(t, ResolverNameserverIPs(cluster), 12)
+}
+
+func TestNameserverSeedsHonourCountAndPins(t *testing.T) {
+	nodes := northstarNodes(6)
+	nodes["node06"] = config.Config{AdvertiseIP: "127.0.0.1", Northstar: config.NorthstarConfig{ConfigPath: "/etc/n.toml"}}
+
+	for name, tc := range map[string]struct {
+		ns   config.NorthstarClusterConfig
+		want []string
+	}{
+		"count below the node count": {config.NorthstarClusterConfig{NameserverCount: 2}, []string{"10.0.0.1", "10.0.0.2"}},
+		"pins first, rest in name order": {
+			config.NorthstarClusterConfig{NameserverNodes: []string{"node05", "node03"}},
+			[]string{"10.0.0.5", "10.0.0.3", "10.0.0.1", "10.0.0.2"},
+		},
+		"pins fill the count": {
+			config.NorthstarClusterConfig{NameserverCount: 2, NameserverNodes: []string{"node04", "node02"}},
+			[]string{"10.0.0.4", "10.0.0.2"},
+		},
+		"unusable pin is skipped": {
+			config.NorthstarClusterConfig{NameserverCount: 2, NameserverNodes: []string{"node06", "node05"}},
+			[]string{"10.0.0.5", "10.0.0.1"},
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			cluster := &config.ClusterConfig{Node: "node01", Nodes: nodes, Northstar: tc.ns}
+			assert.Equal(t, tc.want, seedIPs(NameserverSeeds(cluster)))
+		})
+	}
+}

spinifex/domains/dns/parent.go

@@ -0,0 +1,179 @@
+package dns
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"slices"
+	"strings"
+	"time"
+
+	mdns "github.com/miekg/dns"
+)
+
+// RootServers are the IANA root servers, by IPv4 address.
+var RootServers = []string{
+	"198.41.0.4", "170.247.170.2", "192.33.4.12", "199.7.91.13", "192.203.230.10",
+	"192.5.5.241", "192.112.36.4", "198.97.190.53", "192.36.148.17", "192.58.128.30",
+	"193.0.14.129", "199.7.83.42", "202.12.27.33",
+}
+
+const (
+	parentQueryTimeout = 3 * time.Second
+	maxReferrals       = 16
+)
+
+// ParentLookup reads a domain's delegation from its parent zone's own servers,
+// found by following referrals down from the roots. It never asks a recursive
+// resolver, whose cache can hold a delegation the parent has since withdrawn.
+type ParentLookup struct {
+	// Roots are the servers the walk starts from; nil means RootServers.
+	Roots []string
+	// Exchange sends one query; nil means UDP to port 53, retried over TCP
+	// when truncated.
+	Exchange func(ctx context.Context, m *mdns.Msg, server string) (*mdns.Msg, error)
+	// Resolve finds a name server's addresses when a referral has no glue.
+	Resolve func(ctx context.Context, host string) ([]string, error)
+}
+
+// Delegation returns the NS names the parent of name publishes for it and the
+// parent zone that answered. An empty set with a nil error means the parent
+// answered and does not delegate name.
+func (p *ParentLookup) Delegation(ctx context.Context, name string) (servers []string, parent string, err error) {
+	target := mdns.Fqdn(strings.ToLower(name))
+	addrs := p.Roots
+	if addrs == nil {
+		addrs = RootServers
+	}
+	zone := "."
+	for range maxReferrals {
+		resp, err := p.ask(ctx, target, addrs)
+		if err != nil {
+			return nil, zone, fmt.Errorf("ask the %s servers: %w", zone, err)
+		}
+		owner, ns := referral(resp, target)
+		if owner == "" || resp.Authoritative {
+			// The servers hold target's own zone or its parent: what they list
+			// for target is the answer, and nothing listed is no delegation.
+			return answerNS(resp, target), strings.TrimSuffix(zone, "."), nil
+		}
+		if owner == target {
+			return ns, strings.TrimSuffix(zone, "."), nil
+		}
+		if !mdns.IsSubDomain(zone, owner) || owner == zone {
+			return nil, zone, fmt.Errorf("the %s servers referred %s back to %s", zone, target, owner)
+		}
+		zone = owner
+		if addrs, err = p.addresses(ctx, resp, ns); err != nil {
+			return nil, zone, err
+		}
+	}
+	return nil, zone, fmt.Errorf("more than %d referrals for %s", maxReferrals, target)
+}
+
+// ask sends the query to each server in turn and returns the first answer
+// that is NOERROR or NXDOMAIN.
+func (p *ParentLookup) ask(ctx context.Context, target string, servers []string) (*mdns.Msg, error) {
+	m := new(mdns.Msg)
+	m.SetQuestion(target, mdns.TypeNS)
+	m.RecursionDesired = false
+	exchange := p.Exchange
+	if exchange == nil {
+		exchange = exchangeUDP
+	}
+	lastErr := errors.New("no servers")
+	for _, s := range servers {
+		resp, err := exchange(ctx, m, s)
+		switch {
+		case err != nil:
+			lastErr = err
+		case resp.Rcode != mdns.RcodeSuccess && resp.Rcode != mdns.RcodeNameError:
+			lastErr = fmt.Errorf("%s answered %s", s, mdns.RcodeToString[resp.Rcode])
+		default:
+			return resp, nil
+		}
+		if ctx.Err() != nil {
+			return nil, ctx.Err()
+		}
+	}
+	return nil, lastErr
+}
+
+// referral returns the deepest zone at or above target that the response's
+// authority section delegates, and its NS names.
+func referral(resp *mdns.Msg, target string) (string, []string) {
+	var owner string
+	var ns []string
+	for _, rr := range resp.Ns {
+		r, ok := rr.(*mdns.NS)
+		if !ok {
+			continue
+		}
+		o := strings.ToLower(r.Hdr.Name)
+		if !mdns.IsSubDomain(o, target) {
+			continue
+		}
+		if o != owner && (owner == "" || mdns.CountLabel(o) > mdns.CountLabel(owner)) {
+			owner, ns = o, nil
+		}
+		if o == owner {
+			ns = append(ns, strings.ToLower(r.Ns))
+		}
+	}
+	return owner, ns
+}
+
+func answerNS(resp *mdns.Msg, target string) []string {
+	var ns []string
+	for _, rr := range append(resp.Answer, resp.Ns...) {
+		if r, ok := rr.(*mdns.NS); ok && strings.EqualFold(r.Hdr.Name, target) {
+			ns = append(ns, strings.ToLower(r.Ns))
+		}
+	}
+	return ns
+}
+
+// addresses returns where to send the next query: the referral's glue, or a
+// lookup of the server names when the parent gave none.
+func (p *ParentLookup) addresses(ctx context.Context, resp *mdns.Msg, ns []string) ([]string, error) {
+	var addrs []string
+	for _, rr := range resp.Extra {
+		if a, ok := rr.(*mdns.A); ok && slices.ContainsFunc(ns, func(n string) bool { return strings.EqualFold(n, a.Hdr.Name) }) {
+			addrs = append(addrs, a.A.String())
+		}
+	}
+	if len(addrs) > 0 {
+		return addrs, nil
+	}
+	resolve := p.Resolve
+	if resolve == nil {
+		resolve = net.DefaultResolver.LookupHost
+	}
+	var lastErr error
+	for _, host := range ns {
+		found, err := resolve(ctx, strings.TrimSuffix(host, "."))
+		if err != nil {
+			lastErr = err
+			continue
+		}
+		for _, a := range found {
+			if ip := net.ParseIP(a); ip != nil && ip.To4() != nil {
+				addrs = append(addrs, a)
+			}
+		}
+	}
+	if len(addrs) == 0 {
+		return nil, fmt.Errorf("no address for any of %s: %w", strings.Join(ns, ", "), lastErr)
+	}
+	return addrs, nil
+}
+
+func exchangeUDP(ctx context.Context, m *mdns.Msg, server string) (*mdns.Msg, error) {
+	addr := net.JoinHostPort(server, "53")
+	resp, _, err := (&mdns.Client{Timeout: parentQueryTimeout}).ExchangeContext(ctx, m, addr)
+	if err == nil && resp.Truncated {
+		resp, _, err = (&mdns.Client{Net: "tcp", Timeout: parentQueryTimeout}).ExchangeContext(ctx, m, addr)
+	}
+	return resp, err
+}

spinifex/domains/dns/parent_test.go

@@ -0,0 +1,139 @@
+package dns_test
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	mdns "github.com/miekg/dns"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/dns"
+)
+
+// fakeTree is the DNS hierarchy as each server's address answers it.
+type fakeTree struct {
+	servers map[string]func(q mdns.Question) *mdns.Msg
+	asked   []string
+}
+
+func (f *fakeTree) exchange(_ context.Context, m *mdns.Msg, server string) (*mdns.Msg, error) {
+	f.asked = append(f.asked, server)
+	if m.RecursionDesired {
+		return nil, errors.New("recursion requested")
+	}
+	h, ok := f.servers[server]
+	if !ok {
+		return nil, errors.New("i/o timeout")
+	}
+	resp := h(m.Question[0])
+	rcode, aa := resp.Rcode, resp.Authoritative
+	resp.SetReply(m)
+	resp.Rcode, resp.Authoritative = rcode, aa
+	return resp, nil
+}
+
+func ns(owner string, hosts ...string) []mdns.RR {
+	var out []mdns.RR
+	for _, h := range hosts {
+		out = append(out, &mdns.NS{Hdr: mdns.RR_Header{Name: owner, Rrtype: mdns.TypeNS, Class: mdns.ClassINET, Ttl: 172800}, Ns: h})
+	}
+	return out
+}
+
+func glue(host, ip string) mdns.RR {
+	rr, _ := mdns.NewRR(host + " 172800 IN A " + ip)
+	return rr
+}
+
+func referralTo(owner string, extra []mdns.RR, hosts ...string) *mdns.Msg {
+	return &mdns.Msg{Ns: ns(owner, hosts...), Extra: extra}
+}
+
+func authoritative(rcode int) *mdns.Msg {
+	m := &mdns.Msg{}
+	m.Authoritative = true
+	m.Rcode = rcode
+	return m
+}
+
+func newTree() *fakeTree {
+	return &fakeTree{servers: map[string]func(mdns.Question) *mdns.Msg{
+		"10.0.0.1": func(q mdns.Question) *mdns.Msg {
+			if mdns.IsSubDomain("org.", q.Name) {
+				return referralTo("org.", nil, "a0.org-servers.test.") // no glue
+			}
+			return referralTo("com.", []mdns.RR{glue("a.gtld.test.", "10.0.1.1")}, "a.gtld.test.")
+		},
+		"10.0.1.1": func(q mdns.Question) *mdns.Msg {
+			switch {
+			case mdns.IsSubDomain("example.com.", q.Name):
+				return referralTo("example.com.", nil, "NS1.spx3.net.", "ns2.spx3.net.")
+			case mdns.IsSubDomain("deep.com.", q.Name):
+				return referralTo("deep.com.", []mdns.RR{glue("ns.deep.com.", "10.0.2.1")}, "ns.deep.com.")
+			default:
+				return authoritative(mdns.RcodeNameError)
+			}
+		},
+		"10.0.2.1": func(q mdns.Question) *mdns.Msg {
+			if q.Name == "shop.deep.com." {
+				return referralTo("shop.deep.com.", nil, "ns1.spx3.net.")
+			}
+			return authoritative(mdns.RcodeSuccess) // NODATA: not delegated
+		},
+		"10.0.3.1": func(q mdns.Question) *mdns.Msg {
+			return referralTo(q.Name, nil, "ns3.spx3.net.")
+		},
+	}}
+}
+
+func TestParentLookupFollowsReferralsToTheParent(t *testing.T) {
+	tree := newTree()
+	p := &dns.ParentLookup{
+		Roots:    []string{"10.0.0.9", "10.0.0.1"}, // the first root is down
+		Exchange: tree.exchange,
+		Resolve: func(_ context.Context, host string) ([]string, error) {
+			if host == "a0.org-servers.test" {
+				return []string{"2001:db8::1", "10.0.3.1"}, nil
+			}
+			return nil, errors.New("no such host")
+		},
+	}
+	cases := []struct {
+		name    string
+		servers []string
+		parent  string
+	}{
+		{"Example.COM", []string{"ns1.spx3.net.", "ns2.spx3.net."}, "com"},
+		{"shop.deep.com.", []string{"ns1.spx3.net."}, "deep.com"},
+		{"other.deep.com", nil, "deep.com"},
+		{"unregistered.com", nil, "com"},
+		{"example.org", []string{"ns3.spx3.net."}, "org"},
+	}
+	for _, tc := range cases {
+		servers, parent, err := p.Delegation(context.Background(), tc.name)
+		require.NoError(t, err, tc.name)
+		assert.Equal(t, tc.servers, servers, tc.name)
+		assert.Equal(t, tc.parent, parent, tc.name)
+	}
+}
+
+func TestParentLookupFailsWhenNoServerAnswers(t *testing.T) {
+	tree := newTree()
+	tree.servers["10.0.1.1"] = func(mdns.Question) *mdns.Msg { return &mdns.Msg{MsgHdr: mdns.MsgHdr{Rcode: mdns.RcodeServerFailure}} }
+	p := &dns.ParentLookup{Roots: []string{"10.0.0.1"}, Exchange: tree.exchange}
+	_, parent, err := p.Delegation(context.Background(), "example.com")
+	require.ErrorContains(t, err, "SERVFAIL")
+	assert.Equal(t, "com.", parent)
+}
+
+func TestParentLookupRefusesAReferralUpwards(t *testing.T) {
+	tree := newTree()
+	tree.servers["10.0.1.1"] = func(mdns.Question) *mdns.Msg {
+		return referralTo(".", []mdns.RR{glue("a.root.test.", "10.0.0.1")}, "a.root.test.")
+	}
+	p := &dns.ParentLookup{Roots: []string{"10.0.0.1"}, Exchange: tree.exchange}
+	_, _, err := p.Delegation(context.Background(), "example.com")
+	require.Error(t, err)
+}

spinifex/domains/dns/publisher.go

@@ -0,0 +1,78 @@
+package dns
+
+import (
+	"context"
+	"time"
+
+	nsconfig "github.com/mulgadc/northstar/pkg/config"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/nats-io/nats.go"
+)
+
+// zoneReloadTimeout bounds one wait for every northstar server to load a zone.
+const zoneReloadTimeout = 5 * time.Second
+
+// ZonePublisher writes whole zone documents to the northstar zone store and
+// asks every northstar server to load them, for callers that own a zone
+// outright rather than editing records in one.
+type ZonePublisher struct {
+	s3      *nsconfig.S3Config
+	nc      *nats.Conn
+	cluster func() *config.ClusterConfig
+}
+
+// NewZonePublisher returns a publisher, or false when northstar's S3 zone store
+// is not configured on this node. cluster is read on every reload, so a node
+// joining changes how many acknowledgements are awaited.
+func NewZonePublisher(cfg *config.Config, cluster func() *config.ClusterConfig, nc *nats.Conn) (*ZonePublisher, bool) {
+	zoneCfg, ok := zoneS3Config(cfg)
+	if !ok || nc == nil {
+		return nil, false
+	}
+	return &ZonePublisher{s3: zoneCfg.s3, nc: nc, cluster: cluster}, true
+}
+
+// PutZone writes apex's zone document.
+func (p *ZonePublisher) PutZone(_ context.Context, apex string, body []byte) error {
+	return nsconfig.WriteZoneFile(p.s3, apex, body)
+}
+
+// DeleteZone removes apex's zone document.
+func (p *ZonePublisher) DeleteZone(_ context.Context, apex string) error {
+	return nsconfig.DeleteZoneFile(p.s3, apex)
+}
+
+// Reload asks every northstar server to load apex and returns the version each
+// reports, "" where it serves nothing. A server that failed to load reports a
+// value no digest can equal. complete is whether every server answered,
+// counted by node so a repeated acknowledgement cannot stand in for another.
+func (p *ZonePublisher) Reload(ctx context.Context, apex string) ([]string, bool, error) {
+	want := NorthstarServerCount(p.cluster())
+	ctx, cancel := context.WithTimeout(ctx, zoneReloadTimeout)
+	defer cancel()
+	acks, err := RequestZoneReload(ctx, p.nc, apex, want)
+	if err != nil {
+		return nil, false, err
+	}
+	served := make([]string, 0, len(acks))
+	nodes := map[string]bool{}
+	for _, ack := range acks {
+		nodes[ack.Node] = true
+		if ack.Error != "" {
+			served = append(served, "error: "+ack.Error)
+			continue
+		}
+		served = append(served, ack.ETag)
+	}
+	return served, len(nodes) >= want, nil
+}
+
+// NorthstarServerCount is how many northstar servers the cluster runs, and so
+// how many must acknowledge a reload. A cluster with none configured still runs
+// the local one.
+func NorthstarServerCount(cluster *config.ClusterConfig) int {
+	if cluster == nil {
+		return 1
+	}
+	return max(1, len(configuredNorthstarNodeNames(cluster)))
+}

spinifex/domains/dns/publisher_test.go

@@ -0,0 +1,135 @@
+package dns_test
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/mulgadc/spinifex/spinifex/domains/dns"
+)
+
+// zoneBucket is a path-style S3 endpoint holding whole objects.
+func zoneBucket(t *testing.T) (endpoint string, get func(key string) (string, bool)) {
+	t.Helper()
+	var mu sync.Mutex
+	objects := map[string]string{}
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		key := strings.TrimPrefix(r.URL.Path, "/northstar/")
+		mu.Lock()
+		defer mu.Unlock()
+		switch r.Method {
+		case http.MethodPut:
+			body, _ := io.ReadAll(r.Body)
+			objects[key] = string(body)
+		case http.MethodDelete:
+			delete(objects, key)
+			w.WriteHeader(http.StatusNoContent)
+		default:
+			http.Error(w, "unsupported", http.StatusMethodNotAllowed)
+		}
+	}))
+	t.Cleanup(srv.Close)
+	return srv.URL, func(key string) (string, bool) {
+		mu.Lock()
+		defer mu.Unlock()
+		v, ok := objects[key]
+		return v, ok
+	}
+}
+
+func publisherConfig(t *testing.T, endpoint string) *config.Config {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "northstar.toml")
+	body := fmt.Sprintf("listen = \"0.0.0.0:5300\"\ndefault_domain = \"spx3.net\"\n[s3]\nendpoint = %q\nbucket = \"northstar\"\nregion = \"us-east-1\"\naccess_key = \"RO\"\nsecret_key = \"RO\"\n", endpoint)
+	require.NoError(t, os.WriteFile(path, []byte(body), 0o600))
+	return &config.Config{
+		Predastore: config.PredastoreConfig{AccessKey: "SYSTEM", SecretKey: "SYSTEMSECRET"},
+		Northstar:  config.NorthstarConfig{ConfigPath: path},
+	}
+}
+
+func cluster(nodes ...string) *config.ClusterConfig {
+	c := &config.ClusterConfig{Node: nodes[0], Nodes: map[string]config.Config{}}
+	for _, n := range nodes {
+		c.Nodes[n] = config.Config{Northstar: config.NorthstarConfig{ConfigPath: "/etc/spinifex/northstar.toml"}}
+	}
+	return c
+}
+
+func TestZonePublisherWritesAndDeletesZoneObjects(t *testing.T) {
+	endpoint, get := zoneBucket(t)
+	_, nc := testutil.StartTestNATS(t)
+	pub, ok := dns.NewZonePublisher(publisherConfig(t, endpoint), func() *config.ClusterConfig { return cluster("node1") }, nc)
+	require.True(t, ok)
+
+	require.NoError(t, pub.PutZone(context.Background(), "Example.com", []byte("body")))
+	body, ok := get("example.com.toml")
+	require.True(t, ok)
+	assert.Equal(t, "body", body)
+
+	require.NoError(t, pub.DeleteZone(context.Background(), "example.com"))
+	_, ok = get("example.com.toml")
+	assert.False(t, ok)
+}
+
+func TestZonePublisherNeedsTheZoneStore(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	_, ok := dns.NewZonePublisher(&config.Config{}, func() *config.ClusterConfig { return nil }, nc)
+	assert.False(t, ok)
+}
+
+// A reload is complete only when every northstar node has answered; a failed
+// node reports a version no digest matches, and a node answering twice does not
+// stand in for one that never answered.
+func TestZonePublisherReloadCountsEveryNorthstarNode(t *testing.T) {
+	endpoint, _ := zoneBucket(t)
+	nodes := cluster("node1", "node2", "node3")
+	cases := []struct {
+		name     string
+		acks     []dns.ZoneReloadAck
+		served   []string
+		complete bool
+	}{
+		{"one failed, one silent", []dns.ZoneReloadAck{{Node: "node1", ETag: "v1"}, {Node: "node2", Error: "s3 down"}}, []string{"v1", "error: s3 down"}, false},
+		{"one node twice", []dns.ZoneReloadAck{{Node: "node1", ETag: "v1"}, {Node: "node1", ETag: "v1"}}, []string{"v1", "v1"}, false},
+		{"every node", []dns.ZoneReloadAck{{Node: "node1", ETag: "v1"}, {Node: "node2", ETag: "v1"}, {Node: "node3", ETag: "v1"}}, []string{"v1", "v1", "v1"}, true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			_, nc := testutil.StartTestNATS(t)
+			pub, ok := dns.NewZonePublisher(publisherConfig(t, endpoint), func() *config.ClusterConfig { return nodes }, nc)
+			require.True(t, ok)
+			ackAs(t, nc, func(zone string) []dns.ZoneReloadAck {
+				acks := slices.Clone(tc.acks)
+				for i := range acks {
+					acks[i].Zone = zone
+				}
+				return acks
+			})
+
+			served, complete, err := pub.Reload(context.Background(), "example.com")
+			require.NoError(t, err)
+			assert.Equal(t, tc.complete, complete)
+			assert.ElementsMatch(t, tc.served, served)
+		})
+	}
+}
+
+func TestNorthstarServerCount(t *testing.T) {
+	assert.Equal(t, 1, dns.NorthstarServerCount(nil))
+	assert.Equal(t, 1, dns.NorthstarServerCount(&config.ClusterConfig{Nodes: map[string]config.Config{"node1": {}}}))
+	assert.Equal(t, 3, dns.NorthstarServerCount(cluster("node1", "node2", "node3")))
+}

spinifex/domains/dns/reload.go

@@ -0,0 +1,46 @@
+package dns
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/nats-io/nats.go"
+)
+
+// RequestZoneReload asks every northstar server to reload zone and collects
+// their acknowledgements until want have arrived or ctx ends. The acks gathered
+// so far are returned either way; the error reports only a transport failure.
+func RequestZoneReload(ctx context.Context, nc *nats.Conn, zone string, want int) ([]ZoneReloadAck, error) {
+	payload, err := json.Marshal(ZoneReload{Zone: zone})
+	if err != nil {
+		return nil, fmt.Errorf("marshal zone reload: %w", err)
+	}
+	inbox := nc.NewInbox()
+	sub, err := nc.SubscribeSync(inbox)
+	if err != nil {
+		return nil, fmt.Errorf("subscribe reload acks: %w", err)
+	}
+	defer func() { _ = sub.Unsubscribe() }()
+	if err := nc.PublishRequest(SubjectZoneReload, inbox, payload); err != nil {
+		return nil, fmt.Errorf("publish zone reload: %w", err)
+	}
+
+	var acks []ZoneReloadAck
+	for len(acks) < want {
+		msg, err := sub.NextMsgWithContext(ctx)
+		if err != nil {
+			if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
+				return acks, nil
+			}
+			return acks, fmt.Errorf("receive reload ack: %w", err)
+		}
+		var ack ZoneReloadAck
+		if json.Unmarshal(msg.Data, &ack) != nil || ack.Zone != zone {
+			continue
+		}
+		acks = append(acks, ack)
+	}
+	return acks, nil
+}

spinifex/domains/dns/reload_test.go

@@ -0,0 +1,87 @@
+package dns_test
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/dns"
+)
+
+// ackAs subscribes a fake northstar that answers every reload with the acks
+// reply returns for the requested zone.
+func ackAs(t *testing.T, nc *nats.Conn, reply func(zone string) []dns.ZoneReloadAck) {
+	t.Helper()
+	sub, err := nc.Subscribe(dns.SubjectZoneReload, func(msg *nats.Msg) {
+		var evt dns.ZoneReload
+		require.NoError(t, json.Unmarshal(msg.Data, &evt))
+		for _, ack := range reply(evt.Zone) {
+			body, err := json.Marshal(ack)
+			require.NoError(t, err)
+			require.NoError(t, msg.Respond(body))
+		}
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+}
+
+func TestRequestZoneReloadCollectsOneAckPerServer(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	for _, node := range []string{"node1", "node2", "node3"} {
+		ackAs(t, nc, func(zone string) []dns.ZoneReloadAck {
+			return []dns.ZoneReloadAck{{Zone: zone, Node: node, ETag: "v2"}}
+		})
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	acks, err := dns.RequestZoneReload(ctx, nc, "example.com", 3)
+	require.NoError(t, err)
+	require.Len(t, acks, 3)
+	nodes := []string{acks[0].Node, acks[1].Node, acks[2].Node}
+	assert.ElementsMatch(t, []string{"node1", "node2", "node3"}, nodes)
+	for _, ack := range acks {
+		assert.Equal(t, "v2", ack.ETag)
+	}
+}
+
+func TestRequestZoneReloadIgnoresAcksForOtherZones(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	ackAs(t, nc, func(zone string) []dns.ZoneReloadAck {
+		return []dns.ZoneReloadAck{{Zone: "other.com", Node: "stray"}, {Zone: zone, Node: "node1"}}
+	})
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	acks, err := dns.RequestZoneReload(ctx, nc, "example.com", 1)
+	require.NoError(t, err)
+	require.Len(t, acks, 1)
+	assert.Equal(t, "node1", acks[0].Node)
+}
+
+func TestRequestZoneReloadReturnsPartialAcksAtDeadline(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	ackAs(t, nc, func(zone string) []dns.ZoneReloadAck {
+		return []dns.ZoneReloadAck{{Zone: zone, Node: "node1", Error: "s3 unavailable"}}
+	})
+
+	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
+	defer cancel()
+	acks, err := dns.RequestZoneReload(ctx, nc, "example.com", 2)
+	require.NoError(t, err)
+	require.Len(t, acks, 1)
+	assert.Equal(t, "s3 unavailable", acks[0].Error)
+}
+
+func TestRequestZoneReloadFailsOnClosedConnection(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	nc.Close()
+	_, err := dns.RequestZoneReload(context.Background(), nc, "example.com", 1)
+	require.Error(t, err)
+}

spinifex/domains/dns/reserved.go

@@ -0,0 +1,44 @@
+package dns
+
+import (
+	"regexp"
+	"strings"
+)
+
+// managedLabels are the labels directly below the base domain under which the
+// platform writes its own records.
+var managedLabels = map[string]bool{"compute": true, "elb": true, "eks": true, "rds": true}
+
+var nameserverLabel = regexp.MustCompile(`^ns[0-9]+$`)
+
+// ReservedZoneName reports whether a tenant may not create a hosted zone named
+// name. It refuses the base and internal domains, any ancestor of them, every
+// name below the internal domain, and any name below the base domain that falls
+// under a platform-managed label or a published nameserver.
+func ReservedZoneName(name, baseDomain, internalDomain string) bool {
+	name = normalizeZone(name)
+	if name == "" {
+		return true
+	}
+	for _, zone := range []string{normalizeZone(baseDomain), normalizeZone(privateZoneOrDefault(internalDomain))} {
+		if zone == "" {
+			continue
+		}
+		if name == zone || strings.HasSuffix(zone, "."+name) {
+			return true
+		}
+	}
+	if internal := normalizeZone(privateZoneOrDefault(internalDomain)); strings.HasSuffix(name, "."+internal) {
+		return true
+	}
+	if base := normalizeZone(baseDomain); base != "" && strings.HasSuffix(name, "."+base) {
+		labels := strings.Split(strings.TrimSuffix(name, "."+base), ".")
+		top := labels[len(labels)-1]
+		return managedLabels[top] || nameserverLabel.MatchString(top)
+	}
+	return false
+}
+
+func normalizeZone(name string) string {
+	return strings.TrimSuffix(strings.ToLower(strings.TrimSpace(name)), ".")
+}

spinifex/domains/dns/reserved_test.go

@@ -0,0 +1,46 @@
+package dns_test
+
+import (
+	"testing"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/dns"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestReservedZoneName(t *testing.T) {
+	cases := []struct {
+		name     string
+		reserved bool
+	}{
+		{"spx3.net.", true},
+		{"SPX3.NET", true},
+		{"net", true},
+		{"compute.internal", true},
+		{"ip-10-0-0-1.us-west-1.compute.internal.", true},
+		{"internal", true},
+		{"compute.spx3.net", true},
+		{"ec2-1-2-3-4.us-west-1.compute.spx3.net", true},
+		{"lb.us-west-1.elb.spx3.net", true},
+		{"x.eks.spx3.net", true},
+		{"db.rds.spx3.net", true},
+		{"ns1.spx3.net", true},
+		{"a.ns4.spx3.net", true},
+		{"app.spx3.net", false},
+		{"nsx.spx3.net", false},
+		{"compute.app.spx3.net", false},
+		{"example.com", false},
+		{"compute.example.com", false},
+		{"", true},
+	}
+	for _, c := range cases {
+		assert.Equal(t, c.reserved, dns.ReservedZoneName(c.name, "spx3.net", ""), c.name)
+	}
+}
+
+func TestReservedZoneNameHonoursTheConfiguredDomains(t *testing.T) {
+	assert.True(t, dns.ReservedZoneName("cloud.example.com", "cloud.example.com", "corp.internal"))
+	assert.True(t, dns.ReservedZoneName("example.com", "cloud.example.com", "corp.internal"))
+	assert.True(t, dns.ReservedZoneName("host.corp.internal", "cloud.example.com", "corp.internal"))
+	assert.False(t, dns.ReservedZoneName("spx3.net", "cloud.example.com", "corp.internal"))
+	assert.False(t, dns.ReservedZoneName("app.spx3.net", "", ""))
+}

spinifex/domains/dns/types.go

@@ -67,3 +67,13 @@ type ChangeResult struct {
 type ZoneReload struct {
 	Zone string `json:"zone"`
 }
+
+// ZoneReloadAck is one northstar server's reply to a ZoneReload sent with a
+// reply subject. ETag is the version of the zone it now serves, "" when the
+// zone is absent; Error is set when the reload failed.
+type ZoneReloadAck struct {
+	Zone  string `json:"zone"`
+	Node  string `json:"node"`
+	ETag  string `json:"etag,omitempty"`
+	Error string `json:"error,omitempty"`
+}

spinifex/domains/dns/writer.go

@@ -19,14 +19,16 @@ import (
 // of zone TOML files in s3://northstar/ using the system predastore credentials.
 // Only the reconcile's elected node calls it, so a zone has one writer at a time.
 type Writer struct {
-	enabled      bool
-	s3cfg        *nsconfig.S3Config
-	baseDomain   string
-	nameservers  []nsconfig.NameserverSeed
-	ttl          uint32
-	nc           *nats.Conn
-	quotaEnabled bool
-	quotas       Quotas
+	enabled    bool
+	s3cfg      *nsconfig.S3Config
+	baseDomain string
+	// internalDomain is the private zone, for refusing reserved delegations.
+	internalDomain string
+	nameservers    []nsconfig.NameserverSeed
+	ttl            uint32
+	nc             *nats.Conn
+	quotaEnabled   bool
+	quotas         Quotas
 }
 
 // NewWriter resolves the northstar S3 endpoint/bucket from the node's
@@ -45,6 +47,7 @@ func NewWriter(cfg *config.Config, cluster *config.ClusterConfig, nc *nats.Conn)
 	w.enabled = true
 	w.s3cfg = zoneCfg.s3
 	w.baseDomain = strings.TrimSpace(zoneCfg.server.DefaultDomain)
+	w.internalDomain = ResolveInternalDomain(cfg)
 	// Share the bootstrap's derivation so a zone this writer materialises carries
 	// the same NS topology as one the bootstrap seeds; whichever wins the race,
 	// the zone is correct.

spinifex/domains/route53/activation.go

@@ -0,0 +1,187 @@
+package route53
+
+import (
+	"context"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/route53/ext"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+)
+
+// GetHostedZoneActivation reports whether a zone is served and why not.
+func (s *Service) GetHostedZoneActivation(ctx context.Context, in *ext.GetHostedZoneActivationInput, account string) (*ext.GetHostedZoneActivationOutput, error) {
+	id := bareZoneID(aws.StringValue(in.Id))
+	zone, err := s.store.GetZone(ctx, account, id)
+	if err != nil {
+		return nil, s.zoneErr(id, err)
+	}
+	return &ext.GetHostedZoneActivationOutput{HostedZoneId: aws.String(hostedZonePrefix + zone.ID), Activation: activation(zone)}, nil
+}
+
+// VerifyHostedZoneDelegation checks a zone's delegation now, the console's
+// "Check now", rather than at its next scheduled check.
+func (s *Service) VerifyHostedZoneDelegation(ctx context.Context, in *ext.VerifyHostedZoneDelegationInput, account string) (*ext.VerifyHostedZoneDelegationOutput, error) {
+	id := bareZoneID(aws.StringValue(in.Id))
+	if s.Verifier == nil {
+		return nil, awserrors.Errorf(awserrors.ErrorServiceUnavailable, "delegation checks are not available on this cluster")
+	}
+	zone, err := s.Verifier.Verify(ctx, account, id)
+	if err != nil {
+		return nil, s.zoneErr(id, err)
+	}
+	return &ext.VerifyHostedZoneDelegationOutput{HostedZoneId: aws.String(hostedZonePrefix + zone.ID), Activation: activation(zone)}, nil
+}
+
+func activation(z *ZoneRecord) *ext.Activation {
+	a := &ext.Activation{
+		State:            aws.String(z.State),
+		Reason:           nonEmpty(z.StateReason),
+		NameServers:      aws.StringSlice(z.Nameservers),
+		DelegatedTo:      aws.StringSlice(z.Delegation),
+		DelegationSource: nonEmpty(z.DelegationSource),
+		Override:         nonEmpty(z.Override),
+	}
+	if !z.CheckedAt.IsZero() {
+		a.CheckedAt = aws.Time(z.CheckedAt)
+	}
+	if z.Override != "" {
+		a.Reason = nonEmpty(z.OverrideReason)
+	}
+	return a
+}
+
+// AdminZoneQuery selects zones across every account, by ID or by name.
+type AdminZoneQuery struct {
+	ZoneID string `json:"zone_id,omitempty"`
+	Name   string `json:"name,omitempty"`
+}
+
+// AdminZones is the zones a query matched.
+type AdminZones struct {
+	Zones []ZoneRecord `json:"zones"`
+}
+
+// AdminOverrideInput sets or clears an operator override on a zone.
+type AdminOverrideInput struct {
+	ZoneID string `json:"zone_id"`
+	// Override is StateActive, StateSuspended, or "" to hand the zone back to
+	// the verifier.
+	Override string `json:"override"`
+	Reason   string `json:"reason"`
+}
+
+// AdminDelegateInput names the platform-zone delegations to write or, with
+// Remove, withdraw.
+type AdminDelegateInput struct {
+	Names  []string `json:"names"`
+	Remove bool     `json:"remove,omitempty"`
+}
+
+// AdminDelegateOutput reports how many record changes were applied.
+type AdminDelegateOutput struct {
+	Applied int `json:"applied"`
+}
+
+// AdminFindZones lists the zones matching q in every account. It is for the
+// operator and is never reachable through the gateway.
+func (s *Service) AdminFindZones(ctx context.Context, q *AdminZoneQuery, _ string) (*AdminZones, error) {
+	zones, err := s.store.ListAllZones(ctx)
+	if err != nil {
+		return nil, internal("list hosted zones", err)
+	}
+	id, name := bareZoneID(q.ZoneID), normalizeName(q.Name)
+	out := &AdminZones{Zones: []ZoneRecord{}}
+	for _, z := range zones {
+		if (id == "" || z.ID == id) && (name == "" || z.Name == name) {
+			out.Zones = append(out.Zones, z)
+		}
+	}
+	slices.SortFunc(out.Zones, func(a, b ZoneRecord) int { return olderFirst(&a, &b) })
+	return out, nil
+}
+
+// AdminVerify checks one zone now, in whichever account holds it.
+func (s *Service) AdminVerify(ctx context.Context, q *AdminZoneQuery, _ string) (*AdminZones, error) {
+	z, err := s.adminZone(ctx, q.ZoneID)
+	if err != nil {
+		return nil, err
+	}
+	if s.Verifier == nil {
+		return nil, awserrors.Errorf(awserrors.ErrorServiceUnavailable, "delegation checks are not available on this cluster")
+	}
+	checked, err := s.Verifier.Verify(ctx, z.AccountID, z.ID)
+	if err != nil {
+		return nil, s.zoneErr(z.ID, err)
+	}
+	return &AdminZones{Zones: []ZoneRecord{*checked}}, nil
+}
+
+// AdminOverride activates or suspends a zone whatever its delegation, or
+// clears an override. Activation refuses a name active in another zone.
+func (s *Service) AdminOverride(ctx context.Context, in *AdminOverrideInput, _ string) (*AdminZones, error) {
+	if in.Override != "" && strings.TrimSpace(in.Reason) == "" {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "an override needs a reason")
+	}
+	z, err := s.adminZone(ctx, in.ZoneID)
+	if err != nil {
+		return nil, err
+	}
+	switch in.Override {
+	case StateActive:
+		zones, err := s.store.ListAllZones(ctx)
+		if err != nil {
+			return nil, internal("list hosted zones", err)
+		}
+		for _, other := range zones {
+			if other.Name == z.Name && other.ID != z.ID && other.State == StateActive {
+				return nil, awserrors.Errorf(awserrors.ErrorConflictingDomainExists, "%s is already active in account %s as %s", strings.TrimSuffix(z.Name, "."), other.AccountID, other.ID)
+			}
+		}
+	case StateSuspended, "":
+	default:
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "override must be %s, %s or empty", StateActive, StateSuspended)
+	}
+	err = s.store.MutateZone(ctx, z.AccountID, z.ID, func(r *ZoneRecord) (bool, error) {
+		switch in.Override {
+		case "":
+			// The verifier decides again from here, starting with a check.
+			if r.State == StateSuspended || r.Override == StateActive {
+				r.State = StatePendingNameservers
+			}
+			r.CheckedAt = time.Time{}
+		default:
+			r.State = in.Override
+		}
+		r.Override, r.OverrideReason = in.Override, strings.TrimSpace(in.Reason)
+		r.FailingSince = time.Time{}
+		return true, nil
+	})
+	if err != nil {
+		return nil, s.zoneErr(z.ID, err)
+	}
+	updated, err := s.store.GetZone(ctx, z.AccountID, z.ID)
+	if err != nil {
+		return nil, s.zoneErr(z.ID, err)
+	}
+	return &AdminZones{Zones: []ZoneRecord{*updated}}, nil
+}
+
+func (s *Service) adminZone(ctx context.Context, rawID string) (*ZoneRecord, error) {
+	id := bareZoneID(rawID)
+	if id == "" {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "a hosted zone ID is required")
+	}
+	found, err := s.AdminFindZones(ctx, &AdminZoneQuery{ZoneID: id}, "")
+	if err != nil {
+		return nil, err
+	}
+	if len(found.Zones) == 0 {
+		return nil, s.zoneErr(id, kvstore.ErrNotFound)
+	}
+	return &found.Zones[0], nil
+}

spinifex/domains/route53/activation_test.go

@@ -0,0 +1,111 @@
+package route53_test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/route53"
+	"github.com/mulgadc/spinifex/spinifex/domains/route53/ext"
+)
+
+func TestActivationIsReportedAndCheckedOnDemand(t *testing.T) {
+	svc, v, dns, _ := newVerifier(t)
+	svc.Verifier = v
+	id := createZone(t, svc, acctA, "demand.example")
+
+	got, err := svc.GetHostedZoneActivation(ctx, &ext.GetHostedZoneActivationInput{Id: aws.String(id)}, acctA)
+	require.NoError(t, err)
+	assert.Equal(t, route53.StatePendingNameservers, aws.StringValue(got.Activation.State))
+	assert.Nil(t, got.Activation.CheckedAt, "never checked")
+
+	dns.parents["demand.example"] = ours()
+	verified, err := svc.VerifyHostedZoneDelegation(ctx, &ext.VerifyHostedZoneDelegationInput{Id: aws.String(id)}, acctA)
+	require.NoError(t, err)
+	assert.Equal(t, route53.StateActive, aws.StringValue(verified.Activation.State))
+	assert.Equal(t, ours(), aws.StringValueSlice(verified.Activation.DelegatedTo))
+	assert.NotNil(t, verified.Activation.CheckedAt)
+
+	_, err = svc.GetHostedZoneActivation(ctx, &ext.GetHostedZoneActivationInput{Id: aws.String(id)}, acctB)
+	requireCode(t, err, "NoSuchHostedZone")
+	_, err = svc.VerifyHostedZoneDelegation(ctx, &ext.VerifyHostedZoneDelegationInput{Id: aws.String(id)}, acctB)
+	requireCode(t, err, "NoSuchHostedZone")
+}
+
+func TestAdminFindsZonesAcrossAccounts(t *testing.T) {
+	svc, _, _, c := newVerifier(t)
+	a := createZone(t, svc, acctA, "shared.example")
+	c.advance(time.Second)
+	createZone(t, svc, acctB, "shared.example")
+	createZone(t, svc, acctB, "other.example")
+
+	byName, err := svc.AdminFindZones(ctx, &route53.AdminZoneQuery{Name: "Shared.Example"}, "")
+	require.NoError(t, err)
+	require.Len(t, byName.Zones, 2)
+	assert.Equal(t, acctA, byName.Zones[0].AccountID, "oldest first")
+
+	byID, err := svc.AdminFindZones(ctx, &route53.AdminZoneQuery{ZoneID: a}, "")
+	require.NoError(t, err)
+	require.Len(t, byID.Zones, 1)
+	assert.Equal(t, zoneID(a), byID.Zones[0].ID)
+}
+
+func TestAdminOverrideActivatesSuspendsAndReleases(t *testing.T) {
+	svc, v, dns, c := newVerifier(t)
+	svc.Verifier = v
+	held := createZone(t, svc, acctA, "contested.example")
+	c.advance(time.Second)
+	squat := createZone(t, svc, acctB, "contested.example")
+
+	_, err := svc.AdminOverride(ctx, &route53.AdminOverrideInput{ZoneID: held, Override: route53.StateActive}, "")
+	requireCode(t, err, "InvalidInput")
+
+	out, err := svc.AdminOverride(ctx, &route53.AdminOverrideInput{ZoneID: held, Override: route53.StateActive, Reason: "air-gapped root"}, "")
+	require.NoError(t, err)
+	assert.Equal(t, route53.StateActive, out.Zones[0].State)
+
+	_, err = svc.AdminOverride(ctx, &route53.AdminOverrideInput{ZoneID: squat, Override: route53.StateActive, Reason: "mine"}, "")
+	requireCode(t, err, "ConflictingDomainExists")
+
+	// The delegation check never demotes an operator's activation.
+	pass(t, v)
+	assert.Equal(t, route53.StateActive, zone(t, svc, acctA, held).State)
+
+	_, err = svc.AdminOverride(ctx, &route53.AdminOverrideInput{ZoneID: held, Override: route53.StateSuspended, Reason: "dispute"}, "")
+	require.NoError(t, err)
+	dns.parents["contested.example"] = ours()
+	c.advance(time.Minute)
+	pass(t, v)
+	assert.Equal(t, route53.StateSuspended, zone(t, svc, acctA, held).State)
+	assert.Equal(t, route53.StateActive, zone(t, svc, acctB, squat).State, "suspending a zone frees its name")
+
+	_, err = svc.AdminOverride(ctx, &route53.AdminOverrideInput{ZoneID: held}, "")
+	require.NoError(t, err)
+	z := zone(t, svc, acctA, held)
+	assert.Equal(t, route53.StatePendingNameservers, z.State)
+	assert.Empty(t, z.Override)
+	pass(t, v)
+	assert.Equal(t, "the domain is active in another account", zone(t, svc, acctA, held).StateReason)
+
+	_, err = svc.AdminOverride(ctx, &route53.AdminOverrideInput{ZoneID: "ZNOPE"}, "")
+	requireCode(t, err, "NoSuchHostedZone")
+	_, err = svc.AdminOverride(ctx, &route53.AdminOverrideInput{ZoneID: held, Override: "BOGUS", Reason: "x"}, "")
+	requireCode(t, err, "InvalidInput")
+}
+
+func TestAdminVerifyChecksInTheOwningAccount(t *testing.T) {
+	svc, v, dns, _ := newVerifier(t)
+	id := createZone(t, svc, acctB, "admin.example")
+	_, err := svc.AdminVerify(ctx, &route53.AdminZoneQuery{ZoneID: id}, "")
+	requireCode(t, err, "ServiceUnavailable")
+
+	svc.Verifier = v
+	dns.parents["admin.example"] = ours()
+	out, err := svc.AdminVerify(ctx, &route53.AdminZoneQuery{ZoneID: id}, "")
+	require.NoError(t, err)
+	assert.Equal(t, route53.StateActive, out.Zones[0].State)
+	assert.Equal(t, acctB, out.Zones[0].AccountID)
+}

spinifex/domains/route53/awsapi/authz.go

@@ -0,0 +1,60 @@
+package awsapi
+
+import (
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/route53"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/route53/ext"
+)
+
+// Route 53 ARNs carry no region or account, so the policy gate cannot scope
+// them to the caller; the daemon looks every ID up within the caller's account.
+const arnPrefix = "arn:aws:route53:::"
+
+// ResourceARNs returns the resources an action acts on, for the policy gate.
+// Actions that name no zone or change are authorized against "*".
+func ResourceARNs(input any) []string {
+	switch in := input.(type) {
+	case *route53.GetHostedZoneInput:
+		return zoneARN(in.Id)
+	case *route53.UpdateHostedZoneCommentInput:
+		return zoneARN(in.Id)
+	case *route53.DeleteHostedZoneInput:
+		return zoneARN(in.Id)
+	case *route53.ChangeResourceRecordSetsInput:
+		return zoneARN(in.HostedZoneId)
+	case *route53.ListResourceRecordSetsInput:
+		return zoneARN(in.HostedZoneId)
+	case *ext.GetHostedZoneActivationInput:
+		return zoneARN(in.Id)
+	case *ext.VerifyHostedZoneDelegationInput:
+		return zoneARN(in.Id)
+	case *route53.GetChangeInput:
+		return []string{arnPrefix + "change/" + strings.TrimPrefix(aws.StringValue(in.Id), "/change/")}
+	case *route53.ChangeTagsForResourceInput:
+		return tagARNs(in.ResourceType, in.ResourceId)
+	case *route53.ListTagsForResourceInput:
+		return tagARNs(in.ResourceType, in.ResourceId)
+	case *route53.ListTagsForResourcesInput:
+		var out []string
+		for _, id := range in.ResourceIds {
+			out = append(out, tagARNs(in.ResourceType, id)...)
+		}
+		if len(out) == 0 {
+			return []string{"*"}
+		}
+		return out
+	default:
+		return []string{"*"}
+	}
+}
+
+func zoneARN(id *string) []string {
+	return []string{arnPrefix + "hostedzone/" + strings.TrimPrefix(aws.StringValue(id), "/hostedzone/")}
+}
+
+func tagARNs(resourceType, id *string) []string {
+	return []string{arnPrefix + aws.StringValue(resourceType) + "/" + aws.StringValue(id)}
+}

spinifex/domains/route53/awsapi/codec.go

@@ -0,0 +1,103 @@
+package awsapi
+
+import (
+	"bytes"
+	"encoding/xml"
+	"errors"
+	"log/slog"
+	"net/http"
+	"net/url"
+	"reflect"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	awsxml "github.com/mulgadc/spinifex/spinifex/foundation/aws/xml"
+)
+
+// XMLNamespace is the namespace every Route 53 request and response carries.
+const XMLNamespace = "https://route53.amazonaws.com/doc/2013-04-01/"
+
+// bindInput fills an SDK input struct the way the REST-XML protocol lays it
+// out: members tagged location:"uri" from the path, location:"querystring"
+// from the query, and the rest from the XML body.
+func bindInput(input any, path map[string]string, query url.Values, body []byte) error {
+	v := reflect.ValueOf(input).Elem()
+	t := v.Type()
+	for i := range t.NumField() {
+		f := t.Field(i)
+		var raw string
+		var ok bool
+		switch f.Tag.Get("location") {
+		case "uri":
+			raw, ok = path[f.Tag.Get("locationName")]
+		case "querystring":
+			raw, ok = query.Get(f.Tag.Get("locationName")), query.Has(f.Tag.Get("locationName"))
+		default:
+			continue
+		}
+		if !ok || f.Type != reflect.TypeFor[*string]() {
+			continue
+		}
+		v.Field(i).Set(reflect.ValueOf(aws.String(raw)))
+	}
+	if len(bytes.TrimSpace(body)) == 0 {
+		return nil
+	}
+	if err := xmlutil.UnmarshalXML(input, xml.NewDecoder(bytes.NewReader(body)), ""); err != nil {
+		return awserrors.Errorf(awserrors.ErrorInvalidInput, "the request body is not valid XML: %v", err)
+	}
+	return nil
+}
+
+// writeXML renders output under a <ActionResponse> root in the Route 53
+// namespace, plus the Location header CreateHostedZone carries.
+func writeXML(w http.ResponseWriter, r *http.Request, action string, status int, output any) error {
+	if loc := locationHeader(output); loc != "" {
+		w.Header().Set("Location", "https://"+r.Host+loc)
+	}
+	body, err := awsxml.Marshal(rootElement(action+"Response", output))
+	if err != nil {
+		slog.ErrorContext(r.Context(), "Route 53: marshal response", "action", action, "err", err)
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+	w.Header().Set("Content-Type", "text/xml")
+	w.WriteHeader(status)
+	_, _ = w.Write([]byte(xml.Header))
+	_, _ = w.Write(body)
+	return nil
+}
+
+func rootElement(name string, payload any) any {
+	t := reflect.StructOf([]reflect.StructField{{
+		Name: "Value",
+		Type: reflect.TypeOf(payload),
+		Tag:  reflect.StructTag(`locationName:"` + name + `" xmlURI:"` + XMLNamespace + `"`),
+	}})
+	v := reflect.New(t).Elem()
+	v.Field(0).Set(reflect.ValueOf(payload))
+	return v.Interface()
+}
+
+func locationHeader(output any) string {
+	v := reflect.ValueOf(output)
+	if v.Kind() != reflect.Pointer || v.IsNil() {
+		return ""
+	}
+	f := v.Elem().FieldByName("Location")
+	if !f.IsValid() || f.Kind() != reflect.Pointer || f.IsNil() || f.Elem().Kind() != reflect.String {
+		return ""
+	}
+	return f.Elem().String()
+}
+
+func patternNames(pattern string) []string {
+	var names []string
+	for seg := range strings.SplitSeq(pattern, "/") {
+		if strings.HasPrefix(seg, "{") && strings.HasSuffix(seg, "}") {
+			names = append(names, seg[1:len(seg)-1])
+		}
+	}
+	return names
+}

spinifex/domains/route53/awsapi/registration.go

@@ -0,0 +1,176 @@
+// Package awsapi adapts Route 53's REST-XML API to the route53 domain's
+// NATS-backed service. Generic gateway routing and authorization stay at the
+// ingress boundary.
+package awsapi
+
+import (
+	"context"
+	"errors"
+	"io"
+	"log/slog"
+	"net/http"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/route53"
+	"github.com/mulgadc/bluebottle/pkg/sigv4"
+	"github.com/mulgadc/spinifex/spinifex/domains/route53/ext"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/rest"
+	"github.com/nats-io/nats.go"
+)
+
+// ServiceName is the SigV4 credential-scope service Route 53 clients sign for.
+const ServiceName = "route53"
+
+// natsTimeout bounds the gateway's wait for the daemon's answer.
+const natsTimeout = 30 * time.Second
+
+// Deps are the composed capabilities the Route 53 control plane dispatches
+// to. A nil NATS connection answers every action with ServerInternal.
+type Deps struct {
+	NATS *nats.Conn
+}
+
+// operation is one Route 53 action: its wire input and output types and the
+// status a success returns.
+type operation struct {
+	newInput func() any
+	relay    func(ctx context.Context, nc *nats.Conn, subject string, in any, account string) (any, error)
+	status   int
+}
+
+func op[I, O any](status int) operation {
+	return operation{
+		newInput: func() any { return new(I) },
+		relay: func(ctx context.Context, nc *nats.Conn, subject string, in any, account string) (any, error) {
+			return natsmsg.NATSRequest[O](ctx, nc, subject, in, natsTimeout, account)
+		},
+		status: status,
+	}
+}
+
+const base = "/2013-04-01"
+
+// routes is the Route 53 REST surface Spinifex serves, paths as in the
+// 2013-04-01 model.
+var routes = []rest.Route[operation]{
+	{Method: http.MethodPost, Pattern: base + "/hostedzone", Action: "CreateHostedZone", Handler: op[route53.CreateHostedZoneInput, route53.CreateHostedZoneOutput](http.StatusCreated)},
+	{Method: http.MethodGet, Pattern: base + "/hostedzone", Action: "ListHostedZones", Handler: op[route53.ListHostedZonesInput, route53.ListHostedZonesOutput](http.StatusOK)},
+	{Method: http.MethodGet, Pattern: base + "/hostedzonesbyname", Action: "ListHostedZonesByName", Handler: op[route53.ListHostedZonesByNameInput, route53.ListHostedZonesByNameOutput](http.StatusOK)},
+	{Method: http.MethodGet, Pattern: base + "/hostedzonecount", Action: "GetHostedZoneCount", Handler: op[route53.GetHostedZoneCountInput, route53.GetHostedZoneCountOutput](http.StatusOK)},
+	{Method: http.MethodGet, Pattern: base + "/hostedzone/{Id}", Action: "GetHostedZone", Handler: op[route53.GetHostedZoneInput, route53.GetHostedZoneOutput](http.StatusOK)},
+	{Method: http.MethodPost, Pattern: base + "/hostedzone/{Id}", Action: "UpdateHostedZoneComment", Handler: op[route53.UpdateHostedZoneCommentInput, route53.UpdateHostedZoneCommentOutput](http.StatusOK)},
+	{Method: http.MethodDelete, Pattern: base + "/hostedzone/{Id}", Action: "DeleteHostedZone", Handler: op[route53.DeleteHostedZoneInput, route53.DeleteHostedZoneOutput](http.StatusOK)},
+	{Method: http.MethodPost, Pattern: base + "/hostedzone/{Id}/rrset", Action: "ChangeResourceRecordSets", Handler: op[route53.ChangeResourceRecordSetsInput, route53.ChangeResourceRecordSetsOutput](http.StatusOK)},
+	{Method: http.MethodGet, Pattern: base + "/hostedzone/{Id}/rrset", Action: "ListResourceRecordSets", Handler: op[route53.ListResourceRecordSetsInput, route53.ListResourceRecordSetsOutput](http.StatusOK)},
+	{Method: http.MethodGet, Pattern: base + "/change/{Id}", Action: "GetChange", Handler: op[route53.GetChangeInput, route53.GetChangeOutput](http.StatusOK)},
+	{Method: http.MethodPost, Pattern: base + "/tags/{ResourceType}/{ResourceId}", Action: "ChangeTagsForResource", Handler: op[route53.ChangeTagsForResourceInput, route53.ChangeTagsForResourceOutput](http.StatusOK)},
+	{Method: http.MethodGet, Pattern: base + "/tags/{ResourceType}/{ResourceId}", Action: "ListTagsForResource", Handler: op[route53.ListTagsForResourceInput, route53.ListTagsForResourceOutput](http.StatusOK)},
+	{Method: http.MethodPost, Pattern: base + "/tags/{ResourceType}", Action: "ListTagsForResources", Handler: op[route53.ListTagsForResourcesInput, route53.ListTagsForResourcesOutput](http.StatusOK)},
+
+	// Spinifex extensions, which standard SDKs never call.
+	{Method: http.MethodGet, Pattern: base + "/hostedzone/{Id}/activation", Action: ext.ActionGetHostedZoneActivation, Handler: op[ext.GetHostedZoneActivationInput, ext.GetHostedZoneActivationOutput](http.StatusOK)},
+	{Method: http.MethodPost, Pattern: base + "/hostedzone/{Id}/activation", Action: ext.ActionVerifyHostedZoneDelegation, Handler: op[ext.VerifyHostedZoneDelegationInput, ext.VerifyHostedZoneDelegationOutput](http.StatusOK)},
+}
+
+// extensions are served but are not operations of the AWS model.
+var extensions = []string{ext.ActionGetHostedZoneActivation, ext.ActionVerifyHostedZoneDelegation}
+
+var router = rest.NewRouter(ServiceName, routes)
+
+// NewRegistration returns the Route 53 REST-XML control-plane registration.
+func NewRegistration(deps Deps) dispatch.Registration {
+	return dispatch.Registration{
+		Service:       ServiceName,
+		ResolveAction: resolveAction,
+		Dispatch:      deps.dispatch,
+		Errors:        dispatch.ErrorEnvelopeXML,
+		Inventory:     OperationInventory(),
+	}
+}
+
+// ExtensionActions lists the Spinifex actions served beside the AWS model's.
+func ExtensionActions() []string { return slices.Clone(extensions) }
+
+// OperationInventory lists the AWS model actions the registration serves.
+func OperationInventory() dispatch.Inventory {
+	names := make([]string, 0, len(routes))
+	for _, r := range routes {
+		if !slices.Contains(extensions, r.Action) {
+			names = append(names, r.Action)
+		}
+	}
+	slices.Sort(names)
+	return dispatch.Inventory{Registered: names}
+}
+
+func resolveAction(r *http.Request) string {
+	action, _, _, _ := lookup(r)
+	return action
+}
+
+// lookup matches a request to its route. The model's ChangeResourceRecordSets
+// path ends in a slash, which SDKs send and the router does not match.
+func lookup(r *http.Request) (string, []string, operation, bool) {
+	return router.Lookup(r.Method, strings.TrimSuffix(r.URL.EscapedPath(), "/"))
+}
+
+// dispatch binds the request, authorizes it against the zone or change it
+// names, relays it to the daemon and writes the XML response.
+func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
+	r := inv.Request
+	action, params, operation, ok := lookup(r)
+	if !ok {
+		slog.DebugContext(r.Context(), "Route 53: no route", "method", r.Method, "path", r.URL.Path)
+		return errors.New(awserrors.ErrorInvalidAction)
+	}
+	if inv.AccountID == "" {
+		slog.ErrorContext(r.Context(), "Route 53 dispatch: no account ID in auth context")
+		return errors.New(awserrors.ErrorInternalError)
+	}
+	body, err := io.ReadAll(io.LimitReader(r.Body, sigv4.MaxPayloadLen+1))
+	if err != nil {
+		return errors.New(awserrors.ErrorInvalidInput)
+	}
+	if int64(len(body)) > sigv4.MaxPayloadLen {
+		return errors.New(awserrors.ErrorRequestEntityTooLarge)
+	}
+
+	input := operation.newInput()
+	if err := bindInput(input, pathParams(action, params), r.URL.Query(), body); err != nil {
+		return err
+	}
+	if err := inv.Authorize(ServiceName, action, ResourceARNs(input), nil); err != nil {
+		return err
+	}
+	if d.NATS == nil {
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+	output, err := operation.relay(r.Context(), d.NATS, ServiceName+"."+action, input, inv.AccountID)
+	if err != nil {
+		return err
+	}
+	return writeXML(w, r, action, operation.status, output)
+}
+
+// pathParams names the router's positional params by the route's pattern.
+func pathParams(action string, params []string) map[string]string {
+	for _, rt := range routes {
+		if rt.Action != action {
+			continue
+		}
+		names := patternNames(rt.Pattern)
+		out := make(map[string]string, len(names))
+		for i, n := range names {
+			if i < len(params) {
+				out[n] = params[i]
+			}
+		}
+		return out
+	}
+	return nil
+}

spinifex/domains/route53/awsapi/registration_test.go

@@ -0,0 +1,324 @@
+package awsapi_test
+
+import (
+	"context"
+	"encoding/json"
+	"encoding/xml"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/aws/session"
+	r53 "github.com/aws/aws-sdk-go/service/route53"
+	"github.com/mulgadc/bluebottle/pkg/iampolicy"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/route53"
+	"github.com/mulgadc/spinifex/spinifex/domains/route53/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/domains/route53/ext"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
+)
+
+const account = "000000000011"
+
+type authorized struct {
+	mu    sync.Mutex
+	calls []string
+}
+
+func (a *authorized) record(service, action string, resources []string, _ iampolicy.ConditionKeys) error {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	a.calls = append(a.calls, fmt.Sprintf("%s:%s %s", service, action, strings.Join(resources, ",")))
+	return nil
+}
+
+// serveDomain answers route53.<Method> the way the daemon does, from the real
+// service, so the SDK below exercises the whole wire path.
+func serveDomain(t *testing.T, nc *nats.Conn, svc *route53.Service) {
+	t.Helper()
+	v := reflect.ValueOf(svc)
+	for _, action := range append(awsapi.OperationInventory().Registered, awsapi.ExtensionActions()...) {
+		m := v.MethodByName(action)
+		require.True(t, m.IsValid(), "the service has no method for %s", action)
+		inType := m.Type().In(1).Elem()
+		sub, err := nc.Subscribe("route53."+action, func(msg *nats.Msg) {
+			in := reflect.New(inType)
+			require.NoError(t, json.Unmarshal(msg.Data, in.Interface()))
+			out := m.Call([]reflect.Value{reflect.ValueOf(context.Background()), in, reflect.ValueOf(natsmsg.AccountIDFromMsg(msg))})
+			if err, _ := reflect.TypeAssert[error](out[1]); err != nil {
+				_, message, _ := awserrors.ResolveErrorDetail(err)
+				_ = msg.Respond(awserrors.GenerateErrorPayloadWithMessage(awserrors.ValidErrorCodeFromError(err), message))
+				return
+			}
+			body, err := json.Marshal(out[0].Interface())
+			require.NoError(t, err)
+			_ = msg.Respond(body)
+		})
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = sub.Unsubscribe() })
+	}
+}
+
+// newClient returns an SDK client whose requests run through the
+// registration, with errors rendered as the gateway renders them.
+func newClient(t *testing.T) (*r53.Route53, *authorized) {
+	t.Helper()
+	_, nc, js := testutil.StartTestJetStream(t)
+	svc := route53.NewService(route53.NewStore(js))
+	svc.Nameservers = func() []string { return []string{"ns1.spx3.net", "ns2.spx3.net"} }
+	serveDomain(t, nc, svc)
+
+	reg := awsapi.NewRegistration(awsapi.Deps{NATS: nc})
+	auth := &authorized{}
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		err := reg.Dispatch(w, dispatch.Invocation{Request: r, AccountID: account, Region: "us-east-1", Authorize: auth.record})
+		if err == nil {
+			return
+		}
+		code, message, _ := awserrors.ResolveErrorDetail(err)
+		if code == "" {
+			code = err.Error()
+		}
+		meta := awserrors.LookupErrorMessage(awsapi.ServiceName, code)
+		if message == "" {
+			message = meta.Message
+		}
+		w.Header().Set("Content-Type", "text/xml")
+		w.WriteHeader(meta.HTTPCode)
+		_ = xml.NewEncoder(w).Encode(struct {
+			XMLName xml.Name `xml:"ErrorResponse"`
+			Type    string   `xml:"Error>Type"`
+			Code    string   `xml:"Error>Code"`
+			Message string   `xml:"Error>Message"`
+		}{Type: "Sender", Code: code, Message: message})
+	}))
+	t.Cleanup(srv.Close)
+
+	sess, err := session.NewSession(&aws.Config{
+		Endpoint:    aws.String(srv.URL),
+		Region:      aws.String("us-east-1"),
+		Credentials: credentials.NewStaticCredentials("AKID", "secret", ""),
+		MaxRetries:  aws.Int(0),
+	})
+	require.NoError(t, err)
+	return r53.New(sess), auth
+}
+
+func requireAWSCode(t *testing.T, err error, code string) {
+	t.Helper()
+	var aerr awserr.Error
+	require.ErrorAs(t, err, &aerr)
+	assert.Equal(t, code, aerr.Code(), aerr.Message())
+}
+
+func TestSDKDrivesTheTerraformSequence(t *testing.T) {
+	client, auth := newClient(t)
+
+	created, err := client.CreateHostedZone(&r53.CreateHostedZoneInput{
+		Name: aws.String("example.com"), CallerReference: aws.String("terraform-1"),
+		HostedZoneConfig: &r53.HostedZoneConfig{Comment: aws.String("Managed by Terraform")},
+	})
+	require.NoError(t, err)
+	zoneID := aws.StringValue(created.HostedZone.Id)
+	bare := strings.TrimPrefix(zoneID, "/hostedzone/")
+	assert.True(t, strings.HasSuffix(aws.StringValue(created.Location), "/2013-04-01/hostedzone/"+bare), aws.StringValue(created.Location))
+	assert.Equal(t, []string{"ns1.spx3.net", "ns2.spx3.net"}, aws.StringValueSlice(created.DelegationSet.NameServers))
+	assert.Equal(t, "INSYNC", aws.StringValue(created.ChangeInfo.Status))
+	assert.False(t, created.ChangeInfo.SubmittedAt.IsZero())
+
+	_, err = client.GetChange(&r53.GetChangeInput{Id: created.ChangeInfo.Id})
+	require.NoError(t, err)
+
+	got, err := client.GetHostedZone(&r53.GetHostedZoneInput{Id: aws.String(zoneID)})
+	require.NoError(t, err)
+	assert.Equal(t, "example.com.", aws.StringValue(got.HostedZone.Name))
+	assert.Equal(t, "Managed by Terraform", aws.StringValue(got.HostedZone.Config.Comment))
+
+	tags, err := client.ListTagsForResource(&r53.ListTagsForResourceInput{ResourceType: aws.String("hostedzone"), ResourceId: aws.String(bare)})
+	require.NoError(t, err)
+	assert.Equal(t, bare, aws.StringValue(tags.ResourceTagSet.ResourceId))
+
+	record := &r53.ResourceRecordSet{
+		Name: aws.String("www.example.com"), Type: aws.String("A"), TTL: aws.Int64(300),
+		ResourceRecords: []*r53.ResourceRecord{{Value: aws.String("192.0.2.10")}},
+	}
+	changed, err := client.ChangeResourceRecordSets(&r53.ChangeResourceRecordSetsInput{
+		HostedZoneId: aws.String(bare),
+		ChangeBatch:  &r53.ChangeBatch{Comment: aws.String("Managed by Terraform"), Changes: []*r53.Change{{Action: aws.String("CREATE"), ResourceRecordSet: record}}},
+	})
+	require.NoError(t, err)
+	assert.Equal(t, "Managed by Terraform", aws.StringValue(changed.ChangeInfo.Comment))
+
+	sets, err := client.ListResourceRecordSets(&r53.ListResourceRecordSetsInput{
+		HostedZoneId: aws.String(bare), MaxItems: aws.String("1"),
+		StartRecordName: aws.String("www.example.com."), StartRecordType: aws.String("A"),
+	})
+	require.NoError(t, err)
+	require.Len(t, sets.ResourceRecordSets, 1)
+	assert.Equal(t, "www.example.com.", aws.StringValue(sets.ResourceRecordSets[0].Name))
+	assert.Equal(t, "192.0.2.10", aws.StringValue(sets.ResourceRecordSets[0].ResourceRecords[0].Value))
+	assert.Equal(t, int64(300), aws.Int64Value(sets.ResourceRecordSets[0].TTL))
+
+	_, err = client.ChangeResourceRecordSets(&r53.ChangeResourceRecordSetsInput{
+		HostedZoneId: aws.String(bare),
+		ChangeBatch:  &r53.ChangeBatch{Changes: []*r53.Change{{Action: aws.String("CREATE"), ResourceRecordSet: record}}},
+	})
+	requireAWSCode(t, err, "InvalidChangeBatch")
+
+	_, err = client.DeleteHostedZone(&r53.DeleteHostedZoneInput{Id: aws.String(bare)})
+	requireAWSCode(t, err, "HostedZoneNotEmpty")
+
+	record.Name = aws.String("www.example.com.")
+	_, err = client.ChangeResourceRecordSets(&r53.ChangeResourceRecordSetsInput{
+		HostedZoneId: aws.String(bare),
+		ChangeBatch:  &r53.ChangeBatch{Comment: aws.String("Deleted by Terraform"), Changes: []*r53.Change{{Action: aws.String("DELETE"), ResourceRecordSet: record}}},
+	})
+	require.NoError(t, err)
+	deleted, err := client.DeleteHostedZone(&r53.DeleteHostedZoneInput{Id: aws.String(bare)})
+	require.NoError(t, err)
+	_, err = client.GetChange(&r53.GetChangeInput{Id: deleted.ChangeInfo.Id})
+	require.NoError(t, err)
+
+	_, err = client.GetHostedZone(&r53.GetHostedZoneInput{Id: aws.String(bare)})
+	requireAWSCode(t, err, "NoSuchHostedZone")
+
+	assert.Contains(t, auth.calls, "route53:GetHostedZone arn:aws:route53:::hostedzone/"+bare)
+	assert.Contains(t, auth.calls, "route53:ChangeResourceRecordSets arn:aws:route53:::hostedzone/"+bare)
+	assert.Contains(t, auth.calls, "route53:ListTagsForResource arn:aws:route53:::hostedzone/"+bare)
+	assert.Contains(t, auth.calls, "route53:CreateHostedZone *")
+}
+
+func TestSDKListsAndTags(t *testing.T) {
+	client, _ := newClient(t)
+	var ids []string
+	for _, name := range []string{"b.example.com", "example.com", "example.org"} {
+		out, err := client.CreateHostedZone(&r53.CreateHostedZoneInput{Name: aws.String(name), CallerReference: aws.String(name)})
+		require.NoError(t, err)
+		ids = append(ids, strings.TrimPrefix(aws.StringValue(out.HostedZone.Id), "/hostedzone/"))
+	}
+
+	var listed []string
+	require.NoError(t, client.ListHostedZonesPages(&r53.ListHostedZonesInput{MaxItems: aws.String("2")}, func(p *r53.ListHostedZonesOutput, _ bool) bool {
+		for _, z := range p.HostedZones {
+			listed = append(listed, aws.StringValue(z.Name))
+		}
+		return true
+	}))
+	assert.ElementsMatch(t, []string{"b.example.com.", "example.com.", "example.org."}, listed)
+
+	byName, err := client.ListHostedZonesByName(&r53.ListHostedZonesByNameInput{DNSName: aws.String("example.com"), MaxItems: aws.String("2")})
+	require.NoError(t, err)
+	require.Len(t, byName.HostedZones, 2)
+	assert.Equal(t, "example.com.", aws.StringValue(byName.HostedZones[0].Name))
+	assert.Equal(t, "b.example.com.", aws.StringValue(byName.HostedZones[1].Name))
+	assert.True(t, aws.BoolValue(byName.IsTruncated))
+	assert.Equal(t, "example.org.", aws.StringValue(byName.NextDNSName))
+
+	count, err := client.GetHostedZoneCount(&r53.GetHostedZoneCountInput{})
+	require.NoError(t, err)
+	assert.Equal(t, int64(3), aws.Int64Value(count.HostedZoneCount))
+
+	_, err = client.UpdateHostedZoneComment(&r53.UpdateHostedZoneCommentInput{Id: aws.String(ids[0]), Comment: aws.String("moved")})
+	require.NoError(t, err)
+
+	_, err = client.ChangeTagsForResource(&r53.ChangeTagsForResourceInput{
+		ResourceType: aws.String("hostedzone"), ResourceId: aws.String(ids[0]),
+		AddTags: []*r53.Tag{{Key: aws.String("env"), Value: aws.String("prod")}},
+	})
+	require.NoError(t, err)
+	many, err := client.ListTagsForResources(&r53.ListTagsForResourcesInput{ResourceType: aws.String("hostedzone"), ResourceIds: aws.StringSlice(ids[:2])})
+	require.NoError(t, err)
+	require.Len(t, many.ResourceTagSets, 2)
+	assert.Equal(t, "env", aws.StringValue(many.ResourceTagSets[0].Tags[0].Key))
+}
+
+func TestUnservedRouteIsInvalidAction(t *testing.T) {
+	client, _ := newClient(t)
+	_, err := client.CreateHealthCheck(&r53.CreateHealthCheckInput{
+		CallerReference:   aws.String("hc"),
+		HealthCheckConfig: &r53.HealthCheckConfig{Type: aws.String("HTTP"), FullyQualifiedDomainName: aws.String("example.com")},
+	})
+	requireAWSCode(t, err, "InvalidAction")
+}
+
+func TestOperationInventoryMatchesTheServiceMethods(t *testing.T) {
+	inv := awsapi.OperationInventory()
+	svc := reflect.TypeFor[*route53.Service]()
+	for _, action := range append(inv.Registered, awsapi.ExtensionActions()...) {
+		_, ok := svc.MethodByName(action)
+		assert.True(t, ok, "%s is routed but the service does not implement it", action)
+	}
+	reg := awsapi.NewRegistration(awsapi.Deps{})
+	assert.Equal(t, dispatch.ErrorEnvelopeXML, reg.Errors)
+	assert.Equal(t, "route53", reg.Service)
+}
+
+func TestResolveActionMatchesEveryRoute(t *testing.T) {
+	reg := awsapi.NewRegistration(awsapi.Deps{})
+	cases := map[string]string{
+		"POST /2013-04-01/hostedzone":                    "CreateHostedZone",
+		"GET /2013-04-01/hostedzone":                     "ListHostedZones",
+		"GET /2013-04-01/hostedzonesbyname":              "ListHostedZonesByName",
+		"GET /2013-04-01/hostedzonecount":                "GetHostedZoneCount",
+		"GET /2013-04-01/hostedzone/Z1":                  "GetHostedZone",
+		"POST /2013-04-01/hostedzone/Z1":                 "UpdateHostedZoneComment",
+		"DELETE /2013-04-01/hostedzone/Z1":               "DeleteHostedZone",
+		"POST /2013-04-01/hostedzone/Z1/rrset/":          "ChangeResourceRecordSets",
+		"GET /2013-04-01/hostedzone/Z1/rrset":            "ListResourceRecordSets",
+		"GET /2013-04-01/change/C1":                      "GetChange",
+		"POST /2013-04-01/tags/hostedzone/Z1":            "ChangeTagsForResource",
+		"GET /2013-04-01/tags/hostedzone/Z1":             "ListTagsForResource",
+		"POST /2013-04-01/tags/hostedzone":               "ListTagsForResources",
+		"GET /2013-04-01/hostedzone/Z1/activation":       "GetHostedZoneActivation",
+		"POST /2013-04-01/hostedzone/Z1/activation":      "VerifyHostedZoneDelegation",
+		"POST /2013-04-01/healthcheck":                   "",
+		"GET /2013-04-01/hostedzone/Z1/rrset/extra/more": "",
+	}
+	for req, want := range cases {
+		method, path, _ := strings.Cut(req, " ")
+		r := httptest.NewRequest(method, path, nil)
+		assert.Equal(t, want, reg.ResolveAction(r), req)
+	}
+	assert.Len(t, awsapi.OperationInventory().Registered, 13, "extensions are not AWS model operations")
+}
+
+// The extensions are REST-XML like the rest of the API, so an SDK client can
+// call them as custom operations; this is how the console and spx do.
+func TestExtensionsRoundTripThroughTheSDK(t *testing.T) {
+	client, auth := newClient(t)
+	zone, err := client.CreateHostedZone(&r53.CreateHostedZoneInput{Name: aws.String("example.com"), CallerReference: aws.String("ext")})
+	require.NoError(t, err)
+	id := strings.TrimPrefix(aws.StringValue(zone.HostedZone.Id), "/hostedzone/")
+
+	out := &ext.GetHostedZoneActivationOutput{}
+	req := client.NewRequest(&request.Operation{Name: ext.ActionGetHostedZoneActivation, HTTPMethod: http.MethodGet, HTTPPath: "/2013-04-01/hostedzone/{Id}/activation"},
+		&ext.GetHostedZoneActivationInput{Id: aws.String(id)}, out)
+	require.NoError(t, req.Send())
+	assert.Equal(t, "/hostedzone/"+id, aws.StringValue(out.HostedZoneId))
+	require.NotNil(t, out.Activation)
+	assert.Equal(t, route53.StatePendingNameservers, aws.StringValue(out.Activation.State))
+	assert.Equal(t, []string{"ns1.spx3.net", "ns2.spx3.net"}, aws.StringValueSlice(out.Activation.NameServers))
+	assert.Contains(t, auth.calls, "route53:GetHostedZoneActivation arn:aws:route53:::hostedzone/"+id)
+
+	verify := client.NewRequest(&request.Operation{Name: ext.ActionVerifyHostedZoneDelegation, HTTPMethod: http.MethodPost, HTTPPath: "/2013-04-01/hostedzone/{Id}/activation"},
+		&ext.VerifyHostedZoneDelegationInput{Id: aws.String(id)}, &ext.VerifyHostedZoneDelegationOutput{})
+	requireAWSCode(t, verify.Send(), "ServiceUnavailable")
+
+	missing := client.NewRequest(&request.Operation{Name: ext.ActionGetHostedZoneActivation, HTTPMethod: http.MethodGet, HTTPPath: "/2013-04-01/hostedzone/{Id}/activation"},
+		&ext.GetHostedZoneActivationInput{Id: aws.String("ZNOPE")}, &ext.GetHostedZoneActivationOutput{})
+	requireAWSCode(t, missing.Send(), "NoSuchHostedZone")
+}

spinifex/domains/route53/changes.go

@@ -0,0 +1,278 @@
+package route53
+
+import (
+	"fmt"
+	"net"
+	"slices"
+	"strconv"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/route53"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// AWS default and hard quotas that bound a change batch and a zone.
+const (
+	maxRecordSetsPerZone       = 10_000
+	maxRecordsPerChangeRequest = 1_000
+	maxValueCharsPerChange     = 32_000
+	maxValuesPerRecordSet      = 400
+	maxTTL                     = 2_147_483_647
+	defaultNSTTL               = 172_800
+	defaultSOATTL              = 900
+)
+
+// supportedTypes are the record types a simple-routing record set may carry.
+var supportedTypes = []string{"A", "AAAA", "CAA", "CNAME", "MX", "NS", "PTR", "SOA", "SPF", "SRV", "TXT"}
+
+// applyChangeBatch applies batch to a copy of sets in order, as Route 53 does,
+// and returns the result only if every change is valid. Every problem found is
+// reported in one InvalidChangeBatch, so the caller sees them all at once.
+func applyChangeBatch(zone string, sets []RRSet, batch *route53.ChangeBatch) ([]RRSet, error) {
+	if batch == nil || len(batch.Changes) == 0 {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "ChangeBatch must contain at least one change")
+	}
+	if err := checkBatchSize(batch); err != nil {
+		return nil, err
+	}
+	out := slices.Clone(sets)
+	var problems []string
+	for _, ch := range batch.Changes {
+		if ch == nil || ch.ResourceRecordSet == nil {
+			return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "every change needs an Action and a ResourceRecordSet")
+		}
+		set, err := toRRSet(zone, ch.ResourceRecordSet)
+		if err != nil {
+			if code, _, ok := awserrors.ResolveErrorDetail(err); ok && code != awserrors.ErrorInvalidChangeBatch {
+				return nil, err
+			}
+			problems = append(problems, messageOf(err))
+			continue
+		}
+		var msg string
+		out, msg = applyOne(zone, out, aws.StringValue(ch.Action), set)
+		if msg != "" {
+			problems = append(problems, msg)
+		}
+	}
+	if len(problems) == 0 && len(out) > maxRecordSetsPerZone {
+		problems = append(problems, fmt.Sprintf("The hosted zone %s would exceed the limit of %d resource record sets", zone, maxRecordSetsPerZone))
+	}
+	if len(problems) > 0 {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidChangeBatch, "[%s]", strings.Join(problems, ", "))
+	}
+	return out, nil
+}
+
+func checkBatchSize(batch *route53.ChangeBatch) error {
+	records, chars := 0, 0
+	for _, ch := range batch.Changes {
+		if ch == nil || ch.ResourceRecordSet == nil {
+			continue
+		}
+		weight := 1
+		if aws.StringValue(ch.Action) == route53.ChangeActionUpsert {
+			weight = 2
+		}
+		for _, rr := range ch.ResourceRecordSet.ResourceRecords {
+			records += weight
+			chars += weight * len(aws.StringValue(rr.Value))
+		}
+	}
+	if records > maxRecordsPerChangeRequest {
+		return awserrors.Errorf(awserrors.ErrorInvalidChangeBatch, "[Number of records limit of %d exceeded.]", maxRecordsPerChangeRequest)
+	}
+	if chars > maxValueCharsPerChange {
+		return awserrors.Errorf(awserrors.ErrorInvalidChangeBatch, "[Number of characters limit of %d exceeded.]", maxValueCharsPerChange)
+	}
+	return nil
+}
+
+func applyOne(zone string, sets []RRSet, action string, set RRSet) ([]RRSet, string) {
+	at := slices.IndexFunc(sets, func(s RRSet) bool { return s.Name == set.Name && s.Type == set.Type })
+	label := fmt.Sprintf("[name='%s', type='%s']", set.Name, set.Type)
+	apexDefault := set.Name == zone && (set.Type == "SOA" || set.Type == "NS")
+
+	switch action {
+	case route53.ChangeActionCreate:
+		if at >= 0 {
+			return sets, "Tried to create resource record set " + label + " but it already exists"
+		}
+		if msg := conflicts(zone, sets, set); msg != "" {
+			return sets, msg
+		}
+		return append(sets, set), ""
+	case route53.ChangeActionUpsert:
+		if msg := conflicts(zone, sets, set); msg != "" {
+			return sets, msg
+		}
+		if at >= 0 {
+			sets[at] = set
+			return sets, ""
+		}
+		return append(sets, set), ""
+	case route53.ChangeActionDelete:
+		if at < 0 {
+			return sets, "Tried to delete resource record set " + label + " but it was not found"
+		}
+		if !sameRRSet(sets[at], set) {
+			return sets, "Tried to delete resource record set " + label + " but the values provided do not match the current values"
+		}
+		if apexDefault {
+			return sets, fmt.Sprintf("A HostedZone must contain at least one %s record for the zone itself.", set.Type)
+		}
+		return slices.Delete(sets, at, at+1), ""
+	default:
+		return sets, fmt.Sprintf("Invalid action %q for resource record set %s", action, label)
+	}
+}
+
+// conflicts applies the per-name rules: a CNAME stands alone at its name and
+// never at the apex, and SOA exists only at the apex.
+func conflicts(zone string, sets []RRSet, set RRSet) string {
+	if set.Type == "SOA" && set.Name != zone {
+		return fmt.Sprintf("RRSet of type SOA with DNS name %s is not permitted because SOA records are only permitted at the zone apex", set.Name)
+	}
+	if set.Type == "CNAME" && set.Name == zone {
+		return fmt.Sprintf("RRSet of type CNAME with DNS name %s is not permitted at apex in zone %s", set.Name, zone)
+	}
+	for _, s := range sets {
+		if s.Name != set.Name || s.Type == set.Type {
+			continue
+		}
+		if s.Type == "CNAME" || set.Type == "CNAME" {
+			return fmt.Sprintf("RRSet of type %s with DNS name %s is not permitted because a conflicting RRSet of type %s with the same DNS name already exists in zone %s", set.Type, set.Name, s.Type, zone)
+		}
+	}
+	return ""
+}
+
+func sameRRSet(a, b RRSet) bool {
+	return a.TTL == b.TTL && slices.Equal(sortedCopy(a.Values), sortedCopy(b.Values))
+}
+
+func sortedCopy(v []string) []string {
+	c := slices.Clone(v)
+	slices.Sort(c)
+	return c
+}
+
+// toRRSet validates one wire record set and converts it. Routing policies,
+// alias targets and health checks are refused with InvalidInput.
+func toRRSet(zone string, in *route53.ResourceRecordSet) (RRSet, error) {
+	if in.AliasTarget != nil {
+		return RRSet{}, awserrors.Errorf(awserrors.ErrorInvalidInput, "alias records are not supported")
+	}
+	if in.SetIdentifier != nil || in.Weight != nil || in.Region != nil || in.Failover != nil ||
+		in.GeoLocation != nil || in.GeoProximityLocation != nil || in.MultiValueAnswer != nil || in.CidrRoutingConfig != nil {
+		return RRSet{}, awserrors.Errorf(awserrors.ErrorInvalidInput, "only simple routing is supported; SetIdentifier and routing policies are not")
+	}
+	if in.HealthCheckId != nil || in.TrafficPolicyInstanceId != nil {
+		return RRSet{}, awserrors.Errorf(awserrors.ErrorInvalidInput, "health checks and traffic policies are not supported")
+	}
+	rrType := strings.ToUpper(aws.StringValue(in.Type))
+	if !slices.Contains(supportedTypes, rrType) {
+		return RRSet{}, awserrors.Errorf(awserrors.ErrorInvalidInput, "record type %q is not supported", rrType)
+	}
+	name := normalizeName(aws.StringValue(in.Name))
+	if err := validateRecordName(name, zone); err != nil {
+		return RRSet{}, awserrors.Errorf(awserrors.ErrorInvalidChangeBatch, "%s", messageOf(err))
+	}
+	if in.TTL == nil {
+		return RRSet{}, awserrors.Errorf(awserrors.ErrorInvalidChangeBatch, "Invalid request: Expected exactly one of [AliasTarget, all of [TTL, and ResourceRecords]], but found none in Change with [Name=%s, Type=%s, SetIdentifier=null]", name, rrType)
+	}
+	ttl := aws.Int64Value(in.TTL)
+	if ttl < 0 || ttl > maxTTL {
+		return RRSet{}, awserrors.Errorf(awserrors.ErrorInvalidInput, "TTL %d is out of range", ttl)
+	}
+	if len(in.ResourceRecords) == 0 {
+		return RRSet{}, awserrors.Errorf(awserrors.ErrorInvalidInput, "ResourceRecords must contain at least one record")
+	}
+	if len(in.ResourceRecords) > maxValuesPerRecordSet {
+		return RRSet{}, awserrors.Errorf(awserrors.ErrorInvalidChangeBatch, "RRSet %s %s has more than %d values", name, rrType, maxValuesPerRecordSet)
+	}
+	set := RRSet{Name: name, Type: rrType, TTL: ttl}
+	for _, rr := range in.ResourceRecords {
+		v := strings.TrimSpace(aws.StringValue(rr.Value))
+		if err := validateValue(rrType, v); err != nil {
+			return RRSet{}, awserrors.Errorf(awserrors.ErrorInvalidChangeBatch, "Invalid Resource Record: %s for %s %s", err, name, rrType)
+		}
+		set.Values = append(set.Values, v)
+	}
+	if (rrType == "CNAME" || rrType == "SOA") && len(set.Values) > 1 {
+		return RRSet{}, awserrors.Errorf(awserrors.ErrorInvalidChangeBatch, "RRSet of type %s with DNS name %s has more than one value", rrType, name)
+	}
+	return set, nil
+}
+
+// validateValue checks a record's RDATA in its presentation format.
+func validateValue(rrType, v string) error {
+	if v == "" {
+		return fmt.Errorf("empty value")
+	}
+	f := strings.Fields(v)
+	switch rrType {
+	case "A":
+		if ip := net.ParseIP(v); ip == nil || ip.To4() == nil || strings.Contains(v, ":") {
+			return fmt.Errorf("value %q is not a valid IPv4 address", v)
+		}
+	case "AAAA":
+		if ip := net.ParseIP(v); ip == nil || !strings.Contains(v, ":") {
+			return fmt.Errorf("value %q is not a valid IPv6 address", v)
+		}
+	case "CNAME", "NS", "PTR":
+		return validHostValue(v)
+	case "MX":
+		if len(f) != 2 || !validUint16(f[0]) {
+			return fmt.Errorf("value %q is not \"priority host\"", v)
+		}
+		return validHostValue(f[1])
+	case "SRV":
+		if len(f) != 4 || !validUint16(f[0]) || !validUint16(f[1]) || !validUint16(f[2]) {
+			return fmt.Errorf("value %q is not \"priority weight port target\"", v)
+		}
+		return validHostValue(f[3])
+	case "CAA":
+		if len(f) < 3 || !validUint8(f[0]) {
+			return fmt.Errorf("value %q is not \"flags tag value\"", v)
+		}
+	case "SOA":
+		if len(f) != 7 {
+			return fmt.Errorf("value %q is not a seven-field SOA", v)
+		}
+	case "TXT", "SPF":
+		if _, err := characterStrings(v); err != nil {
+			return fmt.Errorf("FATAL problem: %w encountered with '%s'", err, v)
+		}
+	}
+	return nil
+}
+
+func validHostValue(v string) error {
+	if v == "." {
+		return nil
+	}
+	if _, err := splitLabels(normalizeName(v), true); err != nil {
+		return fmt.Errorf("value %q is not a valid host name", v)
+	}
+	return nil
+}
+
+func validUint16(s string) bool {
+	_, err := strconv.ParseUint(s, 10, 16)
+	return err == nil
+}
+
+func validUint8(s string) bool {
+	_, err := strconv.ParseUint(s, 10, 8)
+	return err == nil
+}
+
+// messageOf returns the client message an awserrors error carries.
+func messageOf(err error) string {
+	if _, msg, ok := awserrors.ResolveErrorDetail(err); ok && msg != "" {
+		return msg
+	}
+	return err.Error()
+}

spinifex/domains/route53/ext/ext.go

@@ -0,0 +1,57 @@
+// Package ext holds the wire types of the Spinifex extensions to the Route 53
+// API: the activation a hosted zone needs before it is served, which AWS has
+// no equivalent of. Standard SDKs never call them.
+package ext
+
+import "time"
+
+// Actions served on the route53 signing service beside the AWS model's.
+const (
+	ActionGetHostedZoneActivation    = "GetHostedZoneActivation"
+	ActionVerifyHostedZoneDelegation = "VerifyHostedZoneDelegation"
+)
+
+// GetHostedZoneActivationInput names the zone whose activation to report.
+type GetHostedZoneActivationInput struct {
+	_  struct{} `locationName:"GetHostedZoneActivationRequest" type:"structure"`
+	Id *string  `location:"uri" locationName:"Id" type:"string" required:"true"`
+}
+
+// VerifyHostedZoneDelegationInput names the zone to check now.
+type VerifyHostedZoneDelegationInput struct {
+	_  struct{} `locationName:"VerifyHostedZoneDelegationRequest" type:"structure"`
+	Id *string  `location:"uri" locationName:"Id" type:"string" required:"true"`
+}
+
+// Activation is whether northstar serves a zone, and what the last check of
+// its delegation found.
+type Activation struct {
+	_ struct{} `type:"structure"`
+
+	// State is PENDING_NAMESERVERS, ACTIVE, NAMESERVERS_LOST or SUSPENDED.
+	State  *string `type:"string"`
+	Reason *string `type:"string"`
+	// NameServers are the servers the parent must delegate the zone to.
+	NameServers []*string  `locationNameList:"NameServer" type:"list"`
+	CheckedAt   *time.Time `type:"timestamp"`
+	// DelegatedTo is the NS set the parent published at the last check, and
+	// DelegationSource the parent zone or server that answered.
+	DelegatedTo      []*string `locationNameList:"NameServer" type:"list"`
+	DelegationSource *string   `type:"string"`
+	// Override is set when an operator activated or suspended the zone.
+	Override *string `type:"string"`
+}
+
+// GetHostedZoneActivationOutput reports a zone's activation.
+type GetHostedZoneActivationOutput struct {
+	_            struct{}    `type:"structure"`
+	HostedZoneId *string     `type:"string"`
+	Activation   *Activation `type:"structure"`
+}
+
+// VerifyHostedZoneDelegationOutput reports the activation the check left.
+type VerifyHostedZoneDelegationOutput struct {
+	_            struct{}    `type:"structure"`
+	HostedZoneId *string     `type:"string"`
+	Activation   *Activation `type:"structure"`
+}

spinifex/domains/route53/ids.go

@@ -0,0 +1,46 @@
+package route53
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/hex"
+	"strings"
+)
+
+const (
+	hostedZonePrefix = "/hostedzone/"
+	changePrefix     = "/change/"
+	idAlphabet       = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+	idRandomLength   = 20
+)
+
+// newZoneID returns an ID in the shape AWS issues: Z and 20 uppercase
+// alphanumerics.
+func newZoneID() string { return newID('Z') }
+
+// newChangeID returns a change ID: C and 20 uppercase alphanumerics.
+func newChangeID() string { return newID('C') }
+
+func newID(prefix byte) string {
+	buf := make([]byte, idRandomLength)
+	_, _ = rand.Read(buf)
+	out := make([]byte, 0, idRandomLength+1)
+	out = append(out, prefix)
+	for _, b := range buf {
+		out = append(out, idAlphabet[int(b)%len(idAlphabet)])
+	}
+	return string(out)
+}
+
+// bareZoneID strips the "/hostedzone/" prefix, which AWS accepts on input and
+// returns on output.
+func bareZoneID(id string) string { return strings.TrimPrefix(strings.TrimSpace(id), hostedZonePrefix) }
+
+func bareChangeID(id string) string { return strings.TrimPrefix(strings.TrimSpace(id), changePrefix) }
+
+// refDigest keys a caller reference, which may hold characters a KV key
+// cannot.
+func refDigest(ref string) string {
+	sum := sha256.Sum256([]byte(ref))
+	return hex.EncodeToString(sum[:16])
+}

spinifex/domains/route53/names.go

@@ -0,0 +1,97 @@
+package route53
+
+import (
+	"slices"
+	"strings"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+// Route 53 accepts names up to 1024 characters with escapes; the decoded DNS
+// limits below are the ones that matter to a resolver.
+const (
+	maxDomainLength = 253
+	maxLabelLength  = 63
+)
+
+// normalizeName lowercases a DNS name and gives it a trailing dot, the form
+// Route 53 returns names in. An escaped "\052" leading label is "*".
+func normalizeName(name string) string {
+	n := strings.ToLower(strings.TrimSpace(name))
+	if strings.HasPrefix(n, `\052.`) {
+		n = "*" + n[len(`\052`):]
+	}
+	if n != "" && !strings.HasSuffix(n, ".") {
+		n += "."
+	}
+	return n
+}
+
+// validateZoneName checks a normalized hosted-zone name. A zone needs at least
+// two labels: a bare TLD is not a domain a customer can delegate to us.
+func validateZoneName(name string) error {
+	labels, err := splitLabels(name, false)
+	if err != nil {
+		return err
+	}
+	if len(labels) < 2 {
+		return awserrors.Errorf(awserrors.ErrorInvalidDomainName, "%s is a top-level domain and cannot be a hosted zone", name)
+	}
+	return nil
+}
+
+// validateRecordName checks a normalized record name and that it lies within
+// zone. A wildcard is allowed only as the whole leftmost label.
+func validateRecordName(name, zone string) error {
+	if _, err := splitLabels(name, true); err != nil {
+		return err
+	}
+	if !withinZone(name, zone) {
+		return awserrors.Errorf(awserrors.ErrorInvalidChangeBatch, "RRSet with DNS name %s is not permitted in zone %s", name, zone)
+	}
+	return nil
+}
+
+func splitLabels(name string, allowWildcard bool) ([]string, error) {
+	trimmed := strings.TrimSuffix(name, ".")
+	if trimmed == "" || len(trimmed) > maxDomainLength {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidDomainName, "%q is not a valid domain name", name)
+	}
+	labels := strings.Split(trimmed, ".")
+	for i, l := range labels {
+		if l == "*" && allowWildcard && i == 0 {
+			continue
+		}
+		if !validLabel(l) {
+			return nil, awserrors.Errorf(awserrors.ErrorInvalidDomainName, "%q is not a valid domain name", name)
+		}
+	}
+	return labels, nil
+}
+
+// validLabel accepts letters, digits, hyphens and underscores, the last for
+// service labels such as _acme-challenge and _sip._tcp.
+func validLabel(l string) bool {
+	if l == "" || len(l) > maxLabelLength || l[0] == '-' || l[len(l)-1] == '-' {
+		return false
+	}
+	for _, c := range l {
+		if (c < 'a' || c > 'z') && (c < '0' || c > '9') && c != '-' && c != '_' {
+			return false
+		}
+	}
+	return true
+}
+
+// withinZone reports whether name is zone or a name below it.
+func withinZone(name, zone string) bool {
+	return name == zone || strings.HasSuffix(name, "."+zone)
+}
+
+// reversedName orders names the way Route 53 lists them: by label, from the
+// rightmost, so a zone's apex sorts before everything under it.
+func reversedName(name string) string {
+	labels := strings.Split(strings.TrimSuffix(name, "."), ".")
+	slices.Reverse(labels)
+	return strings.Join(labels, ".")
+}

spinifex/domains/route53/realizer.go

@@ -0,0 +1,255 @@
+package route53
+
+import (
+	"context"
+	"crypto/md5" //nolint:gosec // matches the S3 ETag northstar reports, not a security boundary
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"log/slog"
+	"math"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/reconciler"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+)
+
+// Publisher is where realized zones go: the northstar zone store and the
+// servers that load from it.
+type Publisher interface {
+	PutZone(ctx context.Context, apex string, body []byte) error
+	DeleteZone(ctx context.Context, apex string) error
+	// Reload asks every northstar server to load apex. served holds the version
+	// each answering server reports, "" for none; complete is whether every
+	// server answered.
+	Reload(ctx context.Context, apex string) (served []string, complete bool, err error)
+}
+
+const (
+	defaultRealizeResync = time.Minute
+	// realizeRetry is how soon a zone some server has not yet loaded is retried.
+	realizeRetry = 5 * time.Second
+)
+
+// Realizer publishes every ACTIVE zone to northstar, withdraws every zone that
+// stops being ACTIVE, and moves changes to INSYNC once every northstar server
+// serves them. One node realizes at a time, through Elect.
+type Realizer struct {
+	store *Store
+	pub   Publisher
+
+	// Elect wins this pass for the calling node; nil means always.
+	Elect func(ctx context.Context) (release func(), ok bool)
+	// Reserved reports platform zone names, which are never written or deleted.
+	Reserved func(name string) bool
+	// Now is the clock; tests replace it.
+	Now func() time.Time
+	// Resync is the drift backstop; zero means one minute.
+	Resync time.Duration
+}
+
+// NewRealizer returns a Realizer over store that publishes through pub.
+func NewRealizer(store *Store, pub Publisher) *Realizer {
+	return &Realizer{store: store, pub: pub, Now: time.Now}
+}
+
+// Run realizes once, then on every zone or record-set change.
+func (r *Realizer) Run(ctx context.Context) {
+	resync := r.Resync
+	if resync <= 0 {
+		resync = defaultRealizeResync
+	}
+	reconciler.Run(ctx, reconciler.Config{
+		Name: "route53",
+		Sources: []reconciler.Source{
+			reconciler.Fixed(r.store.bucket, keyPrefixZone+">"),
+			reconciler.Fixed(r.store.bucket, keyPrefixRRSets+">"),
+		},
+		Reconcile: r.Reconcile,
+		Resync:    resync,
+	})
+}
+
+// Reconcile is one whole-set pass. A zone a server has not yet loaded asks for
+// a prompt revisit, since nothing in KV changes when it does.
+func (r *Realizer) Reconcile(ctx context.Context) (time.Duration, error) {
+	if r.Elect != nil {
+		release, ok := r.Elect(ctx)
+		if !ok {
+			return 0, nil
+		}
+		defer release()
+	}
+	zones, err := r.store.ListAllZones(ctx)
+	if err != nil {
+		return 0, fmt.Errorf("list hosted zones: %w", err)
+	}
+	published, err := r.store.ListPublished(ctx)
+	if err != nil {
+		return 0, fmt.Errorf("list published zones: %w", err)
+	}
+
+	var revisit time.Duration
+	var errs []error
+	served := map[string]bool{}
+	for apex, zone := range r.servedZones(zones) {
+		doc, _, err := r.store.GetRRSets(ctx, zone.AccountID, zone.ID)
+		if errors.Is(err, kvstore.ErrNotFound) {
+			continue // being deleted; withdrawn below
+		}
+		if err != nil {
+			errs = append(errs, fmt.Errorf("read record sets for %s: %w", apex, err))
+			served[apex] = true // unknown, so leave what is published alone
+			continue
+		}
+		served[apex] = true
+		prev, had := published[apex]
+		pending, err := r.realize(ctx, apex, zone, doc, prev, had)
+		if err != nil {
+			errs = append(errs, err)
+		}
+		if pending || err != nil {
+			revisit = realizeRetry
+		}
+	}
+	for apex, p := range published {
+		if served[apex] {
+			continue
+		}
+		pending, err := r.withdraw(ctx, p)
+		if err != nil {
+			errs = append(errs, err)
+		}
+		if pending || err != nil {
+			revisit = realizeRetry
+		}
+	}
+	if err := r.pruneOrphanChanges(ctx, zones); err != nil {
+		errs = append(errs, err)
+	}
+	return revisit, errors.Join(errs...)
+}
+
+// servedZones picks, for each zone name, the ACTIVE zone northstar serves. Only
+// one should be ACTIVE per name; if more are, the oldest is served, so the
+// answer does not flip between them.
+func (r *Realizer) servedZones(zones []ZoneRecord) map[string]*ZoneRecord {
+	out := map[string]*ZoneRecord{}
+	for i := range zones {
+		z := &zones[i]
+		if z.State != StateActive || (r.Reserved != nil && r.Reserved(z.Name)) {
+			continue
+		}
+		apex := strings.TrimSuffix(z.Name, ".")
+		if cur, ok := out[apex]; ok && (cur.CreatedAt.Before(z.CreatedAt) || (cur.CreatedAt.Equal(z.CreatedAt) && cur.ID < z.ID)) {
+			continue
+		}
+		out[apex] = z
+	}
+	return out
+}
+
+// realize writes the zone if northstar does not already hold this version,
+// then reloads until every server serves it.
+func (r *Realizer) realize(ctx context.Context, apex string, zone *ZoneRecord, doc *RRSetsDoc, prev PublishedZone, had bool) (pending bool, err error) {
+	body, err := renderZone(zone, doc)
+	if err != nil {
+		return false, fmt.Errorf("render %s: %w", apex, err)
+	}
+	sum := md5.Sum(body) //nolint:gosec // see import
+	digest := hex.EncodeToString(sum[:])
+	if had && prev.ZoneID == zone.ID && prev.Digest == digest && prev.InSync {
+		return false, nil
+	}
+	next := PublishedZone{Apex: apex, AccountID: zone.AccountID, ZoneID: zone.ID, Version: doc.Version, Digest: digest}
+	if !had || prev.Digest != digest || prev.ZoneID != zone.ID {
+		if err := r.pub.PutZone(ctx, apex, body); err != nil {
+			return false, fmt.Errorf("write zone %s: %w", apex, err)
+		}
+		if err := r.store.SetPublished(ctx, &next); err != nil {
+			return false, fmt.Errorf("record published zone %s: %w", apex, err)
+		}
+		slog.Info("route53: published zone", "zone", apex, "zone_id", zone.ID, "version", doc.Version)
+	}
+	if !r.reloaded(ctx, apex, digest) {
+		return true, nil
+	}
+	if err := r.store.MarkChangesInSync(ctx, zone.AccountID, zone.ID, doc.Version); err != nil {
+		return false, fmt.Errorf("mark changes in sync for %s: %w", apex, err)
+	}
+	if had && prev.ZoneID != zone.ID {
+		// The name moved to another zone; the one it left is no longer served.
+		if err := r.store.MarkChangesInSync(ctx, prev.AccountID, prev.ZoneID, math.MaxUint64); err != nil {
+			return false, fmt.Errorf("mark changes in sync for %s: %w", apex, err)
+		}
+	}
+	next.InSync = true
+	if err := r.store.SetPublished(ctx, &next); err != nil {
+		return false, fmt.Errorf("record published zone %s: %w", apex, err)
+	}
+	return false, nil
+}
+
+// withdraw deletes a zone northstar should no longer serve.
+func (r *Realizer) withdraw(ctx context.Context, p PublishedZone) (pending bool, err error) {
+	if r.Reserved != nil && r.Reserved(p.Apex+".") {
+		slog.Error("route53: refusing to withdraw a platform zone", "zone", p.Apex)
+		return false, nil
+	}
+	if err := r.pub.DeleteZone(ctx, p.Apex); err != nil {
+		return false, fmt.Errorf("delete zone %s: %w", p.Apex, err)
+	}
+	if !r.reloaded(ctx, p.Apex, "") {
+		return true, nil
+	}
+	if err := r.store.MarkChangesInSync(ctx, p.AccountID, p.ZoneID, math.MaxUint64); err != nil {
+		return false, fmt.Errorf("mark changes in sync for %s: %w", p.Apex, err)
+	}
+	if err := r.store.DeletePublished(ctx, p.Apex); err != nil {
+		return false, fmt.Errorf("forget withdrawn zone %s: %w", p.Apex, err)
+	}
+	slog.Info("route53: withdrew zone", "zone", p.Apex, "zone_id", p.ZoneID)
+	return false, nil
+}
+
+// reloaded reports whether every northstar server now serves digest.
+func (r *Realizer) reloaded(ctx context.Context, apex, digest string) bool {
+	served, complete, err := r.pub.Reload(ctx, apex)
+	if err != nil {
+		slog.Warn("route53: zone reload failed", "zone", apex, "error", err)
+		return false
+	}
+	if !complete {
+		return false
+	}
+	for _, v := range served {
+		if v != digest {
+			return false
+		}
+	}
+	return true
+}
+
+// pruneOrphanChanges deletes the INSYNC changes of deleted zones once they are
+// past the retention GetChange honours.
+func (r *Realizer) pruneOrphanChanges(ctx context.Context, zones []ZoneRecord) error {
+	live := make(map[string]bool, len(zones))
+	for _, z := range zones {
+		live[z.AccountID+"/"+z.ID] = true
+	}
+	changes, err := r.store.ListAllChanges(ctx)
+	if err != nil {
+		return fmt.Errorf("list changes: %w", err)
+	}
+	cutoff := r.Now().Add(-changeRetention)
+	for _, c := range changes {
+		if live[c.AccountID+"/"+c.ZoneID] || c.Status != ChangeStatusInSync || !c.SubmittedAt.Before(cutoff) {
+			continue
+		}
+		if err := r.store.DeleteChange(ctx, c.AccountID, c.ID); err != nil {
+			return fmt.Errorf("delete change %s: %w", c.ID, err)
+		}
+	}
+	return nil
+}

spinifex/domains/route53/realizer_test.go

@@ -0,0 +1,318 @@
+package route53_test
+
+import (
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	r53 "github.com/aws/aws-sdk-go/service/route53"
+	nsconfig "github.com/mulgadc/northstar/pkg/config"
+	toml "github.com/pelletier/go-toml/v2"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/route53"
+)
+
+// fakeNorthstar is the zone store plus a set of servers, each of which loads
+// what the store holds only when it is not lagging.
+type fakeNorthstar struct {
+	mu      sync.Mutex
+	objects map[string][]byte
+	served  []map[string]string
+	lagging map[int]bool
+	puts    int
+	deletes int
+}
+
+func newFakeNorthstar(servers int) *fakeNorthstar {
+	f := &fakeNorthstar{objects: map[string][]byte{}, lagging: map[int]bool{}}
+	for range servers {
+		f.served = append(f.served, map[string]string{})
+	}
+	return f
+}
+
+func (f *fakeNorthstar) PutZone(_ context.Context, apex string, body []byte) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	f.objects[apex] = body
+	f.puts++
+	return nil
+}
+
+func (f *fakeNorthstar) DeleteZone(_ context.Context, apex string) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	delete(f.objects, apex)
+	f.deletes++
+	return nil
+}
+
+func (f *fakeNorthstar) Reload(_ context.Context, apex string) ([]string, bool, error) {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	var out []string
+	for i, s := range f.served {
+		if f.lagging[i] {
+			continue
+		}
+		if body, ok := f.objects[apex]; ok {
+			sum := md5.Sum(body)
+			s[apex] = hex.EncodeToString(sum[:])
+		} else {
+			delete(s, apex)
+		}
+		out = append(out, s[apex])
+	}
+	return out, len(out) == len(f.served), nil
+}
+
+func (f *fakeNorthstar) zone(t *testing.T, apex string) nsconfig.ConfigArr {
+	t.Helper()
+	f.mu.Lock()
+	body, ok := f.objects[apex]
+	f.mu.Unlock()
+	require.True(t, ok, "%s is not published", apex)
+	var cfg nsconfig.ConfigArr
+	require.NoError(t, toml.Unmarshal(body, &cfg))
+	return cfg
+}
+
+func (f *fakeNorthstar) has(apex string) bool {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	_, ok := f.objects[apex]
+	return ok
+}
+
+func setState(t *testing.T, svc *route53.Service, account, zoneID, state string) {
+	t.Helper()
+	require.NoError(t, svc.Store().MutateZone(ctx, account, strings.TrimPrefix(zoneID, "/hostedzone/"), func(z *route53.ZoneRecord) (bool, error) {
+		z.State = state
+		return true, nil
+	}))
+}
+
+func changeStatus(t *testing.T, svc *route53.Service, account, changeID string) string {
+	t.Helper()
+	out, err := svc.GetChange(ctx, &r53.GetChangeInput{Id: aws.String(changeID)}, account)
+	require.NoError(t, err)
+	return aws.StringValue(out.ChangeInfo.Status)
+}
+
+func newRealizer(svc *route53.Service, ns *fakeNorthstar) *route53.Realizer {
+	r := route53.NewRealizer(svc.Store(), ns)
+	r.Reserved = svc.Reserved
+	return r
+}
+
+func TestRealizerPublishesOnlyActiveZones(t *testing.T) {
+	svc := newService(t)
+	ns := newFakeNorthstar(2)
+	r := newRealizer(svc, ns)
+	id := createZone(t, svc, acctA, "example.com")
+
+	_, err := r.Reconcile(ctx)
+	require.NoError(t, err)
+	assert.False(t, ns.has("example.com"), "a zone pending its nameservers must not be served")
+
+	setState(t, svc, acctA, id, route53.StateActive)
+	out, err := apply(svc, acctA, id, change("CREATE", "www.example.com", "A", 60, "192.0.2.1"))
+	require.NoError(t, err)
+	assert.Equal(t, "PENDING", aws.StringValue(out.ChangeInfo.Status))
+
+	revisit, err := r.Reconcile(ctx)
+	require.NoError(t, err)
+	assert.Zero(t, revisit)
+	zone := ns.zone(t, "example.com")
+	assert.Equal(t, "example.com", zone.Domain.Domain)
+	assert.Equal(t, "ns1.spx3.net.", zone.Domain.SOA)
+	assert.Contains(t, zone.Records, nsconfig.Records{Domain: "www.", Type: 1, Class: 1, TTL: 60, Address: "192.0.2.1"})
+	assert.Contains(t, zone.Records, nsconfig.Records{Domain: "", Type: 2, Class: 1, TTL: 172800, Address: "ns1.spx3.net."})
+	assert.Equal(t, "INSYNC", changeStatus(t, svc, acctA, aws.StringValue(out.ChangeInfo.Id)))
+
+	puts := ns.puts
+	_, err = r.Reconcile(ctx)
+	require.NoError(t, err)
+	assert.Equal(t, puts, ns.puts, "an unchanged zone is not rewritten")
+}
+
+// A change is INSYNC only once every northstar server serves it, so a server
+// that has not loaded the new version holds it PENDING and asks for a retry.
+func TestRealizerHoldsChangesPendingUntilEveryServerLoads(t *testing.T) {
+	svc := newService(t)
+	ns := newFakeNorthstar(3)
+	r := newRealizer(svc, ns)
+	id := createZone(t, svc, acctA, "example.com")
+	setState(t, svc, acctA, id, route53.StateActive)
+	out, err := apply(svc, acctA, id, change("CREATE", "www.example.com", "A", 60, "192.0.2.1"))
+	require.NoError(t, err)
+
+	ns.lagging[2] = true
+	revisit, err := r.Reconcile(ctx)
+	require.NoError(t, err)
+	assert.Positive(t, revisit)
+	assert.Equal(t, "PENDING", changeStatus(t, svc, acctA, aws.StringValue(out.ChangeInfo.Id)))
+
+	ns.lagging[2] = false
+	puts := ns.puts
+	revisit, err = r.Reconcile(ctx)
+	require.NoError(t, err)
+	assert.Zero(t, revisit)
+	assert.Equal(t, puts, ns.puts, "the retry reloads without rewriting")
+	assert.Equal(t, "INSYNC", changeStatus(t, svc, acctA, aws.StringValue(out.ChangeInfo.Id)))
+}
+
+func TestRealizerWithdrawsDeletedAndDeactivatedZones(t *testing.T) {
+	svc := newService(t)
+	ns := newFakeNorthstar(2)
+	r := newRealizer(svc, ns)
+	gone := createZone(t, svc, acctA, "gone.com")
+	lost := createZone(t, svc, acctA, "lost.com")
+	setState(t, svc, acctA, gone, route53.StateActive)
+	setState(t, svc, acctA, lost, route53.StateActive)
+	_, err := r.Reconcile(ctx)
+	require.NoError(t, err)
+	require.True(t, ns.has("gone.com"))
+	require.True(t, ns.has("lost.com"))
+
+	del, err := svc.DeleteHostedZone(ctx, &r53.DeleteHostedZoneInput{Id: aws.String(gone)}, acctA)
+	require.NoError(t, err)
+	assert.Equal(t, "PENDING", aws.StringValue(del.ChangeInfo.Status))
+	setState(t, svc, acctA, lost, route53.StateNameserversLost)
+
+	_, err = r.Reconcile(ctx)
+	require.NoError(t, err)
+	assert.False(t, ns.has("gone.com"))
+	assert.False(t, ns.has("lost.com"))
+	assert.Equal(t, "INSYNC", changeStatus(t, svc, acctA, aws.StringValue(del.ChangeInfo.Id)))
+	published, err := svc.Store().ListPublished(ctx)
+	require.NoError(t, err)
+	assert.Empty(t, published)
+}
+
+func TestRealizerNeverWritesAReservedZone(t *testing.T) {
+	svc := newService(t)
+	ns := newFakeNorthstar(1)
+	r := newRealizer(svc, ns)
+	id := createZone(t, svc, acctA, "example.com")
+	setState(t, svc, acctA, id, route53.StateActive)
+	r.Reserved = func(name string) bool { return name == "example.com." }
+
+	_, err := r.Reconcile(ctx)
+	require.NoError(t, err)
+	assert.False(t, ns.has("example.com"))
+}
+
+// Two accounts may hold zones of the same name; if both are ACTIVE, the older
+// one is served and the answer does not flip between them.
+func TestRealizerServesTheOldestOfTwoActiveZones(t *testing.T) {
+	svc := newService(t)
+	clock := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
+	svc.Now = func() time.Time { clock = clock.Add(time.Second); return clock }
+	ns := newFakeNorthstar(1)
+	r := newRealizer(svc, ns)
+	first := createZone(t, svc, acctA, "example.com")
+	second := createZone(t, svc, acctB, "example.com")
+	_, err := apply(svc, acctB, second, change("CREATE", "www.example.com", "A", 60, "198.51.100.1"))
+	require.NoError(t, err)
+	setState(t, svc, acctB, second, route53.StateActive)
+	setState(t, svc, acctA, first, route53.StateActive)
+
+	for range 2 {
+		_, err = r.Reconcile(ctx)
+		require.NoError(t, err)
+		for _, rec := range ns.zone(t, "example.com").Records {
+			assert.NotEqual(t, "198.51.100.1", rec.Address, "the newer zone must not be served")
+		}
+	}
+}
+
+func TestRealizerDoesNothingWhenNotElected(t *testing.T) {
+	svc := newService(t)
+	ns := newFakeNorthstar(1)
+	r := newRealizer(svc, ns)
+	r.Elect = func(context.Context) (func(), bool) { return nil, false }
+	id := createZone(t, svc, acctA, "example.com")
+	setState(t, svc, acctA, id, route53.StateActive)
+
+	_, err := r.Reconcile(ctx)
+	require.NoError(t, err)
+	assert.False(t, ns.has("example.com"))
+}
+
+func TestRealizerRendersEveryRecordType(t *testing.T) {
+	svc := newService(t)
+	ns := newFakeNorthstar(1)
+	r := newRealizer(svc, ns)
+	id := createZone(t, svc, acctA, "example.com")
+	setState(t, svc, acctA, id, route53.StateActive)
+	long := strings.Repeat("k", 255)
+	_, err := apply(svc, acctA, id,
+		change("CREATE", "example.com", "MX", 300, "10 mail.example.com."),
+		change("CREATE", "_xmpp._tcp.example.com", "SRV", 300, "1 10 5269 xmpp.example.com."),
+		change("CREATE", "example.com", "CAA", 300, `0 issue "letsencrypt.org"`),
+		change("CREATE", "dkim.example.com", "TXT", 300, `"`+long+`" "tail"`, `"say \"hi\""`),
+		change("CREATE", "*.example.com", "CNAME", 300, "www.example.com."),
+		change("CREATE", "v6.example.com", "AAAA", 300, "2001:db8::1"),
+	)
+	require.NoError(t, err)
+
+	_, err = r.Reconcile(ctx)
+	require.NoError(t, err)
+	records := ns.zone(t, "example.com").Records
+	assert.Contains(t, records, nsconfig.Records{Domain: "", Type: 15, Class: 1, TTL: 300, Preference: 10, Address: "mail.example.com."})
+	assert.Contains(t, records, nsconfig.Records{Domain: "_xmpp._tcp.", Type: 33, Class: 1, TTL: 300, Priority: 1, Weight: 10, Port: 5269, Address: "xmpp.example.com."})
+	assert.Contains(t, records, nsconfig.Records{Domain: "", Type: 257, Class: 1, TTL: 300, CAATag: "issue", Address: "letsencrypt.org"})
+	assert.Contains(t, records, nsconfig.Records{Domain: "dkim.", Type: 16, Class: 1, TTL: 300, Strings: []string{long, "tail"}})
+	assert.Contains(t, records, nsconfig.Records{Domain: "dkim.", Type: 16, Class: 1, TTL: 300, Strings: []string{`say \"hi\"`}})
+	assert.Contains(t, records, nsconfig.Records{Domain: "*.", Type: 5, Class: 1, TTL: 300, Address: "www.example.com."})
+	assert.Contains(t, records, nsconfig.Records{Domain: "v6.", Type: 28, Class: 1, TTL: 300, Address: "2001:db8::1"})
+}
+
+func TestTXTValuesAreValidatedAsCharacterStrings(t *testing.T) {
+	svc := newService(t)
+	id := createZone(t, svc, acctA, "example.com")
+	for _, v := range []string{
+		`"` + strings.Repeat("a", 256) + `"`,
+		`"unterminated`,
+		`"ok" trailing`,
+		`"bad escape \999"`,
+		strings.Repeat("b", 256),
+	} {
+		_, err := apply(svc, acctA, id, change("UPSERT", "t.example.com", "TXT", 60, v))
+		requireCode(t, err, "InvalidChangeBatch")
+	}
+	for _, v := range []string{`"` + strings.Repeat("a", 255) + `"`, `"\255\"x"`, "unquoted", `"a" "b"`} {
+		_, err := apply(svc, acctA, id, change("UPSERT", "t.example.com", "TXT", 60, v))
+		require.NoError(t, err, v)
+	}
+}
+
+func TestRealizerPrunesTheChangesOfDeletedZones(t *testing.T) {
+	svc := newService(t)
+	clock := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
+	svc.Now = func() time.Time { return clock }
+	r := newRealizer(svc, newFakeNorthstar(1))
+	r.Now = func() time.Time { return clock }
+	id := createZone(t, svc, acctA, "example.com")
+	del, err := svc.DeleteHostedZone(ctx, &r53.DeleteHostedZoneInput{Id: aws.String(id)}, acctA)
+	require.NoError(t, err)
+	changeID := aws.StringValue(del.ChangeInfo.Id)
+
+	_, err = r.Reconcile(ctx)
+	require.NoError(t, err)
+	assert.Equal(t, "INSYNC", changeStatus(t, svc, acctA, changeID), "kept while GetChange may still ask")
+
+	clock = clock.Add(8 * 24 * time.Hour)
+	_, err = r.Reconcile(ctx)
+	require.NoError(t, err)
+	_, err = svc.GetChange(ctx, &r53.GetChangeInput{Id: aws.String(changeID)}, acctA)
+	requireCode(t, err, "NoSuchChange")
+}

spinifex/domains/route53/render.go

@@ -0,0 +1,110 @@
+package route53
+
+import (
+	"cmp"
+	"fmt"
+	"slices"
+	"strconv"
+	"strings"
+
+	"github.com/miekg/dns"
+	nsconfig "github.com/mulgadc/northstar/pkg/config"
+)
+
+// renderZone turns a zone's record sets into the northstar zone document. The
+// output depends only on its inputs, so the same version always renders to
+// the same bytes and its digest identifies what northstar should be serving.
+//
+// Northstar synthesizes the SOA from the first field of the stored SOA value,
+// so the record set itself is not emitted.
+func renderZone(zone *ZoneRecord, doc *RRSetsDoc) ([]byte, error) {
+	apex := strings.TrimSuffix(zone.Name, ".")
+	cfg := nsconfig.ConfigArr{
+		Version: 1.0,
+		Domain: nsconfig.Domain{
+			Domain:   apex,
+			Modified: doc.UpdatedAt.UTC(),
+			Active:   true,
+		},
+		Defaults: nsconfig.Defaults{TTL: 300, Type: dns.TypeA, Class: dns.ClassINET},
+	}
+	sets := slices.Clone(doc.Sets)
+	slices.SortFunc(sets, func(a, b RRSet) int {
+		return cmp.Or(cmp.Compare(reversedName(a.Name), reversedName(b.Name)), cmp.Compare(a.Type, b.Type))
+	})
+	for _, set := range sets {
+		if set.Type == "SOA" {
+			if f := strings.Fields(set.Values[0]); len(f) > 0 {
+				cfg.Domain.SOA = f[0]
+			}
+			continue
+		}
+		rtype, ok := dns.StringToType[set.Type]
+		if !ok {
+			return nil, fmt.Errorf("record set %s has unknown type %s", set.Name, set.Type)
+		}
+		label := relativeLabel(set.Name, zone.Name)
+		for _, v := range set.Values {
+			rec, err := renderRecord(label, rtype, uint32(set.TTL), v) //nolint:gosec // TTL is validated to at most maxTTL
+			if err != nil {
+				return nil, fmt.Errorf("record set %s %s: %w", set.Name, set.Type, err)
+			}
+			cfg.Records = append(cfg.Records, rec)
+		}
+	}
+	return nsconfig.RenderZone(cfg)
+}
+
+func renderRecord(label string, rtype uint16, ttl uint32, v string) (nsconfig.Records, error) {
+	rec := nsconfig.Records{Domain: label, Type: rtype, Class: dns.ClassINET, TTL: ttl}
+	f := strings.Fields(v)
+	switch rtype {
+	case dns.TypeMX:
+		rec.Preference = parseUint16(f[0])
+		rec.Address = f[1]
+	case dns.TypeSRV:
+		rec.Priority, rec.Weight, rec.Port = parseUint16(f[0]), parseUint16(f[1]), parseUint16(f[2])
+		rec.Address = f[3]
+	case dns.TypeCAA:
+		flag, _ := strconv.ParseUint(f[0], 10, 8)
+		rec.CAAFlag = uint8(flag)
+		rec.CAATag = f[1]
+		rec.Address = strings.Trim(afterFields(v, 2), `"`)
+	case dns.TypeTXT, dns.TypeSPF:
+		strs, err := characterStrings(v)
+		if err != nil {
+			return rec, err
+		}
+		rec.Strings = strs
+	default:
+		rec.Address = v
+	}
+	return rec, nil
+}
+
+// afterFields returns v with its first n whitespace-separated fields removed.
+func afterFields(v string, n int) string {
+	rest := strings.TrimSpace(v)
+	for range n {
+		i := strings.IndexAny(rest, " \t")
+		if i < 0 {
+			return ""
+		}
+		rest = strings.TrimSpace(rest[i:])
+	}
+	return rest
+}
+
+func parseUint16(s string) uint16 {
+	n, _ := strconv.ParseUint(s, 10, 16)
+	return uint16(n)
+}
+
+// relativeLabel is name relative to zone in northstar's form: "" for the apex
+// and "www." for www.<zone>.
+func relativeLabel(name, zone string) string {
+	if name == zone {
+		return ""
+	}
+	return strings.TrimSuffix(name, zone)
+}

spinifex/domains/route53/service.go

@@ -0,0 +1,634 @@
+// Package route53 is the daemon-side Route 53 control plane: public hosted
+// zones and their record sets in JetStream KV, change tracking for GetChange,
+// and the activation state that decides whether northstar serves a zone.
+package route53
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/route53"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+)
+
+const (
+	defaultMaxZonesPerAccount = 500
+	maxCallerReferenceLength  = 128
+	maxCommentLength          = 256
+	maxListZones              = 100
+	maxListRecordSets         = 300
+	maxTagsPerResource        = 50
+	maxTagResources           = 10
+	changeRetention           = 7 * 24 * time.Hour
+	commitAttempts            = 3
+)
+
+// Service implements the Route 53 operations the gateway forwards over NATS.
+// Methods take the account from the request, never from the input.
+type Service struct {
+	store *Store
+
+	// Nameservers returns the published nameserver host names, without a
+	// trailing dot, that a new zone is delegated to.
+	Nameservers func() []string
+	// Reserved reports whether a zone name belongs to the platform and may not
+	// be created by a tenant.
+	Reserved func(name string) bool
+	// Now is the clock; tests replace it.
+	Now func() time.Time
+	// MaxZonesPerAccount caps CreateHostedZone; zero means the AWS default.
+	MaxZonesPerAccount int
+	// Verifier checks a delegation on demand; nil leaves zones to the schedule.
+	Verifier *Verifier
+}
+
+// NewService returns a Service over store.
+func NewService(store *Store) *Service {
+	return &Service{store: store, Now: time.Now}
+}
+
+// Store returns the service's store, for the realizer and verifier.
+func (s *Service) Store() *Store { return s.store }
+
+func (s *Service) CreateHostedZone(ctx context.Context, in *route53.CreateHostedZoneInput, account string) (*route53.CreateHostedZoneOutput, error) {
+	ref := aws.StringValue(in.CallerReference)
+	if ref == "" || len(ref) > maxCallerReferenceLength {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "CallerReference must be 1 to %d characters", maxCallerReferenceLength)
+	}
+	if in.VPC != nil || (in.HostedZoneConfig != nil && aws.BoolValue(in.HostedZoneConfig.PrivateZone)) {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "private hosted zones are not supported")
+	}
+	if in.DelegationSetId != nil {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "reusable delegation sets are not supported")
+	}
+	comment := ""
+	if in.HostedZoneConfig != nil {
+		comment = aws.StringValue(in.HostedZoneConfig.Comment)
+	}
+	if len(comment) > maxCommentLength {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "Comment must be at most %d characters", maxCommentLength)
+	}
+	name := normalizeName(aws.StringValue(in.Name))
+	if err := validateZoneName(name); err != nil {
+		return nil, err
+	}
+	if s.Reserved != nil && s.Reserved(name) {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidDomainName, "%s is reserved by this cluster and cannot be a hosted zone", name)
+	}
+	var nameservers []string
+	if s.Nameservers != nil {
+		nameservers = s.Nameservers()
+	}
+	if len(nameservers) == 0 {
+		return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "no nameservers are published for this cluster")
+	}
+
+	existing, err := s.store.ListAccountZones(ctx, account)
+	if err != nil {
+		return nil, internal("list hosted zones", err)
+	}
+	limit := s.MaxZonesPerAccount
+	if limit == 0 {
+		limit = defaultMaxZonesPerAccount
+	}
+	if len(existing) >= limit {
+		return nil, awserrors.Errorf(awserrors.ErrorTooManyHostedZones, "account %s already has %d hosted zones", account, len(existing))
+	}
+
+	id := newZoneID()
+	held, err := s.store.ClaimCallerReference(ctx, account, ref, id)
+	if err != nil {
+		return nil, internal("claim caller reference", err)
+	}
+	if held != "" {
+		if _, err := s.store.GetZone(ctx, account, held); !errors.Is(err, kvstore.ErrNotFound) {
+			return nil, awserrors.Errorf(awserrors.ErrorHostedZoneAlreadyExists, "A hosted zone has already been created with the specified caller reference.")
+		}
+		// The earlier claim's zone was never written: take the reference over.
+		if err := s.store.ReleaseCallerReference(ctx, account, ref); err != nil {
+			return nil, internal("release caller reference", err)
+		}
+		if held, err = s.store.ClaimCallerReference(ctx, account, ref, id); err != nil || held != "" {
+			return nil, awserrors.Errorf(awserrors.ErrorPriorRequestNotComplete, "another request with this caller reference is in progress")
+		}
+	}
+
+	now := s.Now().UTC()
+	zone := &ZoneRecord{
+		ID: id, AccountID: account, Name: name, CallerReference: ref, Comment: comment,
+		CreatedAt: now, Nameservers: nameservers, State: StatePendingNameservers,
+	}
+	doc := &RRSetsDoc{ZoneID: id, Version: 1, UpdatedAt: now, Sets: defaultRecordSets(name, nameservers)}
+	if err := s.store.CreateZone(ctx, zone, doc); err != nil {
+		_ = s.store.ReleaseCallerReference(ctx, account, ref)
+		return nil, internal("create hosted zone", err)
+	}
+	change, err := s.recordChange(ctx, zone, doc.Version, "")
+	if err != nil {
+		return nil, err
+	}
+	return &route53.CreateHostedZoneOutput{
+		HostedZone:    hostedZone(zone, len(doc.Sets)),
+		ChangeInfo:    change,
+		DelegationSet: delegationSet(zone),
+		Location:      aws.String("/2013-04-01" + hostedZonePrefix + id),
+	}, nil
+}
+
+func (s *Service) GetHostedZone(ctx context.Context, in *route53.GetHostedZoneInput, account string) (*route53.GetHostedZoneOutput, error) {
+	zone, doc, _, err := s.zoneWithSets(ctx, account, aws.StringValue(in.Id))
+	if err != nil {
+		return nil, err
+	}
+	return &route53.GetHostedZoneOutput{HostedZone: hostedZone(zone, len(doc.Sets)), DelegationSet: delegationSet(zone)}, nil
+}
+
+func (s *Service) GetHostedZoneCount(ctx context.Context, _ *route53.GetHostedZoneCountInput, account string) (*route53.GetHostedZoneCountOutput, error) {
+	zones, err := s.store.ListAccountZones(ctx, account)
+	if err != nil {
+		return nil, internal("list hosted zones", err)
+	}
+	return &route53.GetHostedZoneCountOutput{HostedZoneCount: aws.Int64(int64(len(zones)))}, nil
+}
+
+func (s *Service) ListHostedZones(ctx context.Context, in *route53.ListHostedZonesInput, account string) (*route53.ListHostedZonesOutput, error) {
+	limit, err := maxItems(in.MaxItems, maxListZones)
+	if err != nil {
+		return nil, err
+	}
+	if in.DelegationSetId != nil {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "reusable delegation sets are not supported")
+	}
+	zones, err := s.sortedZones(ctx, account, func(a, b ZoneRecord) int { return strings.Compare(a.ID, b.ID) })
+	if err != nil {
+		return nil, err
+	}
+	if aws.StringValue(in.HostedZoneType) == route53.HostedZoneTypePrivateHostedZone {
+		zones = nil
+	}
+	marker := bareZoneID(aws.StringValue(in.Marker))
+	start := 0
+	if marker != "" {
+		start = slices.IndexFunc(zones, func(z ZoneRecord) bool { return z.ID >= marker })
+		if start < 0 {
+			start = len(zones)
+		}
+	}
+	page, truncated := pageOf(zones, start, limit)
+	out := &route53.ListHostedZonesOutput{
+		HostedZones: s.hostedZones(ctx, page),
+		IsTruncated: aws.Bool(truncated),
+		MaxItems:    aws.String(strconv.Itoa(limit)),
+		Marker:      in.Marker,
+	}
+	if out.Marker == nil {
+		out.Marker = aws.String("")
+	}
+	if truncated {
+		out.NextMarker = aws.String(zones[start+limit].ID)
+	}
+	return out, nil
+}
+
+func (s *Service) ListHostedZonesByName(ctx context.Context, in *route53.ListHostedZonesByNameInput, account string) (*route53.ListHostedZonesByNameOutput, error) {
+	limit, err := maxItems(in.MaxItems, maxListZones)
+	if err != nil {
+		return nil, err
+	}
+	dnsName := normalizeName(aws.StringValue(in.DNSName))
+	startID := bareZoneID(aws.StringValue(in.HostedZoneId))
+	if startID != "" && dnsName == "" {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "HostedZoneId requires DNSName")
+	}
+	zones, err := s.sortedZones(ctx, account, compareByName)
+	if err != nil {
+		return nil, err
+	}
+	start := 0
+	if dnsName != "" {
+		from := ZoneRecord{Name: dnsName, ID: startID}
+		start = slices.IndexFunc(zones, func(z ZoneRecord) bool { return compareByName(z, from) >= 0 })
+		if start < 0 {
+			start = len(zones)
+		}
+	}
+	page, truncated := pageOf(zones, start, limit)
+	out := &route53.ListHostedZonesByNameOutput{
+		HostedZones:  s.hostedZones(ctx, page),
+		IsTruncated:  aws.Bool(truncated),
+		MaxItems:     aws.String(strconv.Itoa(limit)),
+		DNSName:      in.DNSName,
+		HostedZoneId: in.HostedZoneId,
+	}
+	if truncated {
+		next := zones[start+limit]
+		out.NextDNSName = aws.String(next.Name)
+		out.NextHostedZoneId = aws.String(next.ID)
+	}
+	return out, nil
+}
+
+func (s *Service) UpdateHostedZoneComment(ctx context.Context, in *route53.UpdateHostedZoneCommentInput, account string) (*route53.UpdateHostedZoneCommentOutput, error) {
+	comment := aws.StringValue(in.Comment)
+	if len(comment) > maxCommentLength {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "Comment must be at most %d characters", maxCommentLength)
+	}
+	id := bareZoneID(aws.StringValue(in.Id))
+	err := s.store.MutateZone(ctx, account, id, func(z *ZoneRecord) (bool, error) {
+		if z.Comment == comment {
+			return false, nil
+		}
+		z.Comment = comment
+		return true, nil
+	})
+	if err != nil {
+		return nil, s.zoneErr(id, err)
+	}
+	return s.updatedZone(ctx, account, id)
+}
+
+func (s *Service) updatedZone(ctx context.Context, account, id string) (*route53.UpdateHostedZoneCommentOutput, error) {
+	zone, doc, _, err := s.zoneWithSets(ctx, account, id)
+	if err != nil {
+		return nil, err
+	}
+	return &route53.UpdateHostedZoneCommentOutput{HostedZone: hostedZone(zone, len(doc.Sets))}, nil
+}
+
+func (s *Service) DeleteHostedZone(ctx context.Context, in *route53.DeleteHostedZoneInput, account string) (*route53.DeleteHostedZoneOutput, error) {
+	zone, doc, rev, err := s.zoneWithSets(ctx, account, aws.StringValue(in.Id))
+	if err != nil {
+		return nil, err
+	}
+	for _, set := range doc.Sets {
+		if set.Name != zone.Name || (set.Type != "SOA" && set.Type != "NS") {
+			return nil, awserrors.Errorf(awserrors.ErrorHostedZoneNotEmpty, "The specified hosted zone contains non-required resource record sets and so cannot be deleted.")
+		}
+	}
+	if err := s.store.DeleteZone(ctx, zone, rev); err != nil {
+		if errors.Is(err, kvstore.ErrConflict) {
+			return nil, awserrors.Errorf(awserrors.ErrorPriorRequestNotComplete, "the hosted zone changed while it was being deleted")
+		}
+		return nil, internal("delete hosted zone", err)
+	}
+	// The deletion's change outlives the zone, so GetChange can report it.
+	change, err := s.recordChange(ctx, zone, doc.Version+1, "")
+	if err != nil {
+		return nil, err
+	}
+	return &route53.DeleteHostedZoneOutput{ChangeInfo: change}, nil
+}
+
+func (s *Service) ChangeResourceRecordSets(ctx context.Context, in *route53.ChangeResourceRecordSetsInput, account string) (*route53.ChangeResourceRecordSetsOutput, error) {
+	var comment string
+	if in.ChangeBatch != nil {
+		comment = aws.StringValue(in.ChangeBatch.Comment)
+	}
+	if len(comment) > maxCommentLength {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "Comment must be at most %d characters", maxCommentLength)
+	}
+	for range commitAttempts {
+		zone, doc, rev, err := s.zoneWithSets(ctx, account, aws.StringValue(in.HostedZoneId))
+		if err != nil {
+			return nil, err
+		}
+		sets, err := applyChangeBatch(zone.Name, doc.Sets, in.ChangeBatch)
+		if err != nil {
+			return nil, err
+		}
+		next := &RRSetsDoc{ZoneID: zone.ID, Version: doc.Version + 1, UpdatedAt: s.Now().UTC(), Sets: sets}
+		if err := s.store.CommitRRSets(ctx, account, next, rev); err != nil {
+			if errors.Is(err, kvstore.ErrConflict) {
+				continue
+			}
+			return nil, internal("commit record sets", err)
+		}
+		change, err := s.recordChange(ctx, zone, next.Version, comment)
+		if err != nil {
+			return nil, err
+		}
+		return &route53.ChangeResourceRecordSetsOutput{ChangeInfo: change}, nil
+	}
+	return nil, awserrors.Errorf(awserrors.ErrorPriorRequestNotComplete, "the hosted zone is being changed by another request")
+}
+
+func (s *Service) ListResourceRecordSets(ctx context.Context, in *route53.ListResourceRecordSetsInput, account string) (*route53.ListResourceRecordSetsOutput, error) {
+	limit, err := maxItems(in.MaxItems, maxListRecordSets)
+	if err != nil {
+		return nil, err
+	}
+	if in.StartRecordIdentifier != nil {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "StartRecordIdentifier is not supported: no record set has a SetIdentifier")
+	}
+	startType := strings.ToUpper(aws.StringValue(in.StartRecordType))
+	startName := normalizeName(aws.StringValue(in.StartRecordName))
+	if startType != "" && startName == "" {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "The input is not valid: StartRecordType requires StartRecordName")
+	}
+	_, doc, _, err := s.zoneWithSets(ctx, account, aws.StringValue(in.HostedZoneId))
+	if err != nil {
+		return nil, err
+	}
+	sets := slices.Clone(doc.Sets)
+	slices.SortFunc(sets, compareRRSets)
+	start := 0
+	if startName != "" {
+		from := RRSet{Name: startName, Type: startType}
+		start = slices.IndexFunc(sets, func(r RRSet) bool { return compareRRSets(r, from) >= 0 })
+		if start < 0 {
+			start = len(sets)
+		}
+	}
+	page, truncated := pageOf(sets, start, limit)
+	out := &route53.ListResourceRecordSetsOutput{
+		ResourceRecordSets: make([]*route53.ResourceRecordSet, 0, len(page)),
+		IsTruncated:        aws.Bool(truncated),
+		MaxItems:           aws.String(strconv.Itoa(limit)),
+	}
+	for _, r := range page {
+		out.ResourceRecordSets = append(out.ResourceRecordSets, wireRRSet(r))
+	}
+	if truncated {
+		next := sets[start+limit]
+		out.NextRecordName = aws.String(escapeName(next.Name))
+		out.NextRecordType = aws.String(next.Type)
+	}
+	return out, nil
+}
+
+func (s *Service) GetChange(ctx context.Context, in *route53.GetChangeInput, account string) (*route53.GetChangeOutput, error) {
+	id := bareChangeID(aws.StringValue(in.Id))
+	c, err := s.store.GetChange(ctx, account, id)
+	if errors.Is(err, kvstore.ErrNotFound) {
+		return nil, awserrors.Errorf(awserrors.ErrorNoSuchChange, "A change with the specified change ID does not exist: %s", id)
+	}
+	if err != nil {
+		return nil, internal("get change", err)
+	}
+	return &route53.GetChangeOutput{ChangeInfo: changeInfo(c)}, nil
+}
+
+func (s *Service) ChangeTagsForResource(ctx context.Context, in *route53.ChangeTagsForResourceInput, account string) (*route53.ChangeTagsForResourceOutput, error) {
+	id, err := taggedZoneID(aws.StringValue(in.ResourceType), aws.StringValue(in.ResourceId))
+	if err != nil {
+		return nil, err
+	}
+	for _, t := range in.AddTags {
+		key := aws.StringValue(t.Key)
+		if key == "" || len(key) > 128 || len(aws.StringValue(t.Value)) > 256 {
+			return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "tag keys must be 1 to 128 characters and values at most 256")
+		}
+	}
+	err = s.store.MutateZone(ctx, account, id, func(z *ZoneRecord) (bool, error) {
+		if z.Tags == nil {
+			z.Tags = map[string]string{}
+		}
+		for _, k := range in.RemoveTagKeys {
+			delete(z.Tags, aws.StringValue(k))
+		}
+		for _, t := range in.AddTags {
+			z.Tags[aws.StringValue(t.Key)] = aws.StringValue(t.Value)
+		}
+		if len(z.Tags) > maxTagsPerResource {
+			return false, awserrors.Errorf(awserrors.ErrorRoute53TooManyTagsForZone, "a hosted zone can have at most %d tags", maxTagsPerResource)
+		}
+		return true, nil
+	})
+	if err != nil {
+		return nil, s.zoneErr(id, err)
+	}
+	return &route53.ChangeTagsForResourceOutput{}, nil
+}
+
+func (s *Service) ListTagsForResource(ctx context.Context, in *route53.ListTagsForResourceInput, account string) (*route53.ListTagsForResourceOutput, error) {
+	id, err := taggedZoneID(aws.StringValue(in.ResourceType), aws.StringValue(in.ResourceId))
+	if err != nil {
+		return nil, err
+	}
+	zone, err := s.store.GetZone(ctx, account, id)
+	if err != nil {
+		return nil, s.zoneErr(id, err)
+	}
+	return &route53.ListTagsForResourceOutput{ResourceTagSet: tagSet(zone)}, nil
+}
+
+func (s *Service) ListTagsForResources(ctx context.Context, in *route53.ListTagsForResourcesInput, account string) (*route53.ListTagsForResourcesOutput, error) {
+	if len(in.ResourceIds) == 0 || len(in.ResourceIds) > maxTagResources {
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidInput, "ResourceIds must name 1 to %d resources", maxTagResources)
+	}
+	out := &route53.ListTagsForResourcesOutput{ResourceTagSets: []*route53.ResourceTagSet{}}
+	for _, rid := range in.ResourceIds {
+		id, err := taggedZoneID(aws.StringValue(in.ResourceType), aws.StringValue(rid))
+		if err != nil {
+			return nil, err
+		}
+		zone, err := s.store.GetZone(ctx, account, id)
+		if err != nil {
+			return nil, s.zoneErr(id, err)
+		}
+		out.ResourceTagSets = append(out.ResourceTagSets, tagSet(zone))
+	}
+	return out, nil
+}
+
+// zoneWithSets loads an account's zone and its record-set document.
+func (s *Service) zoneWithSets(ctx context.Context, account, rawID string) (*ZoneRecord, *RRSetsDoc, uint64, error) {
+	id := bareZoneID(rawID)
+	if id == "" {
+		return nil, nil, 0, awserrors.Errorf(awserrors.ErrorInvalidInput, "a hosted zone ID is required")
+	}
+	zone, err := s.store.GetZone(ctx, account, id)
+	if err != nil {
+		return nil, nil, 0, s.zoneErr(id, err)
+	}
+	doc, rev, err := s.store.GetRRSets(ctx, account, id)
+	if err != nil {
+		return nil, nil, 0, s.zoneErr(id, err)
+	}
+	return zone, doc, rev, nil
+}
+
+func (s *Service) zoneErr(id string, err error) error {
+	if errors.Is(err, kvstore.ErrNotFound) {
+		return awserrors.Errorf(awserrors.ErrorNoSuchHostedZone, "No hosted zone found with ID: %s", id)
+	}
+	if _, _, ok := awserrors.ResolveErrorDetail(err); ok {
+		return err
+	}
+	return internal("read hosted zone", err)
+}
+
+// recordChange stores the change a write produced. A zone that is not being
+// served is in sync as soon as the write is durable; an active one waits for
+// every northstar server to load the version.
+func (s *Service) recordChange(ctx context.Context, zone *ZoneRecord, version uint64, comment string) (*route53.ChangeInfo, error) {
+	now := s.Now().UTC()
+	c := &ChangeRecord{
+		ID: newChangeID(), AccountID: zone.AccountID, ZoneID: zone.ID, Version: version,
+		Status: ChangeStatusInSync, Comment: comment, SubmittedAt: now,
+	}
+	if zone.State == StateActive {
+		c.Status = ChangeStatusPending
+	}
+	if err := s.store.PutChange(ctx, c); err != nil {
+		return nil, internal("record change", err)
+	}
+	_ = s.store.PruneChanges(ctx, zone.AccountID, zone.ID, now.Add(-changeRetention), false)
+	return changeInfo(c), nil
+}
+
+func (s *Service) sortedZones(ctx context.Context, account string, cmp func(a, b ZoneRecord) int) ([]ZoneRecord, error) {
+	zones, err := s.store.ListAccountZones(ctx, account)
+	if err != nil {
+		return nil, internal("list hosted zones", err)
+	}
+	slices.SortFunc(zones, cmp)
+	return zones, nil
+}
+
+func (s *Service) hostedZones(ctx context.Context, zones []ZoneRecord) []*route53.HostedZone {
+	out := make([]*route53.HostedZone, 0, len(zones))
+	for i := range zones {
+		count := 0
+		if doc, _, err := s.store.GetRRSets(ctx, zones[i].AccountID, zones[i].ID); err == nil {
+			count = len(doc.Sets)
+		}
+		out = append(out, hostedZone(&zones[i], count))
+	}
+	return out
+}
+
+func compareByName(a, b ZoneRecord) int {
+	if c := strings.Compare(reversedName(a.Name), reversedName(b.Name)); c != 0 {
+		return c
+	}
+	return strings.Compare(a.ID, b.ID)
+}
+
+func compareRRSets(a, b RRSet) int {
+	if c := strings.Compare(reversedName(a.Name), reversedName(b.Name)); c != 0 {
+		return c
+	}
+	return strings.Compare(a.Type, b.Type)
+}
+
+func pageOf[T any](items []T, start, limit int) ([]T, bool) {
+	end := min(start+limit, len(items))
+	return items[start:end], end < len(items)
+}
+
+func maxItems(raw *string, ceiling int) (int, error) {
+	if raw == nil || *raw == "" {
+		return ceiling, nil
+	}
+	n, err := strconv.Atoi(*raw)
+	if err != nil || n < 1 {
+		return 0, awserrors.Errorf(awserrors.ErrorInvalidInput, "MaxItems must be a positive integer")
+	}
+	return min(n, ceiling), nil
+}
+
+func taggedZoneID(resourceType, id string) (string, error) {
+	switch resourceType {
+	case route53.TagResourceTypeHostedzone:
+		return bareZoneID(id), nil
+	case route53.TagResourceTypeHealthcheck:
+		return "", awserrors.Errorf(awserrors.ErrorRoute53NoSuchHealthCheck, "No health check exists with the ID %s", id)
+	default:
+		return "", awserrors.Errorf(awserrors.ErrorInvalidInput, "ResourceType must be hostedzone or healthcheck")
+	}
+}
+
+func defaultRecordSets(zone string, nameservers []string) []RRSet {
+	ns := make([]string, 0, len(nameservers))
+	for _, n := range nameservers {
+		ns = append(ns, n+".")
+	}
+	hostmaster := "hostmaster." + zone
+	if _, parent, ok := strings.Cut(nameservers[0], "."); ok {
+		hostmaster = "hostmaster." + parent + "."
+	}
+	soa := fmt.Sprintf("%s %s 1 7200 900 1209600 86400", ns[0], hostmaster)
+	return []RRSet{
+		{Name: zone, Type: "NS", TTL: defaultNSTTL, Values: ns},
+		{Name: zone, Type: "SOA", TTL: defaultSOATTL, Values: []string{soa}},
+	}
+}
+
+func hostedZone(z *ZoneRecord, count int) *route53.HostedZone {
+	return &route53.HostedZone{
+		Id:              aws.String(hostedZonePrefix + z.ID),
+		Name:            aws.String(z.Name),
+		CallerReference: aws.String(z.CallerReference),
+		Config: &route53.HostedZoneConfig{
+			Comment:     nonEmpty(z.Comment),
+			PrivateZone: aws.Bool(false),
+		},
+		ResourceRecordSetCount: aws.Int64(int64(count)),
+	}
+}
+
+func delegationSet(z *ZoneRecord) *route53.DelegationSet {
+	return &route53.DelegationSet{NameServers: aws.StringSlice(z.Nameservers)}
+}
+
+func changeInfo(c *ChangeRecord) *route53.ChangeInfo {
+	submitted := c.SubmittedAt
+	return &route53.ChangeInfo{
+		Id:          aws.String(changePrefix + c.ID),
+		Status:      aws.String(c.Status),
+		SubmittedAt: &submitted,
+		Comment:     nonEmpty(c.Comment),
+	}
+}
+
+func tagSet(z *ZoneRecord) *route53.ResourceTagSet {
+	keys := make([]string, 0, len(z.Tags))
+	for k := range z.Tags {
+		keys = append(keys, k)
+	}
+	slices.Sort(keys)
+	tags := make([]*route53.Tag, 0, len(keys))
+	for _, k := range keys {
+		tags = append(tags, &route53.Tag{Key: aws.String(k), Value: aws.String(z.Tags[k])})
+	}
+	return &route53.ResourceTagSet{
+		ResourceType: aws.String(route53.TagResourceTypeHostedzone),
+		ResourceId:   aws.String(z.ID),
+		Tags:         tags,
+	}
+}
+
+func wireRRSet(r RRSet) *route53.ResourceRecordSet {
+	out := &route53.ResourceRecordSet{Name: aws.String(escapeName(r.Name)), Type: aws.String(r.Type), TTL: aws.Int64(r.TTL)}
+	for _, v := range r.Values {
+		out.ResourceRecords = append(out.ResourceRecords, &route53.ResourceRecord{Value: aws.String(v)})
+	}
+	return out
+}
+
+// escapeName renders a wildcard label the way Route 53 returns it, as \052.
+func escapeName(name string) string {
+	if strings.HasPrefix(name, "*.") {
+		return `\052` + name[1:]
+	}
+	return name
+}
+
+func nonEmpty(s string) *string {
+	if s == "" {
+		return nil
+	}
+	return aws.String(s)
+}
+
+func internal(what string, err error) error {
+	return awserrors.Errorf(awserrors.ErrorServerInternal, "%s: %v", what, err)
+}

spinifex/domains/route53/service_test.go

@@ -0,0 +1,510 @@
+package route53_test
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	r53 "github.com/aws/aws-sdk-go/service/route53"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/route53"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+)
+
+const (
+	acctA = "000000000011"
+	acctB = "000000000012"
+)
+
+var ctx = context.Background()
+
+func newService(t *testing.T) *route53.Service {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	svc := route53.NewService(route53.NewStore(js))
+	svc.Nameservers = func() []string { return []string{"ns1.spx3.net", "ns2.spx3.net"} }
+	svc.Reserved = func(name string) bool { return name == "spx3.net." || strings.HasSuffix(name, ".compute.internal.") }
+	return svc
+}
+
+func createZone(t *testing.T, svc *route53.Service, account, name string) string {
+	t.Helper()
+	out, err := svc.CreateHostedZone(ctx, &r53.CreateHostedZoneInput{
+		Name: aws.String(name), CallerReference: aws.String("ref-" + name),
+	}, account)
+	require.NoError(t, err)
+	return aws.StringValue(out.HostedZone.Id)
+}
+
+func requireCode(t *testing.T, err error, code string) string {
+	t.Helper()
+	require.Error(t, err)
+	got, msg, ok := awserrors.ResolveErrorDetail(err)
+	require.True(t, ok, "error carries no AWS code: %v", err)
+	require.Equal(t, code, got, "message: %s", msg)
+	return msg
+}
+
+func change(action, name, rrType string, ttl int64, values ...string) *r53.Change {
+	set := &r53.ResourceRecordSet{Name: aws.String(name), Type: aws.String(rrType), TTL: aws.Int64(ttl)}
+	for _, v := range values {
+		set.ResourceRecords = append(set.ResourceRecords, &r53.ResourceRecord{Value: aws.String(v)})
+	}
+	return &r53.Change{Action: aws.String(action), ResourceRecordSet: set}
+}
+
+func apply(svc *route53.Service, account, zoneID string, changes ...*r53.Change) (*r53.ChangeResourceRecordSetsOutput, error) {
+	return svc.ChangeResourceRecordSets(ctx, &r53.ChangeResourceRecordSetsInput{
+		HostedZoneId: aws.String(zoneID), ChangeBatch: &r53.ChangeBatch{Changes: changes},
+	}, account)
+}
+
+func listSets(t *testing.T, svc *route53.Service, account, zoneID string) []string {
+	t.Helper()
+	out, err := svc.ListResourceRecordSets(ctx, &r53.ListResourceRecordSetsInput{HostedZoneId: aws.String(zoneID)}, account)
+	require.NoError(t, err)
+	var got []string
+	for _, s := range out.ResourceRecordSets {
+		var vals []string
+		for _, r := range s.ResourceRecords {
+			vals = append(vals, aws.StringValue(r.Value))
+		}
+		got = append(got, fmt.Sprintf("%s %s %d %s", aws.StringValue(s.Name), aws.StringValue(s.Type), aws.Int64Value(s.TTL), strings.Join(vals, ",")))
+	}
+	return got
+}
+
+func TestCreateHostedZoneReturnsAWSShape(t *testing.T) {
+	svc := newService(t)
+	out, err := svc.CreateHostedZone(ctx, &r53.CreateHostedZoneInput{
+		Name:             aws.String("Example.COM"),
+		CallerReference:  aws.String("terraform-1"),
+		HostedZoneConfig: &r53.HostedZoneConfig{Comment: aws.String("Managed by Terraform")},
+	}, acctA)
+	require.NoError(t, err)
+
+	id := aws.StringValue(out.HostedZone.Id)
+	assert.Regexp(t, `^/hostedzone/Z[A-Z0-9]{20}$`, id)
+	assert.Equal(t, "example.com.", aws.StringValue(out.HostedZone.Name))
+	assert.Equal(t, "Managed by Terraform", aws.StringValue(out.HostedZone.Config.Comment))
+	assert.False(t, aws.BoolValue(out.HostedZone.Config.PrivateZone))
+	assert.Equal(t, int64(2), aws.Int64Value(out.HostedZone.ResourceRecordSetCount))
+	assert.Equal(t, []string{"ns1.spx3.net", "ns2.spx3.net"}, aws.StringValueSlice(out.DelegationSet.NameServers))
+	assert.Equal(t, "/2013-04-01"+id, aws.StringValue(out.Location))
+	assert.Regexp(t, `^/change/C[A-Z0-9]{20}$`, aws.StringValue(out.ChangeInfo.Id))
+	assert.Equal(t, route53.ChangeStatusInSync, aws.StringValue(out.ChangeInfo.Status), "a zone not yet served is in sync once durable")
+
+	assert.Equal(t, []string{
+		"example.com. NS 172800 ns1.spx3.net.,ns2.spx3.net.",
+		"example.com. SOA 900 ns1.spx3.net. hostmaster.spx3.net. 1 7200 900 1209600 86400",
+	}, listSets(t, svc, acctA, id))
+
+	got, err := svc.GetHostedZone(ctx, &r53.GetHostedZoneInput{Id: aws.String(strings.TrimPrefix(id, "/hostedzone/"))}, acctA)
+	require.NoError(t, err)
+	assert.Equal(t, id, aws.StringValue(got.HostedZone.Id), "a bare ID and a prefixed one name the same zone")
+	assert.Equal(t, []string{"ns1.spx3.net", "ns2.spx3.net"}, aws.StringValueSlice(got.DelegationSet.NameServers))
+}
+
+func TestCreateHostedZoneRejections(t *testing.T) {
+	svc := newService(t)
+	createZone(t, svc, acctA, "example.com")
+
+	cases := []struct {
+		name string
+		in   *r53.CreateHostedZoneInput
+		code string
+	}{
+		{"repeated caller reference", &r53.CreateHostedZoneInput{Name: aws.String("other.com"), CallerReference: aws.String("ref-example.com")}, awserrors.ErrorHostedZoneAlreadyExists},
+		{"bare TLD", &r53.CreateHostedZoneInput{Name: aws.String("com"), CallerReference: aws.String("r1")}, awserrors.ErrorInvalidDomainName},
+		{"bad label", &r53.CreateHostedZoneInput{Name: aws.String("-bad.com"), CallerReference: aws.String("r2")}, awserrors.ErrorInvalidDomainName},
+		{"platform zone", &r53.CreateHostedZoneInput{Name: aws.String("spx3.net"), CallerReference: aws.String("r3")}, awserrors.ErrorInvalidDomainName},
+		{"private zone", &r53.CreateHostedZoneInput{Name: aws.String("p.com"), CallerReference: aws.String("r4"), VPC: &r53.VPC{VPCId: aws.String("vpc-1")}}, awserrors.ErrorInvalidInput},
+		{"delegation set", &r53.CreateHostedZoneInput{Name: aws.String("d.com"), CallerReference: aws.String("r5"), DelegationSetId: aws.String("N1")}, awserrors.ErrorInvalidInput},
+		{"no caller reference", &r53.CreateHostedZoneInput{Name: aws.String("n.com")}, awserrors.ErrorInvalidInput},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			_, err := svc.CreateHostedZone(ctx, tc.in, acctA)
+			requireCode(t, err, tc.code)
+		})
+	}
+
+	// The same reference in another account is a different request.
+	_, err := svc.CreateHostedZone(ctx, &r53.CreateHostedZoneInput{Name: aws.String("example.com"), CallerReference: aws.String("ref-example.com")}, acctB)
+	require.NoError(t, err)
+}
+
+func TestCreateHostedZoneEnforcesAccountLimit(t *testing.T) {
+	svc := newService(t)
+	svc.MaxZonesPerAccount = 2
+	createZone(t, svc, acctA, "a.com")
+	createZone(t, svc, acctA, "b.com")
+	_, err := svc.CreateHostedZone(ctx, &r53.CreateHostedZoneInput{Name: aws.String("c.com"), CallerReference: aws.String("c")}, acctA)
+	requireCode(t, err, awserrors.ErrorTooManyHostedZones)
+	createZone(t, svc, acctB, "c.com")
+}
+
+func TestZonesAreInvisibleToOtherAccounts(t *testing.T) {
+	svc := newService(t)
+	id := createZone(t, svc, acctA, "example.com")
+
+	_, err := svc.GetHostedZone(ctx, &r53.GetHostedZoneInput{Id: aws.String(id)}, acctB)
+	requireCode(t, err, awserrors.ErrorNoSuchHostedZone)
+	_, err = apply(svc, acctB, id, change("CREATE", "www.example.com", "A", 60, "192.0.2.1"))
+	requireCode(t, err, awserrors.ErrorNoSuchHostedZone)
+	_, err = svc.DeleteHostedZone(ctx, &r53.DeleteHostedZoneInput{Id: aws.String(id)}, acctB)
+	requireCode(t, err, awserrors.ErrorNoSuchHostedZone)
+
+	list, err := svc.ListHostedZones(ctx, &r53.ListHostedZonesInput{}, acctB)
+	require.NoError(t, err)
+	assert.Empty(t, list.HostedZones)
+}
+
+func TestChangeResourceRecordSetsCreateUpsertDelete(t *testing.T) {
+	svc := newService(t)
+	id := createZone(t, svc, acctA, "example.com")
+
+	_, err := apply(svc, acctA, id,
+		change("CREATE", "www.example.com", "A", 300, "192.0.2.10"),
+		change("CREATE", "*.example.com", "TXT", 60, `"hello"`),
+		change("CREATE", "example.com", "MX", 300, "10 mail.example.com"),
+	)
+	require.NoError(t, err)
+
+	_, err = apply(svc, acctA, id, change("UPSERT", "WWW.example.com.", "A", 120, "192.0.2.11", "192.0.2.12"))
+	require.NoError(t, err)
+
+	assert.Equal(t, []string{
+		"example.com. MX 300 10 mail.example.com",
+		"example.com. NS 172800 ns1.spx3.net.,ns2.spx3.net.",
+		"example.com. SOA 900 ns1.spx3.net. hostmaster.spx3.net. 1 7200 900 1209600 86400",
+		`\052.example.com. TXT 60 "hello"`,
+		"www.example.com. A 120 192.0.2.11,192.0.2.12",
+	}, listSets(t, svc, acctA, id))
+
+	msg := requireCode(t, func() error {
+		_, err := apply(svc, acctA, id, change("DELETE", "www.example.com", "A", 120, "192.0.2.11"))
+		return err
+	}(), awserrors.ErrorInvalidChangeBatch)
+	assert.Contains(t, msg, "Tried to delete resource record set [name='www.example.com.', type='A'] but the values provided do not match the current values")
+
+	_, err = apply(svc, acctA, id, change("DELETE", "www.example.com.", "A", 120, "192.0.2.12", "192.0.2.11"))
+	require.NoError(t, err, "a DELETE matches values regardless of order")
+	assert.Len(t, listSets(t, svc, acctA, id), 4)
+}
+
+func TestChangeBatchIsAllOrNothing(t *testing.T) {
+	svc := newService(t)
+	id := createZone(t, svc, acctA, "example.com")
+	_, err := apply(svc, acctA, id, change("CREATE", "www.example.com", "A", 60, "192.0.2.1"))
+	require.NoError(t, err)
+
+	msg := requireCode(t, func() error {
+		_, err := apply(svc, acctA, id,
+			change("CREATE", "api.example.com", "A", 60, "192.0.2.2"),
+			change("CREATE", "www.example.com", "A", 60, "192.0.2.3"),
+			change("DELETE", "gone.example.com", "A", 60, "192.0.2.4"),
+		)
+		return err
+	}(), awserrors.ErrorInvalidChangeBatch)
+	assert.Contains(t, msg, "Tried to create resource record set [name='www.example.com.', type='A'] but it already exists")
+	assert.Contains(t, msg, "Tried to delete resource record set [name='gone.example.com.', type='A'] but it was not found")
+	assert.Len(t, listSets(t, svc, acctA, id), 3, "the valid change in a failed batch must not land")
+
+	_, err = apply(svc, acctA, id,
+		change("DELETE", "www.example.com", "A", 60, "192.0.2.1"),
+		change("CREATE", "www.example.com", "CNAME", 60, "lb.example.net"),
+	)
+	require.NoError(t, err, "changes apply in order, so a delete frees the name for a CNAME in the same batch")
+}
+
+func TestChangeBatchValidation(t *testing.T) {
+	svc := newService(t)
+	id := createZone(t, svc, acctA, "example.com")
+	_, err := apply(svc, acctA, id, change("CREATE", "www.example.com", "A", 60, "192.0.2.1"))
+	require.NoError(t, err)
+
+	alias := change("CREATE", "lb.example.com", "A", 60)
+	alias.ResourceRecordSet.TTL = nil
+	alias.ResourceRecordSet.AliasTarget = &r53.AliasTarget{DNSName: aws.String("x.elb.spx3.net"), HostedZoneId: aws.String("Z1"), EvaluateTargetHealth: aws.Bool(false)}
+	weighted := change("CREATE", "w.example.com", "A", 60, "192.0.2.1")
+	weighted.ResourceRecordSet.SetIdentifier = aws.String("a")
+	weighted.ResourceRecordSet.Weight = aws.Int64(10)
+
+	cases := []struct {
+		name   string
+		change *r53.Change
+		code   string
+		msg    string
+	}{
+		{"CNAME beside A", change("CREATE", "www.example.com", "CNAME", 60, "x.example.net"), awserrors.ErrorInvalidChangeBatch, "conflicting RRSet of type A"},
+		{"CNAME at apex", change("CREATE", "example.com", "CNAME", 60, "x.example.net"), awserrors.ErrorInvalidChangeBatch, "not permitted at apex"},
+		{"name outside zone", change("CREATE", "www.example.org", "A", 60, "192.0.2.1"), awserrors.ErrorInvalidChangeBatch, "is not permitted in zone example.com."},
+		{"apex NS delete", change("DELETE", "example.com", "NS", 172800, "ns1.spx3.net.", "ns2.spx3.net."), awserrors.ErrorInvalidChangeBatch, "must contain at least one NS record"},
+		{"SOA below apex", change("CREATE", "sub.example.com", "SOA", 60, "a. b. 1 2 3 4 5"), awserrors.ErrorInvalidChangeBatch, "only permitted at the zone apex"},
+		{"bad IPv4", change("CREATE", "v4.example.com", "A", 60, "2001:db8::1"), awserrors.ErrorInvalidChangeBatch, "not a valid IPv4 address"},
+		{"bad IPv6", change("CREATE", "v6.example.com", "AAAA", 60, "192.0.2.1"), awserrors.ErrorInvalidChangeBatch, "not a valid IPv6 address"},
+		{"bad MX", change("CREATE", "example.com", "MX", 60, "mail.example.com"), awserrors.ErrorInvalidChangeBatch, "priority host"},
+		{"two CNAME values", change("CREATE", "c.example.com", "CNAME", 60, "a.example.net", "b.example.net"), awserrors.ErrorInvalidChangeBatch, "more than one value"},
+		{"bad action", change("REPLACE", "r.example.com", "A", 60, "192.0.2.1"), awserrors.ErrorInvalidChangeBatch, "Invalid action"},
+		{"alias", alias, awserrors.ErrorInvalidInput, "alias"},
+		{"weighted", weighted, awserrors.ErrorInvalidInput, "simple routing"},
+		{"unsupported type", change("CREATE", "ds.example.com", "DS", 60, "1 2 3 abc"), awserrors.ErrorInvalidInput, "not supported"},
+		{"negative TTL", change("CREATE", "t.example.com", "A", -1, "192.0.2.1"), awserrors.ErrorInvalidInput, "out of range"},
+		{"no values", change("CREATE", "e.example.com", "A", 60), awserrors.ErrorInvalidInput, "at least one record"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			_, err := apply(svc, acctA, id, tc.change)
+			assert.Contains(t, requireCode(t, err, tc.code), tc.msg)
+		})
+	}
+
+	_, err = apply(svc, acctA, id)
+	requireCode(t, err, awserrors.ErrorInvalidInput)
+}
+
+func TestChangeBatchSizeLimits(t *testing.T) {
+	svc := newService(t)
+	id := createZone(t, svc, acctA, "example.com")
+	values := make([]string, 0, 401)
+	for i := range 401 {
+		values = append(values, fmt.Sprintf("192.0.%d.%d", i/250, i%250+1))
+	}
+	_, err := apply(svc, acctA, id, change("CREATE", "many.example.com", "A", 60, values...))
+	assert.Contains(t, requireCode(t, err, awserrors.ErrorInvalidChangeBatch), "more than 400 values")
+
+	var batch []*r53.Change
+	for i := range 3 {
+		batch = append(batch, change("UPSERT", fmt.Sprintf("h%d.example.com", i), "A", 60, values[:200]...))
+	}
+	_, err = apply(svc, acctA, id, batch...)
+	assert.Contains(t, requireCode(t, err, awserrors.ErrorInvalidChangeBatch), "Number of records limit of 1000 exceeded", "an UPSERT counts each record twice")
+}
+
+func TestConcurrentChangeBatchesBothLand(t *testing.T) {
+	svc := newService(t)
+	id := createZone(t, svc, acctA, "example.com")
+	var wg sync.WaitGroup
+	errs := make([]error, 2)
+	for i := range 2 {
+		wg.Go(func() {
+			_, errs[i] = apply(svc, acctA, id, change("CREATE", fmt.Sprintf("h%d.example.com", i), "A", 60, "192.0.2.1"))
+		})
+	}
+	wg.Wait()
+	require.NoError(t, errs[0])
+	require.NoError(t, errs[1])
+	assert.Len(t, listSets(t, svc, acctA, id), 4)
+}
+
+func TestListResourceRecordSetsPagesInRoute53Order(t *testing.T) {
+	svc := newService(t)
+	id := createZone(t, svc, acctA, "example.com")
+	_, err := apply(svc, acctA, id,
+		change("CREATE", "www.example.com", "A", 300, "192.0.2.10"),
+		change("CREATE", "www.example.com", "AAAA", 300, "2001:db8::10"),
+		change("CREATE", "a.b.example.com", "A", 300, "192.0.2.11"),
+		change("CREATE", "z.example.com", "A", 300, "192.0.2.12"),
+	)
+	require.NoError(t, err)
+
+	// The read Terraform makes for one aws_route53_record.
+	out, err := svc.ListResourceRecordSets(ctx, &r53.ListResourceRecordSetsInput{
+		HostedZoneId: aws.String(id), MaxItems: aws.String("1"),
+		StartRecordName: aws.String("www.example.com."), StartRecordType: aws.String("A"),
+	}, acctA)
+	require.NoError(t, err)
+	require.Len(t, out.ResourceRecordSets, 1)
+	assert.Equal(t, "www.example.com.", aws.StringValue(out.ResourceRecordSets[0].Name))
+	assert.Equal(t, "A", aws.StringValue(out.ResourceRecordSets[0].Type))
+	assert.True(t, aws.BoolValue(out.IsTruncated))
+	assert.Equal(t, "www.example.com.", aws.StringValue(out.NextRecordName))
+	assert.Equal(t, "AAAA", aws.StringValue(out.NextRecordType))
+
+	var names []string
+	in := &r53.ListResourceRecordSetsInput{HostedZoneId: aws.String(id), MaxItems: aws.String("2")}
+	for {
+		page, err := svc.ListResourceRecordSets(ctx, in, acctA)
+		require.NoError(t, err)
+		for _, s := range page.ResourceRecordSets {
+			names = append(names, aws.StringValue(s.Name)+" "+aws.StringValue(s.Type))
+		}
+		if !aws.BoolValue(page.IsTruncated) {
+			break
+		}
+		in.StartRecordName, in.StartRecordType = page.NextRecordName, page.NextRecordType
+	}
+	assert.Equal(t, []string{
+		"example.com. NS", "example.com. SOA", "a.b.example.com. A",
+		"www.example.com. A", "www.example.com. AAAA", "z.example.com. A",
+	}, names, "names sort by reversed label, so b.example.com sorts before www")
+
+	_, err = svc.ListResourceRecordSets(ctx, &r53.ListResourceRecordSetsInput{HostedZoneId: aws.String(id), StartRecordType: aws.String("A")}, acctA)
+	requireCode(t, err, awserrors.ErrorInvalidInput)
+}
+
+func TestDeleteHostedZoneRequiresOnlyDefaultRecords(t *testing.T) {
+	svc := newService(t)
+	id := createZone(t, svc, acctA, "example.com")
+	_, err := apply(svc, acctA, id, change("CREATE", "sub.example.com", "NS", 300, "ns.elsewhere.net"))
+	require.NoError(t, err)
+
+	_, err = svc.DeleteHostedZone(ctx, &r53.DeleteHostedZoneInput{Id: aws.String(id)}, acctA)
+	requireCode(t, err, awserrors.ErrorHostedZoneNotEmpty)
+
+	_, err = apply(svc, acctA, id, change("DELETE", "sub.example.com", "NS", 300, "ns.elsewhere.net"))
+	require.NoError(t, err)
+	out, err := svc.DeleteHostedZone(ctx, &r53.DeleteHostedZoneInput{Id: aws.String(id)}, acctA)
+	require.NoError(t, err)
+
+	got, err := svc.GetChange(ctx, &r53.GetChangeInput{Id: out.ChangeInfo.Id}, acctA)
+	require.NoError(t, err, "Terraform polls GetChange after DeleteHostedZone")
+	assert.Equal(t, route53.ChangeStatusInSync, aws.StringValue(got.ChangeInfo.Status))
+
+	_, err = svc.GetHostedZone(ctx, &r53.GetHostedZoneInput{Id: aws.String(id)}, acctA)
+	requireCode(t, err, awserrors.ErrorNoSuchHostedZone)
+
+	// The caller reference is released with the zone.
+	_, err = svc.CreateHostedZone(ctx, &r53.CreateHostedZoneInput{Name: aws.String("example.com"), CallerReference: aws.String("ref-example.com")}, acctA)
+	require.NoError(t, err)
+}
+
+func TestListHostedZonesPaging(t *testing.T) {
+	svc := newService(t)
+	for _, n := range []string{"b.example.com", "example.org", "a.example.com", "example.com"} {
+		createZone(t, svc, acctA, n)
+	}
+	createZone(t, svc, acctB, "other.com")
+
+	var ids []string
+	in := &r53.ListHostedZonesInput{MaxItems: aws.String("3")}
+	for {
+		out, err := svc.ListHostedZones(ctx, in, acctA)
+		require.NoError(t, err)
+		for _, z := range out.HostedZones {
+			ids = append(ids, aws.StringValue(z.Id))
+		}
+		if !aws.BoolValue(out.IsTruncated) {
+			break
+		}
+		in.Marker = out.NextMarker
+	}
+	assert.Len(t, ids, 4)
+
+	byName, err := svc.ListHostedZonesByName(ctx, &r53.ListHostedZonesByNameInput{}, acctA)
+	require.NoError(t, err)
+	var names []string
+	for _, z := range byName.HostedZones {
+		names = append(names, aws.StringValue(z.Name))
+	}
+	assert.Equal(t, []string{"example.com.", "a.example.com.", "b.example.com.", "example.org."}, names)
+
+	from, err := svc.ListHostedZonesByName(ctx, &r53.ListHostedZonesByNameInput{DNSName: aws.String("b.example.com"), MaxItems: aws.String("1")}, acctA)
+	require.NoError(t, err)
+	require.Len(t, from.HostedZones, 1)
+	assert.Equal(t, "b.example.com.", aws.StringValue(from.HostedZones[0].Name))
+	assert.Equal(t, "example.org.", aws.StringValue(from.NextDNSName))
+
+	count, err := svc.GetHostedZoneCount(ctx, &r53.GetHostedZoneCountInput{}, acctA)
+	require.NoError(t, err)
+	assert.Equal(t, int64(4), aws.Int64Value(count.HostedZoneCount))
+
+	private, err := svc.ListHostedZones(ctx, &r53.ListHostedZonesInput{HostedZoneType: aws.String("PrivateHostedZone")}, acctA)
+	require.NoError(t, err)
+	assert.Empty(t, private.HostedZones)
+}
+
+func TestHostedZoneTags(t *testing.T) {
+	svc := newService(t)
+	id := createZone(t, svc, acctA, "example.com")
+	bare := strings.TrimPrefix(id, "/hostedzone/")
+
+	_, err := svc.ChangeTagsForResource(ctx, &r53.ChangeTagsForResourceInput{
+		ResourceType: aws.String("hostedzone"), ResourceId: aws.String(bare),
+		AddTags: []*r53.Tag{{Key: aws.String("env"), Value: aws.String("prod")}, {Key: aws.String("team"), Value: aws.String("dns")}},
+	}, acctA)
+	require.NoError(t, err)
+	_, err = svc.ChangeTagsForResource(ctx, &r53.ChangeTagsForResourceInput{
+		ResourceType: aws.String("hostedzone"), ResourceId: aws.String(bare), RemoveTagKeys: aws.StringSlice([]string{"team"}),
+	}, acctA)
+	require.NoError(t, err)
+
+	out, err := svc.ListTagsForResource(ctx, &r53.ListTagsForResourceInput{ResourceType: aws.String("hostedzone"), ResourceId: aws.String(bare)}, acctA)
+	require.NoError(t, err)
+	assert.Equal(t, bare, aws.StringValue(out.ResourceTagSet.ResourceId))
+	require.Len(t, out.ResourceTagSet.Tags, 1)
+	assert.Equal(t, "env", aws.StringValue(out.ResourceTagSet.Tags[0].Key))
+
+	many, err := svc.ListTagsForResources(ctx, &r53.ListTagsForResourcesInput{ResourceType: aws.String("hostedzone"), ResourceIds: aws.StringSlice([]string{bare})}, acctA)
+	require.NoError(t, err)
+	assert.Len(t, many.ResourceTagSets, 1)
+
+	_, err = svc.ListTagsForResource(ctx, &r53.ListTagsForResourceInput{ResourceType: aws.String("hostedzone"), ResourceId: aws.String(bare)}, acctB)
+	requireCode(t, err, awserrors.ErrorNoSuchHostedZone)
+	_, err = svc.ListTagsForResource(ctx, &r53.ListTagsForResourceInput{ResourceType: aws.String("healthcheck"), ResourceId: aws.String("hc")}, acctA)
+	requireCode(t, err, awserrors.ErrorRoute53NoSuchHealthCheck)
+
+	var tags []*r53.Tag
+	for i := range 51 {
+		tags = append(tags, &r53.Tag{Key: aws.String(fmt.Sprintf("k%d", i)), Value: aws.String("v")})
+	}
+	_, err = svc.ChangeTagsForResource(ctx, &r53.ChangeTagsForResourceInput{ResourceType: aws.String("hostedzone"), ResourceId: aws.String(bare), AddTags: tags}, acctA)
+	requireCode(t, err, awserrors.ErrorRoute53TooManyTagsForZone)
+}
+
+func TestChangesOnAnActiveZoneWaitForSync(t *testing.T) {
+	svc := newService(t)
+	id := createZone(t, svc, acctA, "example.com")
+	bare := strings.TrimPrefix(id, "/hostedzone/")
+	require.NoError(t, svc.Store().MutateZone(ctx, acctA, bare, func(z *route53.ZoneRecord) (bool, error) {
+		z.State = route53.StateActive
+		return true, nil
+	}))
+
+	out, err := apply(svc, acctA, id, change("CREATE", "www.example.com", "A", 60, "192.0.2.1"))
+	require.NoError(t, err)
+	assert.Equal(t, route53.ChangeStatusPending, aws.StringValue(out.ChangeInfo.Status))
+
+	doc, _, err := svc.Store().GetRRSets(ctx, acctA, bare)
+	require.NoError(t, err)
+	require.NoError(t, svc.Store().MarkChangesInSync(ctx, acctA, bare, doc.Version))
+	got, err := svc.GetChange(ctx, &r53.GetChangeInput{Id: out.ChangeInfo.Id}, acctA)
+	require.NoError(t, err)
+	assert.Equal(t, route53.ChangeStatusInSync, aws.StringValue(got.ChangeInfo.Status))
+
+	_, err = svc.GetChange(ctx, &r53.GetChangeInput{Id: out.ChangeInfo.Id}, acctB)
+	requireCode(t, err, awserrors.ErrorNoSuchChange)
+}
+
+func TestUpdateHostedZoneComment(t *testing.T) {
+	svc := newService(t)
+	id := createZone(t, svc, acctA, "example.com")
+	out, err := svc.UpdateHostedZoneComment(ctx, &r53.UpdateHostedZoneCommentInput{Id: aws.String(id), Comment: aws.String("moved")}, acctA)
+	require.NoError(t, err)
+	assert.Equal(t, "moved", aws.StringValue(out.HostedZone.Config.Comment))
+	_, err = svc.UpdateHostedZoneComment(ctx, &r53.UpdateHostedZoneCommentInput{Id: aws.String(id), Comment: aws.String(strings.Repeat("x", 257))}, acctA)
+	requireCode(t, err, awserrors.ErrorInvalidInput)
+}
+
+func TestChangeRecordsArePrunedAfterRetention(t *testing.T) {
+	svc := newService(t)
+	now := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
+	svc.Now = func() time.Time { return now }
+	id := createZone(t, svc, acctA, "example.com")
+	first, err := apply(svc, acctA, id, change("CREATE", "a.example.com", "A", 60, "192.0.2.1"))
+	require.NoError(t, err)
+
+	now = now.Add(8 * 24 * time.Hour)
+	_, err = apply(svc, acctA, id, change("CREATE", "b.example.com", "A", 60, "192.0.2.1"))
+	require.NoError(t, err)
+	_, err = svc.GetChange(ctx, &r53.GetChangeInput{Id: first.ChangeInfo.Id}, acctA)
+	requireCode(t, err, awserrors.ErrorNoSuchChange)
+}

spinifex/domains/route53/store.go

@@ -0,0 +1,340 @@
+package route53
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// KVBucketRoute53 holds every hosted zone, its record sets and its changes.
+const (
+	KVBucketRoute53        = "spinifex-route53"
+	KVBucketRoute53Version = 1
+	// KVBucketRoute53Realize elects the one node that publishes zones.
+	KVBucketRoute53Realize = "spinifex-route53-realize"
+	// KVBucketRoute53Verify elects the one node that checks delegations.
+	KVBucketRoute53Verify = "spinifex-route53-verify"
+
+	keyPrefixZone     = "zone."
+	keyPrefixRRSets   = "rrsets."
+	keyPrefixChange   = "change."
+	keyPrefixCallerRf = "ref."
+	keyPrefixPublish  = "published."
+	casAttempts       = 20
+)
+
+// Activation states. A zone is served by northstar only while ACTIVE.
+const (
+	StatePendingNameservers = "PENDING_NAMESERVERS"
+	StateActive             = "ACTIVE"
+	StateNameserversLost    = "NAMESERVERS_LOST"
+	// StateSuspended is an operator's deactivation; only the operator lifts it.
+	StateSuspended = "SUSPENDED"
+)
+
+// Change statuses, as GetChange reports them.
+const (
+	ChangeStatusPending = "PENDING"
+	ChangeStatusInSync  = "INSYNC"
+)
+
+// ZoneRecord is a hosted zone's identity and configuration. Its record sets
+// live in a separate RRSetsDoc so a change batch is one CAS on one document.
+type ZoneRecord struct {
+	ID              string            `json:"id"`
+	AccountID       string            `json:"account_id"`
+	Name            string            `json:"name"`
+	CallerReference string            `json:"caller_reference"`
+	Comment         string            `json:"comment,omitempty"`
+	CreatedAt       time.Time         `json:"created_at"`
+	Nameservers     []string          `json:"nameservers"`
+	Tags            map[string]string `json:"tags,omitempty"`
+
+	State       string    `json:"state"`
+	StateReason string    `json:"state_reason,omitempty"`
+	CheckedAt   time.Time `json:"checked_at"`
+	// Delegation is the NS set the parent published at the last check, and
+	// DelegationSource the parent zone or server that answered.
+	Delegation       []string  `json:"delegation,omitempty"`
+	DelegationSource string    `json:"delegation_source,omitempty"`
+	FailingSince     time.Time `json:"failing_since,omitzero"`
+	// Override is set when an operator activated or suspended the zone; the
+	// verifier leaves such a zone alone.
+	Override       string `json:"override,omitempty"`
+	OverrideReason string `json:"override_reason,omitempty"`
+}
+
+// RRSetsDoc is a zone's complete record-set document. Version increases by one
+// on every committed change batch and is what a realized zone is stamped with.
+type RRSetsDoc struct {
+	ZoneID    string    `json:"zone_id"`
+	Version   uint64    `json:"version"`
+	UpdatedAt time.Time `json:"updated_at"`
+	Sets      []RRSet   `json:"sets"`
+}
+
+// RRSet is one simple-routing record set. Name is normalized.
+type RRSet struct {
+	Name   string   `json:"name"`
+	Type   string   `json:"type"`
+	TTL    int64    `json:"ttl"`
+	Values []string `json:"values"`
+}
+
+// ChangeRecord tracks one submitted change for GetChange.
+type ChangeRecord struct {
+	ID          string    `json:"id"`
+	AccountID   string    `json:"account_id"`
+	ZoneID      string    `json:"zone_id"`
+	Version     uint64    `json:"version"`
+	Status      string    `json:"status"`
+	Comment     string    `json:"comment,omitempty"`
+	SubmittedAt time.Time `json:"submitted_at"`
+}
+
+// PublishedZone records what the realizer last wrote to northstar for a zone
+// name. It is how a withdrawn zone is found, since its own records may be gone.
+type PublishedZone struct {
+	Apex      string `json:"apex"`
+	AccountID string `json:"account_id"`
+	ZoneID    string `json:"zone_id"`
+	Version   uint64 `json:"version"`
+	Digest    string `json:"digest"`
+	InSync    bool   `json:"in_sync"`
+}
+
+type callerRef struct {
+	ZoneID string `json:"zone_id"`
+}
+
+// Store reads and writes the route53 bucket. Keys are account-scoped, so an
+// owner lookup never reads another account's records.
+type Store struct {
+	bucket  *kvstore.Bucket
+	zones   *kvstore.Store[ZoneRecord]
+	rrsets  *kvstore.Store[RRSetsDoc]
+	changes *kvstore.Store[ChangeRecord]
+	refs    *kvstore.Store[callerRef]
+	publish *kvstore.Store[PublishedZone]
+}
+
+// NewStore returns a Store over js. The bucket is created on first use.
+func NewStore(js jetstream.JetStream) *Store {
+	b := kvstore.NewBucket(js, kvstore.Config{
+		Name:        KVBucketRoute53,
+		History:     KVBucketRoute53Version,
+		Description: "Route 53 hosted zones, record sets and changes",
+		Missing:     "route53 store has no JetStream connection",
+		Attempts:    casAttempts,
+		Exhausted: func(key string, attempts int) error {
+			return fmt.Errorf("route53 store: exceeded %d CAS attempts updating %s", attempts, key)
+		},
+	})
+	return &Store{
+		bucket:  b,
+		zones:   kvstore.On[ZoneRecord](b),
+		rrsets:  kvstore.On[RRSetsDoc](b),
+		changes: kvstore.On[ChangeRecord](b),
+		refs:    kvstore.On[callerRef](b),
+		publish: kvstore.On[PublishedZone](b),
+	}
+}
+
+func zoneKey(account, id string) string   { return keyPrefixZone + kvutil.AccountKey(account, id) }
+func rrsetsKey(account, id string) string { return keyPrefixRRSets + kvutil.AccountKey(account, id) }
+func changeKey(account, id string) string { return keyPrefixChange + kvutil.AccountKey(account, id) }
+func refKey(account, ref string) string {
+	return keyPrefixCallerRf + kvutil.AccountKey(account, refDigest(ref))
+}
+
+// GetZone returns an account's zone, or kvstore.ErrNotFound.
+func (s *Store) GetZone(ctx context.Context, account, id string) (*ZoneRecord, error) {
+	z, _, err := s.zones.Get(ctx, zoneKey(account, id))
+	return z, err
+}
+
+// ListAccountZones returns every zone the account owns.
+func (s *Store) ListAccountZones(ctx context.Context, account string) ([]ZoneRecord, error) {
+	return s.zones.List(ctx, keyPrefixZone+account+".")
+}
+
+// ListAllZones returns every zone in every account, for the realizer and the
+// verifier, which act across accounts.
+func (s *Store) ListAllZones(ctx context.Context) ([]ZoneRecord, error) {
+	return s.zones.List(ctx, keyPrefixZone)
+}
+
+// MutateZone applies fn to a zone under CAS.
+func (s *Store) MutateZone(ctx context.Context, account, id string, fn func(*ZoneRecord) (bool, error)) error {
+	return s.zones.Mutate(ctx, zoneKey(account, id), fn)
+}
+
+// GetRRSets returns a zone's record sets and the revision to CAS against.
+func (s *Store) GetRRSets(ctx context.Context, account, id string) (*RRSetsDoc, uint64, error) {
+	return s.rrsets.Get(ctx, rrsetsKey(account, id))
+}
+
+// CommitRRSets writes doc if the document is still at rev.
+func (s *Store) CommitRRSets(ctx context.Context, account string, doc *RRSetsDoc, rev uint64) error {
+	_, err := s.rrsets.CompareAndSet(ctx, rrsetsKey(account, doc.ZoneID), doc, rev)
+	return err
+}
+
+// GetChange returns an account's change, or kvstore.ErrNotFound.
+func (s *Store) GetChange(ctx context.Context, account, id string) (*ChangeRecord, error) {
+	c, _, err := s.changes.Get(ctx, changeKey(account, id))
+	return c, err
+}
+
+// PutChange stores a new change.
+func (s *Store) PutChange(ctx context.Context, c *ChangeRecord) error {
+	_, err := s.changes.Create(ctx, changeKey(c.AccountID, c.ID), c)
+	return err
+}
+
+// ListZoneChanges returns every change recorded for one zone.
+func (s *Store) ListZoneChanges(ctx context.Context, account, zoneID string) ([]ChangeRecord, error) {
+	all, err := s.changes.List(ctx, keyPrefixChange+account+".")
+	if err != nil {
+		return nil, err
+	}
+	out := all[:0]
+	for _, c := range all {
+		if c.ZoneID == zoneID {
+			out = append(out, c)
+		}
+	}
+	return out, nil
+}
+
+// MarkChangesInSync moves every PENDING change for a zone at or below version
+// to INSYNC.
+func (s *Store) MarkChangesInSync(ctx context.Context, account, zoneID string, version uint64) error {
+	changes, err := s.ListZoneChanges(ctx, account, zoneID)
+	if err != nil {
+		return err
+	}
+	for _, c := range changes {
+		if c.Status != ChangeStatusPending || c.Version > version {
+			continue
+		}
+		err := s.changes.Mutate(ctx, changeKey(account, c.ID), func(rec *ChangeRecord) (bool, error) {
+			if rec.Status == ChangeStatusInSync {
+				return false, nil
+			}
+			rec.Status = ChangeStatusInSync
+			return true, nil
+		})
+		if err != nil && !errors.Is(err, kvstore.ErrNotFound) {
+			return err
+		}
+	}
+	return nil
+}
+
+// ClaimCallerReference records that ref created zoneID. It returns the zone
+// already holding ref when the claim loses.
+func (s *Store) ClaimCallerReference(ctx context.Context, account, ref, zoneID string) (existing string, err error) {
+	if _, err := s.refs.Create(ctx, refKey(account, ref), &callerRef{ZoneID: zoneID}); err != nil {
+		if !errors.Is(err, kvstore.ErrExists) {
+			return "", err
+		}
+		held, _, getErr := s.refs.Get(ctx, refKey(account, ref))
+		if getErr != nil {
+			return "", getErr
+		}
+		return held.ZoneID, nil
+	}
+	return "", nil
+}
+
+// CreateZone writes a new zone and its initial record sets.
+func (s *Store) CreateZone(ctx context.Context, z *ZoneRecord, doc *RRSetsDoc) error {
+	if _, err := s.rrsets.Create(ctx, rrsetsKey(z.AccountID, z.ID), doc); err != nil {
+		return err
+	}
+	if _, err := s.zones.Create(ctx, zoneKey(z.AccountID, z.ID), z); err != nil {
+		_ = s.bucket.Delete(ctx, rrsetsKey(z.AccountID, z.ID))
+		return err
+	}
+	return nil
+}
+
+// DeleteZone removes a zone whose record sets are still at rev, returning
+// kvstore.ErrConflict if a change landed since they were read. The record-set
+// document goes first, so a concurrent change batch cannot commit into a zone
+// that is being deleted; the zone's earlier changes go with it.
+func (s *Store) DeleteZone(ctx context.Context, z *ZoneRecord, rev uint64) error {
+	if err := s.bucket.CompareAndDelete(ctx, rrsetsKey(z.AccountID, z.ID), rev); err != nil {
+		return err
+	}
+	if err := s.bucket.Delete(ctx, zoneKey(z.AccountID, z.ID)); err != nil {
+		return err
+	}
+	if err := s.bucket.Delete(ctx, refKey(z.AccountID, z.CallerReference)); err != nil {
+		return err
+	}
+	return s.PruneChanges(ctx, z.AccountID, z.ID, time.Time{}, true)
+}
+
+// PruneChanges deletes a zone's INSYNC changes submitted before cutoff, or all
+// of them when all is set. A PENDING change is kept until it is in sync.
+func (s *Store) PruneChanges(ctx context.Context, account, zoneID string, cutoff time.Time, all bool) error {
+	changes, err := s.ListZoneChanges(ctx, account, zoneID)
+	if err != nil {
+		return err
+	}
+	for _, c := range changes {
+		if !all && (c.Status != ChangeStatusInSync || !c.SubmittedAt.Before(cutoff)) {
+			continue
+		}
+		if err := s.bucket.Delete(ctx, changeKey(account, c.ID)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// ReleaseCallerReference drops a claim whose zone was never created, so the
+// caller can retry with the same reference.
+func (s *Store) ReleaseCallerReference(ctx context.Context, account, ref string) error {
+	return s.bucket.Delete(ctx, refKey(account, ref))
+}
+
+// ListAllChanges returns every change in every account.
+func (s *Store) ListAllChanges(ctx context.Context) ([]ChangeRecord, error) {
+	return s.changes.List(ctx, keyPrefixChange)
+}
+
+// DeleteChange removes one change.
+func (s *Store) DeleteChange(ctx context.Context, account, id string) error {
+	return s.bucket.Delete(ctx, changeKey(account, id))
+}
+
+// ListPublished returns the published zones keyed by apex.
+func (s *Store) ListPublished(ctx context.Context) (map[string]PublishedZone, error) {
+	all, err := s.publish.List(ctx, keyPrefixPublish)
+	if err != nil {
+		return nil, err
+	}
+	out := make(map[string]PublishedZone, len(all))
+	for _, p := range all {
+		out[p.Apex] = p
+	}
+	return out, nil
+}
+
+// SetPublished records what was written for p.Apex.
+func (s *Store) SetPublished(ctx context.Context, p *PublishedZone) error {
+	return s.publish.Set(ctx, keyPrefixPublish+p.Apex, p)
+}
+
+// DeletePublished forgets a withdrawn zone.
+func (s *Store) DeletePublished(ctx context.Context, apex string) error {
+	return s.bucket.Delete(ctx, keyPrefixPublish+apex)
+}

spinifex/domains/route53/txt.go

@@ -0,0 +1,91 @@
+package route53
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+)
+
+const maxCharacterString = 255
+
+// characterStrings splits a TXT or SPF value into its character-strings, kept
+// in presentation format so their escapes reach the wire unchanged. A value
+// with no opening quote is one string, as Route 53 treats it.
+func characterStrings(v string) ([]string, error) {
+	v = strings.TrimSpace(v)
+	if !strings.HasPrefix(v, `"`) {
+		n, err := decodedLength(v)
+		if err != nil {
+			return nil, err
+		}
+		if n > maxCharacterString {
+			return nil, errTXTTooLong
+		}
+		return []string{v}, nil
+	}
+	var out []string
+	for v != "" {
+		if v[0] != '"' {
+			return nil, errUnquoted
+		}
+		end := closingQuote(v)
+		if end < 0 {
+			return nil, errUnquoted
+		}
+		s := v[1:end]
+		n, err := decodedLength(s)
+		if err != nil {
+			return nil, err
+		}
+		if n > maxCharacterString {
+			return nil, errTXTTooLong
+		}
+		out = append(out, s)
+		v = strings.TrimLeft(v[end+1:], " \t")
+	}
+	return out, nil
+}
+
+var (
+	errTXTTooLong = errors.New("TXTRDATATooLong (TXT RDATA string too long)")
+	errUnquoted   = errors.New("InvalidCharacterString (Value should be enclosed in quotation marks)")
+)
+
+// closingQuote returns the index of the quote closing the string opened at
+// v[0], skipping escaped characters, or -1.
+func closingQuote(v string) int {
+	for i := 1; i < len(v); i++ {
+		switch v[i] {
+		case '\\':
+			i++
+		case '"':
+			return i
+		}
+	}
+	return -1
+}
+
+// decodedLength is the wire length of a presentation-format string, in which
+// \ddd is one byte and \c is the character c.
+func decodedLength(s string) (int, error) {
+	n := 0
+	for i := 0; i < len(s); i++ {
+		if s[i] == '\\' {
+			switch {
+			case i+3 < len(s) && isDigit(s[i+1]) && isDigit(s[i+2]) && isDigit(s[i+3]):
+				if int(s[i+1]-'0')*100+int(s[i+2]-'0')*10+int(s[i+3]-'0') > 255 {
+					return 0, fmt.Errorf("escape \\%s is not a byte", s[i+1:i+4])
+				}
+				i += 3
+			case i+1 < len(s):
+				i++
+			default:
+				return 0, errors.New("value ends in a bare backslash")
+			}
+		}
+		n++
+	}
+	return n, nil
+}
+
+func isDigit(c byte) bool { return c >= '0' && c <= '9' }

spinifex/domains/route53/verifier.go

@@ -0,0 +1,296 @@
+package route53
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/reconciler"
+)
+
+const (
+	defaultVerifyInterval = time.Minute
+	// A new zone is checked every pass for this long, while its owner is most
+	// likely changing the registrar, and hourly after it.
+	newZoneWindow = 48 * time.Hour
+	slowRecheck   = time.Hour
+	activeRecheck = 24 * time.Hour
+	// lostAfter is how long an active zone's delegation may fail before the
+	// zone stops being served.
+	lostAfter = 72 * time.Hour
+
+	reasonContested = "the domain is active in another account"
+)
+
+// Delegation is the NS set a parent zone publishes for a name.
+type Delegation struct {
+	Source  string
+	Servers []string
+}
+
+// Verifier activates a zone once its parent delegates it to this cluster's
+// name servers, and stops serving one whose delegation has moved away.
+type Verifier struct {
+	store *Store
+
+	// Nameservers are this cluster's published name servers.
+	Nameservers func() []string
+	// Platform reads a delegation from a platform zone this cluster serves; ok
+	// is false when the name lies under none of them.
+	Platform func(ctx context.Context, name string) (d Delegation, ok bool, err error)
+	// Public asks the parent's authoritative servers, without recursion.
+	Public func(ctx context.Context, name string) (Delegation, error)
+	// Elect wins this pass for the calling node; nil means always.
+	Elect func(ctx context.Context) (release func(), ok bool)
+	Now   func() time.Time
+	// Interval is the pass period; zero means one minute.
+	Interval time.Duration
+}
+
+// NewVerifier returns a Verifier over store.
+func NewVerifier(store *Store) *Verifier {
+	return &Verifier{store: store, Now: time.Now}
+}
+
+// Run verifies on every interval. Nothing in KV changes when a registrar does,
+// so the loop is driven by time alone.
+func (v *Verifier) Run(ctx context.Context) {
+	interval := v.Interval
+	if interval <= 0 {
+		interval = defaultVerifyInterval
+	}
+	reconciler.Run(ctx, reconciler.Config{Name: "route53-verify", Reconcile: v.Reconcile, Resync: interval})
+}
+
+// Reconcile checks every zone that is due.
+func (v *Verifier) Reconcile(ctx context.Context) (time.Duration, error) {
+	if v.Elect != nil {
+		release, ok := v.Elect(ctx)
+		if !ok {
+			return 0, nil
+		}
+		defer release()
+	}
+	zones, err := v.store.ListAllZones(ctx)
+	if err != nil {
+		return 0, fmt.Errorf("list hosted zones: %w", err)
+	}
+	now := v.Now().UTC()
+	var due []*ZoneRecord
+	for i := range zones {
+		if checkDue(&zones[i], now) {
+			due = append(due, &zones[i])
+		}
+	}
+	return 0, v.verify(ctx, zones, due, now)
+}
+
+// Verify checks one zone now, whatever its schedule, and returns it updated.
+func (v *Verifier) Verify(ctx context.Context, account, id string) (*ZoneRecord, error) {
+	zones, err := v.store.ListAllZones(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("list hosted zones: %w", err)
+	}
+	i := slices.IndexFunc(zones, func(z ZoneRecord) bool { return z.AccountID == account && z.ID == id })
+	if i < 0 {
+		return v.store.GetZone(ctx, account, id) // reports not-found the usual way
+	}
+	if zones[i].Override == "" {
+		if err := v.verify(ctx, zones, []*ZoneRecord{&zones[i]}, v.Now().UTC()); err != nil {
+			return nil, err
+		}
+	}
+	return v.store.GetZone(ctx, account, id)
+}
+
+// checkDue reports whether a zone's next check has come round.
+func checkDue(z *ZoneRecord, now time.Time) bool {
+	if z.Override != "" || z.State == StateSuspended {
+		return false
+	}
+	var every time.Duration
+	switch {
+	case z.State == StateActive && z.FailingSince.IsZero():
+		every = activeRecheck
+	case z.State == StateActive:
+		every = slowRecheck
+	case now.Sub(z.CreatedAt) >= newZoneWindow:
+		every = slowRecheck
+	}
+	return now.Sub(z.CheckedAt) >= every
+}
+
+type outcome struct {
+	zone   *ZoneRecord
+	d      Delegation
+	reason string // empty when the delegation passes
+	err    error  // the check itself could not be made
+}
+
+func (v *Verifier) verify(ctx context.Context, all []ZoneRecord, due []*ZoneRecord, now time.Time) error {
+	ours := v.ourNameservers()
+	results := make([]outcome, 0, len(due))
+	for _, z := range due {
+		d, err := v.delegation(ctx, all, z)
+		o := outcome{zone: z, d: d, err: err}
+		if err == nil {
+			o.reason = compareDelegation(z.Name, d, ours)
+		}
+		results = append(results, o)
+	}
+
+	// First match wins: the earliest-created zone of a name that passes is the
+	// one activated, and no zone activates while another of its name is active.
+	slices.SortStableFunc(results, func(a, b outcome) int { return olderFirst(a.zone, b.zone) })
+	active := map[string]string{}
+	for _, z := range all {
+		if z.State == StateActive {
+			active[z.Name] = z.AccountID + "/" + z.ID
+		}
+	}
+
+	var errs []error
+	for _, o := range results {
+		self := o.zone.AccountID + "/" + o.zone.ID
+		contested := false
+		if holder, ok := active[o.zone.Name]; ok && holder != self {
+			contested = true
+		}
+		err := v.store.MutateZone(ctx, o.zone.AccountID, o.zone.ID, func(z *ZoneRecord) (bool, error) {
+			if z.Override != "" || z.State == StateSuspended {
+				return false, nil
+			}
+			applyOutcome(z, o, contested, now)
+			return true, nil
+		})
+		if err != nil {
+			errs = append(errs, fmt.Errorf("record check of %s: %w", o.zone.Name, err))
+			continue
+		}
+		if o.err == nil && o.reason == "" && !contested {
+			if _, ok := active[o.zone.Name]; !ok {
+				active[o.zone.Name] = self
+				slog.Info("route53: zone activated", "zone", o.zone.Name, "zone_id", o.zone.ID, "account", o.zone.AccountID, "parent", o.d.Source)
+			}
+		}
+	}
+	return errors.Join(errs...)
+}
+
+// applyOutcome moves a zone's activation state for one check.
+func applyOutcome(z *ZoneRecord, o outcome, contested bool, now time.Time) {
+	z.CheckedAt = now
+	if o.err != nil {
+		// Not knowing is not failing: an unreachable parent moves nothing.
+		z.StateReason = "the delegation could not be checked: " + o.err.Error()
+		return
+	}
+	z.Delegation, z.DelegationSource = o.d.Servers, o.d.Source
+	switch {
+	case o.reason != "":
+		z.StateReason = o.reason
+		if z.State == StateActive {
+			if z.FailingSince.IsZero() {
+				z.FailingSince = now
+			}
+			if now.Sub(z.FailingSince) >= lostAfter {
+				z.State = StateNameserversLost
+				slog.Warn("route53: zone lost its delegation", "zone", z.Name, "zone_id", z.ID, "reason", o.reason)
+			}
+		}
+	case z.State == StateActive:
+		z.StateReason, z.FailingSince = "", time.Time{}
+	case contested:
+		z.StateReason = reasonContested
+	default:
+		z.State, z.StateReason, z.FailingSince = StateActive, "", time.Time{}
+	}
+}
+
+// delegation finds the parent of z and reads what it publishes. A parent this
+// cluster serves is read from its records, never through DNS.
+func (v *Verifier) delegation(ctx context.Context, all []ZoneRecord, z *ZoneRecord) (Delegation, error) {
+	if parent := enclosingZone(all, z); parent != nil {
+		doc, _, err := v.store.GetRRSets(ctx, parent.AccountID, parent.ID)
+		if err != nil {
+			return Delegation{}, fmt.Errorf("read the parent zone %s: %w", parent.Name, err)
+		}
+		d := Delegation{Source: "hosted zone " + strings.TrimSuffix(parent.Name, ".")}
+		for _, set := range doc.Sets {
+			if set.Name == z.Name && set.Type == "NS" {
+				d.Servers = set.Values
+			}
+		}
+		return d, nil
+	}
+	if v.Platform != nil {
+		d, ok, err := v.Platform(ctx, z.Name)
+		if err != nil || ok {
+			return d, err
+		}
+	}
+	if v.Public == nil {
+		return Delegation{}, errors.New("no public DNS lookup is configured")
+	}
+	return v.Public(ctx, z.Name)
+}
+
+// enclosingZone returns the most specific active zone strictly above z, from
+// any account: that zone, not the public parent, decides who holds z.
+func enclosingZone(all []ZoneRecord, z *ZoneRecord) *ZoneRecord {
+	var best *ZoneRecord
+	for i := range all {
+		p := &all[i]
+		if p.State != StateActive || p.Name == z.Name || !withinZone(z.Name, p.Name) {
+			continue
+		}
+		if best == nil || len(p.Name) > len(best.Name) || (p.Name == best.Name && olderFirst(p, best) < 0) {
+			best = p
+		}
+	}
+	return best
+}
+
+func (v *Verifier) ourNameservers() map[string]bool {
+	ours := map[string]bool{}
+	if v.Nameservers != nil {
+		for _, ns := range v.Nameservers() {
+			ours[normalizeName(ns)] = true
+		}
+	}
+	return ours
+}
+
+// compareDelegation returns why d does not delegate name to this cluster, or ""
+// when it does. Every listed server must be ours: with a foreign one in the
+// set, a share of resolvers would get answers we do not control.
+func compareDelegation(name string, d Delegation, ours map[string]bool) string {
+	if len(ours) == 0 {
+		return "this cluster publishes no name servers"
+	}
+	if len(d.Servers) == 0 {
+		return fmt.Sprintf("%s does not delegate %s", d.Source, strings.TrimSuffix(name, "."))
+	}
+	var foreign []string
+	for _, s := range d.Servers {
+		if n := normalizeName(s); !ours[n] {
+			foreign = append(foreign, strings.TrimSuffix(n, "."))
+		}
+	}
+	if len(foreign) > 0 {
+		slices.Sort(foreign)
+		return fmt.Sprintf("%s delegates %s to name servers that are not this cluster's: %s", d.Source, strings.TrimSuffix(name, "."), strings.Join(foreign, ", "))
+	}
+	return ""
+}
+
+func olderFirst(a, b *ZoneRecord) int {
+	if c := a.CreatedAt.Compare(b.CreatedAt); c != 0 {
+		return c
+	}
+	return strings.Compare(a.ID, b.ID)
+}

spinifex/domains/route53/verifier_test.go

@@ -0,0 +1,274 @@
+package route53_test
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	r53 "github.com/aws/aws-sdk-go/service/route53"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/route53"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+)
+
+// fakeDNS is the public DNS tree: what each name's parent publishes for it.
+type fakeDNS struct {
+	parents map[string][]string
+	lookups int
+	err     error
+}
+
+func (f *fakeDNS) lookup(_ context.Context, name string) (route53.Delegation, error) {
+	f.lookups++
+	if f.err != nil {
+		return route53.Delegation{}, f.err
+	}
+	return route53.Delegation{Source: "the parent's name servers", Servers: f.parents[strings.TrimSuffix(name, ".")]}, nil
+}
+
+type clock struct{ t time.Time }
+
+func (c *clock) now() time.Time          { return c.t }
+func (c *clock) advance(d time.Duration) { c.t = c.t.Add(d) }
+func newClock() *clock                   { return &clock{t: time.Date(2026, 10, 24, 0, 0, 0, 0, time.UTC)} }
+func ours() []string                     { return []string{"ns1.spx3.net", "ns2.spx3.net"} }
+func zoneID(id string) string            { return strings.TrimPrefix(id, "/hostedzone/") }
+
+func newVerifier(t *testing.T) (*route53.Service, *route53.Verifier, *fakeDNS, *clock) {
+	t.Helper()
+	svc := newService(t)
+	c := newClock()
+	svc.Now = c.now
+	dns := &fakeDNS{parents: map[string][]string{}}
+	v := route53.NewVerifier(svc.Store())
+	v.Nameservers = ours
+	v.Public = dns.lookup
+	v.Now = c.now
+	return svc, v, dns, c
+}
+
+func zone(t *testing.T, svc *route53.Service, account, id string) *route53.ZoneRecord {
+	t.Helper()
+	z, err := svc.Store().GetZone(ctx, account, zoneID(id))
+	require.NoError(t, err)
+	return z
+}
+
+func pass(t *testing.T, v *route53.Verifier) {
+	t.Helper()
+	_, err := v.Reconcile(ctx)
+	require.NoError(t, err)
+}
+
+func TestVerifierActivatesOnlyAFullDelegationToUs(t *testing.T) {
+	svc, v, dns, _ := newVerifier(t)
+	cases := map[string]struct {
+		parent []string
+		state  string
+		reason string
+	}{
+		"all.example":     {[]string{"NS1.spx3.net.", "ns2.spx3.net"}, route53.StateActive, ""},
+		"subset.example":  {[]string{"ns2.spx3.net."}, route53.StateActive, ""},
+		"mixed.example":   {[]string{"ns1.spx3.net.", "ns-1.awsdns-01.org."}, route53.StatePendingNameservers, "not this cluster's: ns-1.awsdns-01.org"},
+		"foreign.example": {[]string{"dana.ns.cloudflare.com."}, route53.StatePendingNameservers, "not this cluster's"},
+		"none.example":    {nil, route53.StatePendingNameservers, "does not delegate none.example"},
+	}
+	ids := map[string]string{}
+	for name, tc := range cases {
+		dns.parents[name] = tc.parent
+		ids[name] = createZone(t, svc, acctA, name)
+	}
+	pass(t, v)
+	for name, tc := range cases {
+		z := zone(t, svc, acctA, ids[name])
+		assert.Equal(t, tc.state, z.State, name)
+		assert.Contains(t, z.StateReason, tc.reason, name)
+		assert.Equal(t, tc.parent, z.Delegation, name)
+	}
+}
+
+func TestVerifierFirstMatchWins(t *testing.T) {
+	svc, v, dns, c := newVerifier(t)
+	dns.parents["example.com"] = ours()
+	first := createZone(t, svc, acctA, "example.com")
+	c.advance(time.Second)
+	second := createZone(t, svc, acctB, "example.com")
+	pass(t, v)
+
+	assert.Equal(t, route53.StateActive, zone(t, svc, acctA, first).State)
+	later := zone(t, svc, acctB, second)
+	assert.Equal(t, route53.StatePendingNameservers, later.State)
+	assert.Equal(t, "the domain is active in another account", later.StateReason)
+
+	// The claim is not released by the loser checking again.
+	c.advance(time.Minute)
+	pass(t, v)
+	assert.Equal(t, route53.StatePendingNameservers, zone(t, svc, acctB, second).State)
+}
+
+func TestVerifierScheduleBacksOffAfterTheFirstTwoDays(t *testing.T) {
+	svc, v, dns, c := newVerifier(t)
+	createZone(t, svc, acctA, "slow.example")
+	pass(t, v)
+	require.Equal(t, 1, dns.lookups)
+
+	c.advance(time.Minute)
+	pass(t, v)
+	assert.Equal(t, 2, dns.lookups, "a new zone is checked every pass")
+
+	c.advance(48 * time.Hour)
+	pass(t, v)
+	c.advance(30 * time.Minute)
+	pass(t, v)
+	assert.Equal(t, 3, dns.lookups, "an old pending zone is checked hourly")
+	c.advance(30 * time.Minute)
+	pass(t, v)
+	assert.Equal(t, 4, dns.lookups)
+}
+
+func TestVerifierLosesAZoneOnlyAfterThreeDaysOfFailure(t *testing.T) {
+	svc, v, dns, c := newVerifier(t)
+	dns.parents["moved.example"] = ours()
+	id := createZone(t, svc, acctA, "moved.example")
+	pass(t, v)
+	require.Equal(t, route53.StateActive, zone(t, svc, acctA, id).State)
+
+	// The owner moves the domain away; the daily re-check notices.
+	dns.parents["moved.example"] = []string{"ns-1.awsdns-01.org."}
+	c.advance(24 * time.Hour)
+	pass(t, v)
+	z := zone(t, svc, acctA, id)
+	assert.Equal(t, route53.StateActive, z.State, "one failed check is not a lost delegation")
+	assert.Equal(t, c.t, z.FailingSince)
+
+	c.advance(71 * time.Hour)
+	pass(t, v)
+	assert.Equal(t, route53.StateActive, zone(t, svc, acctA, id).State)
+	c.advance(time.Hour)
+	pass(t, v)
+	assert.Equal(t, route53.StateNameserversLost, zone(t, svc, acctA, id).State)
+
+	// It comes back when the delegation does.
+	dns.parents["moved.example"] = ours()
+	c.advance(time.Hour)
+	pass(t, v)
+	z = zone(t, svc, acctA, id)
+	assert.Equal(t, route53.StateActive, z.State)
+	assert.True(t, z.FailingSince.IsZero())
+}
+
+func TestVerifierTreatsAnUnreachableParentAsUnknown(t *testing.T) {
+	svc, v, dns, c := newVerifier(t)
+	dns.parents["flaky.example"] = ours()
+	id := createZone(t, svc, acctA, "flaky.example")
+	pass(t, v)
+
+	dns.err = errors.New("i/o timeout")
+	for range 5 {
+		c.advance(24 * time.Hour)
+		pass(t, v)
+	}
+	z := zone(t, svc, acctA, id)
+	assert.Equal(t, route53.StateActive, z.State)
+	assert.True(t, z.FailingSince.IsZero())
+	assert.Contains(t, z.StateReason, "could not be checked: i/o timeout")
+}
+
+// A subdomain of an active zone is decided by that zone's records, never by
+// public DNS, so account B cannot take a name under account A's domain.
+func TestVerifierReadsAnActiveParentZoneLocally(t *testing.T) {
+	svc, v, dns, c := newVerifier(t)
+	dns.parents["example.com"] = ours()
+	dns.parents["shop.example.com"] = ours() // what a lookup would wrongly accept
+	parent := createZone(t, svc, acctA, "example.com")
+	pass(t, v)
+	require.Equal(t, route53.StateActive, zone(t, svc, acctA, parent).State)
+
+	child := createZone(t, svc, acctB, "shop.example.com")
+	c.advance(time.Minute)
+	pass(t, v)
+	z := zone(t, svc, acctB, child)
+	assert.Equal(t, route53.StatePendingNameservers, z.State)
+	assert.Equal(t, "hosted zone example.com does not delegate shop.example.com", z.StateReason)
+
+	_, err := apply(svc, acctA, parent, change("CREATE", "shop.example.com", "NS", 300, "ns1.spx3.net.", "ns2.spx3.net."))
+	require.NoError(t, err)
+	c.advance(time.Minute)
+	pass(t, v)
+	z = zone(t, svc, acctB, child)
+	assert.Equal(t, route53.StateActive, z.State)
+	assert.Equal(t, "hosted zone example.com", z.DelegationSource)
+}
+
+func TestVerifierReadsAPlatformZoneLocally(t *testing.T) {
+	svc, v, dns, _ := newVerifier(t)
+	v.Platform = func(_ context.Context, name string) (route53.Delegation, bool, error) {
+		if !strings.HasSuffix(name, ".spx3.net.") {
+			return route53.Delegation{}, false, nil
+		}
+		return route53.Delegation{Source: "platform zone spx3.net", Servers: ours()}, true, nil
+	}
+	id := createZone(t, svc, acctA, "app.spx3.net")
+	pass(t, v)
+	assert.Equal(t, route53.StateActive, zone(t, svc, acctA, id).State)
+	assert.Zero(t, dns.lookups)
+}
+
+func TestVerifierLeavesOverriddenZonesAlone(t *testing.T) {
+	svc, v, dns, c := newVerifier(t)
+	dns.parents["held.example"] = ours()
+	id := createZone(t, svc, acctA, "held.example")
+	require.NoError(t, svc.Store().MutateZone(ctx, acctA, zoneID(id), func(z *route53.ZoneRecord) (bool, error) {
+		z.State, z.Override = route53.StateSuspended, route53.StateSuspended
+		return true, nil
+	}))
+	pass(t, v)
+	c.advance(time.Hour)
+	_, err := v.Verify(ctx, acctA, zoneID(id))
+	require.NoError(t, err)
+	assert.Equal(t, route53.StateSuspended, zone(t, svc, acctA, id).State)
+	assert.Zero(t, dns.lookups)
+}
+
+func TestVerifyChecksImmediately(t *testing.T) {
+	svc, v, dns, _ := newVerifier(t)
+	id := createZone(t, svc, acctA, "now.example")
+	pass(t, v)
+	dns.parents["now.example"] = ours()
+
+	z, err := v.Verify(ctx, acctA, zoneID(id))
+	require.NoError(t, err)
+	assert.Equal(t, route53.StateActive, z.State)
+
+	_, err = v.Verify(ctx, acctB, zoneID(id))
+	require.ErrorIs(t, err, kvstore.ErrNotFound, "another account's zone is not found")
+}
+
+func TestVerifierNeedsPublishedNameservers(t *testing.T) {
+	svc, v, dns, _ := newVerifier(t)
+	v.Nameservers = func() []string { return nil }
+	dns.parents["any.example"] = ours()
+	id := createZone(t, svc, acctA, "any.example")
+	pass(t, v)
+	z := zone(t, svc, acctA, id)
+	assert.Equal(t, route53.StatePendingNameservers, z.State)
+	assert.Equal(t, "this cluster publishes no name servers", z.StateReason)
+}
+
+func TestActivatedZoneChangesWaitForSync(t *testing.T) {
+	svc, v, dns, _ := newVerifier(t)
+	dns.parents["sync.example"] = ours()
+	id := createZone(t, svc, acctA, "sync.example")
+	pass(t, v)
+	out, err := svc.ChangeResourceRecordSets(ctx, &r53.ChangeResourceRecordSetsInput{
+		HostedZoneId: aws.String(id),
+		ChangeBatch:  &r53.ChangeBatch{Changes: []*r53.Change{change("CREATE", "www.sync.example", "A", 60, "192.0.2.1")}},
+	}, acctA)
+	require.NoError(t, err)
+	assert.Equal(t, "PENDING", aws.StringValue(out.ChangeInfo.Status))
+}

spinifex/foundation/aws/errors/awserrors.go

@@ -409,6 +409,19 @@ var (
 	// that exists but has not been issued yet (still PENDING_VALIDATION), so a
 	// client polling for the body can tell "not ready" apart from "not mine".
 	ErrorACMRequestInProgress = "RequestInProgressException"
+	// Route 53 REST-XML error codes, from the 2013-04-01 API reference.
+	ErrorNoSuchHostedZone          = "NoSuchHostedZone"
+	ErrorHostedZoneAlreadyExists   = "HostedZoneAlreadyExists"
+	ErrorHostedZoneNotEmpty        = "HostedZoneNotEmpty"
+	ErrorInvalidChangeBatch        = "InvalidChangeBatch"
+	ErrorNoSuchChange              = "NoSuchChange"
+	ErrorTooManyHostedZones        = "TooManyHostedZones"
+	ErrorInvalidDomainName         = "InvalidDomainName"
+	ErrorConflictingDomainExists   = "ConflictingDomainExists"
+	ErrorPriorRequestNotComplete   = "PriorRequestNotComplete"
+	ErrorRoute53InvalidArgument    = "InvalidArgument"
+	ErrorRoute53NoSuchHealthCheck  = "NoSuchHealthCheck"
+	ErrorRoute53TooManyTagsForZone = "TooManyTags"
 	// ECS JSON-1.1 exception codes. ECS clients key on the "Exception"-suffixed
 	// __type; do not cross-wire with the EC2/ACM not-found codes above.
 	ErrorECSClusterNotFound                             = "ClusterNotFoundException"
@@ -742,12 +755,16 @@ var errorLookupByService = withNeutralValidationMessages(map[string]map[string]E
 	"rds": {
 		ErrorOperationNotSupported: {HTTPCode: 400, Message: "The specified RDS action is not supported in the RDS v1 API."},
 	},
+	// Route 53 throttles with a 400, not the 503 the shared entry carries.
+	"route53": {
+		ErrorThrottling: {HTTPCode: 400, Message: "Rate exceeded."},
+	},
 })
 
 // nonEC2Services are the gateway's services other than EC2, whose validation
 // errors must not fall back to ErrorLookup's EC2 wording.
 var nonEC2Services = []string{
-	"iam", "sts", "elasticloadbalancing", "eks", "ecs", "ecr", "acm", "rds", "tagging", "spinifex",
+	"iam", "sts", "elasticloadbalancing", "eks", "ecs", "ecr", "acm", "rds", "route53", "tagging", "spinifex",
 	"bedrock", "bedrock-runtime", "bedrock-agent", "bedrock-agent-runtime",
 }
 
@@ -1349,4 +1366,18 @@ var ErrorLookup = map[string]ErrorMessage{
 	ErrorModelErrorException:           {HTTPCode: 424, Message: "The request failed because of an error while running the model."},
 	ErrorServiceQuotaExceededException: {HTTPCode: 400, Message: "The number of requests exceeds the service quota. Resubmit your request later."},
 	ErrorConflictException:             {HTTPCode: 409, Message: "There was a conflict performing an operation."},
+
+	// Route 53.
+	ErrorNoSuchHostedZone:          {HTTPCode: 404, Message: "No hosted zone exists with the ID that you specified."},
+	ErrorHostedZoneAlreadyExists:   {HTTPCode: 409, Message: "The hosted zone you're trying to create already exists."},
+	ErrorHostedZoneNotEmpty:        {HTTPCode: 400, Message: "The hosted zone contains resource records that are not SOA or NS records."},
+	ErrorInvalidChangeBatch:        {HTTPCode: 400, Message: "This exception contains a list of messages that might contain one or more error messages."},
+	ErrorNoSuchChange:              {HTTPCode: 404, Message: "A change with the specified change ID does not exist."},
+	ErrorTooManyHostedZones:        {HTTPCode: 400, Message: "This operation can't be completed because the current account has reached the limit on the number of hosted zones."},
+	ErrorInvalidDomainName:         {HTTPCode: 400, Message: "The specified domain name is not valid."},
+	ErrorConflictingDomainExists:   {HTTPCode: 400, Message: "The domain name is already in use by another hosted zone."},
+	ErrorPriorRequestNotComplete:   {HTTPCode: 400, Message: "Route 53 is still processing a previous request for this hosted zone. Try again later."},
+	ErrorRoute53InvalidArgument:    {HTTPCode: 400, Message: "Parameter name is not valid."},
+	ErrorRoute53NoSuchHealthCheck:  {HTTPCode: 404, Message: "No health check exists with the specified ID."},
+	ErrorRoute53TooManyTagsForZone: {HTTPCode: 400, Message: "The resource has too many tags."},
 }

spinifex/gateway/auth.go

@@ -158,9 +158,9 @@ func (gw *GatewayConfig) SigV4AuthMiddleware() func(http.Handler) http.Handler {
 				return
 			}
 
-			// Region is pinned to the gateway; service is the client-claimed scope,
-			// already gated against supportedServices above.
-			if _, err := sig.Verify(secret, gw.Region, sig.Credential.Service); err != nil {
+			// Region is the gateway's, or a global service's fixed signing region;
+			// service is the client-claimed scope, gated against supportedServices above.
+			if _, err := sig.Verify(secret, gw.expectedSigningRegion(sig.Credential.Service, sig.Credential.Region), sig.Credential.Service); err != nil {
 				// A mismatch means our canonical request differs from the one the client
 				// signed, so log ours to diff against the SDK's. Warn, not Debug: these
 				// failures are intermittent and never reproduce on demand.
@@ -558,3 +558,20 @@ func signedService(r *http.Request) string {
 	svc, _ := r.Context().Value(ctxService).(string)
 	return svc
 }
+
+// globalSigningRegions maps a global AWS service to the region its clients sign
+// with when they use the default endpoint. A client given an endpoint override
+// signs with its own configured region instead, so both are accepted.
+var globalSigningRegions = map[string]string{
+	"route53": "us-east-1",
+}
+
+// expectedSigningRegion returns the credential-scope region a request for
+// service must carry: the global service's signing region when the client
+// claimed it, else the gateway's own region.
+func (gw *GatewayConfig) expectedSigningRegion(service, claimed string) string {
+	if global, ok := globalSigningRegions[service]; ok && claimed == global {
+		return global
+	}
+	return gw.Region
+}

spinifex/gateway/auth_test.go

@@ -2580,14 +2580,14 @@ func TestRedactedCanonicalRequest_MasksSessionToken(t *testing.T) {
 // credentials, must carry InvalidAction (the same code an unimplemented
 // action on a served service already returns) rather than a signature
 // failure that never happened, and must render in the generic REST-XML
-// envelope rather than the EC2 query shape a REST-XML client like route53's
-// cannot deserialize.
+// envelope rather than the EC2 query shape a non-query client cannot
+// deserialize.
 func TestSigV4Auth_UnservedServiceNamesServiceNotCredentials(t *testing.T) {
 	handler := setupTestApp(testAccessKey, testSecretKey)
 
 	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Host = "localhost:9999"
-	signTestRequestAs(t, req, nil, testAccessKey, testSecretKey, "route53", time.Now().UTC())
+	signTestRequestAs(t, req, nil, testAccessKey, testSecretKey, "sqs", time.Now().UTC())
 
 	resp := doRequest(handler, req)
 	require.Equal(t, http.StatusBadRequest, resp.StatusCode)
@@ -2606,7 +2606,7 @@ func TestSigV4Auth_UnservedServiceNamesServiceNotCredentials(t *testing.T) {
 
 	assert.Equal(t, "ErrorResponse", xmlRootName(t, body))
 	assert.Equal(t, awserrors.ErrorInvalidAction, parsed.Error.Code)
-	assert.Contains(t, parsed.Error.Message, "route53")
+	assert.Contains(t, parsed.Error.Message, "sqs")
 	assert.NotContains(t, parsed.Error.Message, "credentials",
 		"nothing was wrong with the credentials, which were never checked")
 }
@@ -2803,3 +2803,20 @@ func TestSigV4Auth_ResolvesCtxActionForNonQueryServices(t *testing.T) {
 		})
 	}
 }
+
+// Route 53 is global: the AWS CLI and SDKs sign it for us-east-1 against the
+// default endpoint and for the configured region against an override. Every
+// other service stays pinned to the gateway's region.
+func TestExpectedSigningRegion(t *testing.T) {
+	gw := &GatewayConfig{Region: "ap-southeast-2"}
+	for _, tc := range []struct {
+		service, claimed, want string
+	}{
+		{"route53", "us-east-1", "us-east-1"},
+		{"route53", "ap-southeast-2", "ap-southeast-2"},
+		{"route53", "eu-west-1", "ap-southeast-2"},
+		{"ec2", "us-east-1", "ap-southeast-2"},
+	} {
+		assert.Equalf(t, tc.want, gw.expectedSigningRegion(tc.service, tc.claimed), "%s signed for %s", tc.service, tc.claimed)
+	}
+}

spinifex/gateway/dispatch_seam_test.go

@@ -385,7 +385,7 @@ func TestDispatchSeam_UnregisteredUnknownServiceStillInvalidAction(t *testing.T)
 
 	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	req.Host = "localhost:9999"
-	signTestRequestAs(t, req, nil, testAccessKey, testSecretKey, "route53", time.Now().UTC())
+	signTestRequestAs(t, req, nil, testAccessKey, testSecretKey, "sqs", time.Now().UTC())
 
 	resp := doRequest(r, req)
 	require.Equal(t, http.StatusBadRequest, resp.StatusCode)
@@ -400,7 +400,7 @@ func TestDispatchSeam_UnregisteredUnknownServiceStillInvalidAction(t *testing.T)
 	}
 	require.NoError(t, xml.Unmarshal(body, &parsed))
 	assert.Equal(t, awserrors.ErrorInvalidAction, parsed.Error.Code)
-	assert.Contains(t, parsed.Error.Message, "route53")
+	assert.Contains(t, parsed.Error.Message, "sqs")
 }
 
 // --- h. ValidateServices overlap guard --------------------------------------

spinifex/gateway/gateway.go

@@ -464,10 +464,10 @@ func (gw *GatewayConfig) writeThrottleError(w http.ResponseWriter, r *http.Reque
 	svc, _ := r.Context().Value(ctxService).(string)
 
 	errorCode := awserrors.ErrorRequestLimitExceeded
-	if svc == "iam" || svc == "sts" || svc == "eks" {
+	if svc == "iam" || svc == "sts" || svc == "eks" || svc == "route53" {
 		errorCode = awserrors.ErrorThrottling
 	}
-	errorMsg := awserrors.ErrorLookup[errorCode]
+	errorMsg := awserrors.LookupErrorMessage(svc, errorCode)
 
 	// AWS JSON 1.1 services (EKS/ECS/bedrock family, …) get a JSON body.
 	if gw.jsonErrorService(svc) {

spinifex/gateway/gateway_test.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	acmawsapi "github.com/mulgadc/spinifex/spinifex/domains/acm/awsapi"
 	awsapi "github.com/mulgadc/spinifex/spinifex/domains/ecr/awsapi"
+	route53awsapi "github.com/mulgadc/spinifex/spinifex/domains/route53/awsapi"
 	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"io"
 	"maps"
@@ -241,8 +242,9 @@ func TestErrorHandler_NonEC2ValidationDefaults_DropEC2Wording(t *testing.T) {
 	b := dispatch.NewBuilder()
 	require.NoError(t, b.Register(awsapi.NewRegistration(awsapi.Deps{})))
 	require.NoError(t, b.Register(acmawsapi.NewRegistration(acmawsapi.Deps{})))
+	require.NoError(t, b.Register(route53awsapi.NewRegistration(route53awsapi.Deps{})))
 	gw.Services = b.Build()
-	services := append(slices.Collect(maps.Keys(supportedServices)), awsapi.ServiceName, acmawsapi.ServiceName)
+	services := append(slices.Collect(maps.Keys(supportedServices)), awsapi.ServiceName, acmawsapi.ServiceName, route53awsapi.ServiceName)
 	for _, svc := range services {
 		if svc == "ec2" {
 			continue
@@ -1460,6 +1462,23 @@ func TestWriteThrottleError_IAM(t *testing.T) {
 	assert.Contains(t, string(body), "<ErrorResponse>")
 }
 
+// Route 53 throttles with a 400 carrying the Throttling code, which SDK
+// retryers classify by code rather than status.
+func TestWriteThrottleError_Route53(t *testing.T) {
+	gw := &GatewayConfig{DisableLogging: true}
+	req := httptest.NewRequest(http.MethodGet, "/2013-04-01/hostedzone", nil)
+	req = req.WithContext(context.WithValue(req.Context(), ctxService, "route53"))
+	w := httptest.NewRecorder()
+
+	gw.writeThrottleError(w, req)
+
+	resp := w.Result()
+	body, _ := io.ReadAll(resp.Body)
+	assert.Equal(t, http.StatusBadRequest, resp.StatusCode)
+	assert.Contains(t, string(body), "<Code>Throttling</Code>")
+	assert.Contains(t, string(body), "<ErrorResponse>")
+}
+
 func TestWriteThrottleError_Bedrock(t *testing.T) {
 	gw := &GatewayConfig{DisableLogging: true}
 

spinifex/handlers/iam/managed_policies.go

@@ -258,6 +258,53 @@ var awsManagedPolicyDocs = map[string]string{
 		]
 	}`,
 
+	"arn:aws:iam::aws:policy/AmazonRoute53FullAccess": `{
+		"Version": "2012-10-17",
+		"Statement": [
+			{
+				"Effect": "Allow",
+				"Action": [
+					"route53:*",
+					"route53domains:*",
+					"cloudfront:ListDistributions",
+					"elasticloadbalancing:DescribeLoadBalancers",
+					"elasticbeanstalk:DescribeEnvironments",
+					"s3:ListBucket",
+					"s3:GetBucketLocation",
+					"s3:GetBucketWebsite",
+					"ec2:DescribeVpcs",
+					"ec2:DescribeVpcEndpoints",
+					"ec2:DescribeRegions",
+					"sns:ListTopics",
+					"sns:ListSubscriptionsByTopic",
+					"cloudwatch:DescribeAlarms",
+					"cloudwatch:GetMetricStatistics"
+				],
+				"Resource": "*"
+			},
+			{
+				"Effect": "Allow",
+				"Action": "apigateway:GET",
+				"Resource": "arn:aws:apigateway:*::/domainnames"
+			}
+		]
+	}`,
+
+	"arn:aws:iam::aws:policy/AmazonRoute53ReadOnlyAccess": `{
+		"Version": "2012-10-17",
+		"Statement": [
+			{
+				"Effect": "Allow",
+				"Action": [
+					"route53:Get*",
+					"route53:List*",
+					"route53:TestDNSAnswer"
+				],
+				"Resource": "*"
+			}
+		]
+	}`,
+
 	"arn:aws:iam::aws:policy/AmazonEKSServicePolicy": `{
 		"Version": "2012-10-17",
 		"Statement": [

spinifex/handlers/iam/managed_policies_test.go

@@ -159,3 +159,30 @@ func TestResolveAttachedPolicy_UnmodeledRDSManagedARNGrantsNothing(t *testing.T)
 		assert.Empty(t, doc.Statement)
 	}
 }
+
+// Terraform users commonly carry only a Route 53 managed policy, so both must
+// resolve, and the read-only one must not reach a mutation.
+func TestRoute53ManagedShims(t *testing.T) {
+	t.Parallel()
+	full, ok := builtinManagedPolicyDoc("arn:aws:iam::aws:policy/AmazonRoute53FullAccess")
+	require.True(t, ok)
+	readOnly, ok := builtinManagedPolicyDoc("arn:aws:iam::aws:policy/AmazonRoute53ReadOnlyAccess")
+	require.True(t, ok)
+
+	reads := []string{"route53:GetHostedZone", "route53:ListHostedZones", "route53:ListResourceRecordSets",
+		"route53:GetChange", "route53:ListTagsForResource"}
+	writes := []string{"route53:CreateHostedZone", "route53:DeleteHostedZone",
+		"route53:ChangeResourceRecordSets", "route53:ChangeTagsForResource"}
+	for _, action := range append(append([]string{}, reads...), writes...) {
+		assert.Equal(t, iampolicy.Allow, iampolicy.EvaluateWithKeys(action, "*", []PolicyDocument{full}, nil),
+			"AmazonRoute53FullAccess must grant %s", action)
+	}
+	for _, action := range reads {
+		assert.Equal(t, iampolicy.Allow, iampolicy.EvaluateWithKeys(action, "*", []PolicyDocument{readOnly}, nil),
+			"AmazonRoute53ReadOnlyAccess must grant %s", action)
+	}
+	for _, action := range writes {
+		assert.Equal(t, iampolicy.Deny, iampolicy.EvaluateWithKeys(action, "*", []PolicyDocument{readOnly}, nil),
+			"AmazonRoute53ReadOnlyAccess must not grant %s", action)
+	}
+}

spinifex/ingress/aws/dispatch/registry_test.go

@@ -139,12 +139,12 @@ func TestLookupServesEveryNameOfOneRegistration(t *testing.T) {
 func TestUnknownServiceKeepsTheEstablishedResult(t *testing.T) {
 	b := NewBuilder()
 	require.NoError(t, b.Register(validReg("ecr")))
-	_, ok := b.Build().Lookup("route53")
+	_, ok := b.Build().Lookup("sqs")
 	assert.False(t, ok)
 
-	code, msg := UnknownService("route53")
+	code, msg := UnknownService("sqs")
 	assert.Equal(t, awserrors.ErrorInvalidAction, code)
-	assert.Equal(t, `Service "route53" is not served by this gateway.`, msg)
+	assert.Equal(t, `Service "sqs" is not served by this gateway.`, msg)
 }
 
 func TestResolveActionIsAdvisory(t *testing.T) {

spinifex/ingress/aws/dispatch/selector_test.go

@@ -88,7 +88,7 @@ func TestSelectorLegacyFallback(t *testing.T) {
 
 	assert.True(t, s.Served("ec2"))
 	assert.True(t, s.Served("acm"))
-	assert.False(t, s.Served("route53"))
+	assert.False(t, s.Served("sqs"))
 	assert.Equal(t, "LegacyAction", s.ResolveAction(req, "ec2"))
 	assert.False(t, s.JSONErrors("ec2"))
 	assert.True(t, s.JSONErrors("acm"))

spinifex/ingress/aws/invariants/deps_test.go

@@ -81,8 +81,9 @@ var declaredInventoryReaders = map[string]bool{
 // implementations ADR-0002 acceptance 3 confines to the awsgw role,
 // alongside the registration contract itself.
 var domainAWSAPIImpls = map[string]string{
-	modulePrefix + "domains/ecr/awsapi": "ECR implementation",
-	modulePrefix + "domains/acm/awsapi": "ACM implementation",
+	modulePrefix + "domains/ecr/awsapi":     "ECR implementation",
+	modulePrefix + "domains/acm/awsapi":     "ACM implementation",
+	modulePrefix + "domains/route53/awsapi": "Route 53 implementation",
 }
 
 // ADR-0002 acceptance 3: the awsgw role is the only production composition

spinifex/operator/cli/admin_dns.go

@@ -0,0 +1,231 @@
+package cli
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"slices"
+	"strings"
+	"text/tabwriter"
+	"time"
+
+	"github.com/miekg/dns"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	dnsdomain "github.com/mulgadc/spinifex/spinifex/domains/dns"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var dnsCmd = &cobra.Command{
+	Use:   "dns",
+	Short: "Inspect the cluster's authoritative DNS",
+}
+
+var dnsNameserversCmd = &cobra.Command{
+	Use:   "nameservers",
+	Short: "Print the published nsN nameservers and check their delegation",
+	Long: `Print the nameservers this cluster publishes for its base domain: each nsN
+name, the node it is assigned to and that node's address. At most four are
+published however many nodes run northstar; [northstar] nameserver_count and
+nameserver_nodes in spinifex.toml set the count and pin the order.
+
+With --check, ask one of the parent zone's servers (for spx3.net, a .net
+server) for the base domain's delegation without recursion, and report for
+each nsN whether the parent lists it and whether its glue matches the node's
+address. Exits non-zero if any does not.`,
+	RunE: runDNSNameservers,
+}
+
+func init() {
+	adminCmd.AddCommand(dnsCmd)
+	dnsCmd.AddCommand(dnsNameserversCmd)
+	dnsNameserversCmd.Flags().Bool("check", false, "Compare against the delegation published by the parent zone")
+	dnsNameserversCmd.Flags().Duration("timeout", 5*time.Second, "Timeout for each DNS query made by --check")
+}
+
+func runDNSNameservers(cmd *cobra.Command, _ []string) error {
+	cfgPath := viper.GetString("config")
+	if cfgPath == "" {
+		cfgPath = DefaultConfigFile()
+	}
+	cluster, err := config.LoadConfig(cfgPath)
+	if err != nil {
+		return fmt.Errorf("load config: %w", err)
+	}
+	local := cluster.Nodes[cluster.Node]
+	base := strings.TrimSuffix(dnsdomain.ResolveBaseDomain(&local), ".")
+	if base == "" {
+		return errors.New("no northstar base domain is configured on this node")
+	}
+	published := dnsdomain.PublishedNameservers(cluster)
+	if len(published) == 0 {
+		return errors.New("no node qualifies as a nameserver: none runs northstar on a non-loopback address")
+	}
+
+	out := cmd.OutOrStdout()
+	check, _ := cmd.Flags().GetBool("check")
+	if !check {
+		printNameservers(out, base, published)
+		return nil
+	}
+
+	timeout, _ := cmd.Flags().GetDuration("timeout")
+	ctx, cancel := context.WithTimeout(cmd.Context(), 4*timeout)
+	defer cancel()
+	listed, glue, server, err := queryParentDelegation(ctx, base, timeout)
+	if err != nil {
+		return err
+	}
+	fmt.Fprintf(out, "Delegation of %s as published by %s\n\n", base, server)
+	rows := compareDelegation(base, published, listed, glue)
+	printDelegation(out, rows)
+	for _, r := range rows {
+		if r.Status != delegationOK {
+			return errors.New("the parent zone's delegation does not match the published nameservers")
+		}
+	}
+	return nil
+}
+
+func printNameservers(out io.Writer, base string, published []dnsdomain.Nameserver) {
+	w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
+	fmt.Fprintln(w, "NAME\tNODE\tADDRESS")
+	for _, ns := range published {
+		fmt.Fprintf(w, "%s.%s\t%s\t%s\n", ns.Host, base, ns.Node, ns.IP)
+	}
+	_ = w.Flush()
+}
+
+type delegationStatus string
+
+const (
+	delegationOK      delegationStatus = "ok"
+	delegationMissing delegationStatus = "not delegated"
+	delegationNoGlue  delegationStatus = "no glue"
+	delegationBadGlue delegationStatus = "glue mismatch"
+	delegationNotOurs delegationStatus = "not published here"
+)
+
+type delegationRow struct {
+	Name   string
+	Want   string
+	Glue   []string
+	Status delegationStatus
+}
+
+// compareDelegation checks the parent's NS names and glue for base against the
+// published nameservers. A name the parent lists that this cluster does not
+// publish is reported too, since resolvers will send queries to it.
+func compareDelegation(base string, published []dnsdomain.Nameserver, listed []string, glue map[string][]string) []delegationRow {
+	norm := func(name string) string { return strings.ToLower(strings.TrimSuffix(name, ".")) }
+	listedSet := make(map[string]bool, len(listed))
+	for _, n := range listed {
+		listedSet[norm(n)] = true
+	}
+	normGlue := make(map[string][]string, len(glue))
+	for n, ips := range glue {
+		normGlue[norm(n)] = ips
+	}
+
+	rows := make([]delegationRow, 0, len(published)+len(listed))
+	ours := make(map[string]bool, len(published))
+	for _, ns := range published {
+		name := norm(ns.Host + "." + base)
+		ours[name] = true
+		row := delegationRow{Name: name, Want: ns.IP, Glue: normGlue[name]}
+		switch {
+		case !listedSet[name]:
+			row.Status = delegationMissing
+		case len(row.Glue) == 0:
+			row.Status = delegationNoGlue
+		case !slices.Contains(row.Glue, ns.IP):
+			row.Status = delegationBadGlue
+		default:
+			row.Status = delegationOK
+		}
+		rows = append(rows, row)
+	}
+	for _, n := range listed {
+		if name := norm(n); !ours[name] {
+			rows = append(rows, delegationRow{Name: name, Glue: normGlue[name], Status: delegationNotOurs})
+			ours[name] = true
+		}
+	}
+	return rows
+}
+
+func printDelegation(out io.Writer, rows []delegationRow) {
+	w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
+	fmt.Fprintln(w, "NAME\tWANT\tPARENT GLUE\tSTATUS")
+	for _, r := range rows {
+		want, glue := r.Want, strings.Join(r.Glue, ",")
+		if want == "" {
+			want = "-"
+		}
+		if glue == "" {
+			glue = "-"
+		}
+		fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", r.Name, want, glue, r.Status)
+	}
+	_ = w.Flush()
+}
+
+// queryParentDelegation asks a server of base's parent zone, without
+// recursion, for base's NS set.
+func queryParentDelegation(ctx context.Context, base string, timeout time.Duration) ([]string, map[string][]string, string, error) {
+	parent := base[strings.Index(base, ".")+1:]
+	if parent == base {
+		return nil, nil, "", fmt.Errorf("%s has no parent zone to check", base)
+	}
+	servers, err := net.DefaultResolver.LookupNS(ctx, parent)
+	if err != nil || len(servers) == 0 {
+		return nil, nil, "", fmt.Errorf("find the %s servers: %w", parent, err)
+	}
+	addrs := make([]string, 0, len(servers))
+	for _, s := range servers {
+		addrs = append(addrs, net.JoinHostPort(strings.TrimSuffix(s.Host, "."), "53"))
+	}
+	return exchangeDelegation(ctx, base, addrs, timeout)
+}
+
+// exchangeDelegation asks each server in turn for base's NS set and returns,
+// from the first that answers, the names it lists and the glue addresses in
+// its additional section.
+func exchangeDelegation(ctx context.Context, base string, servers []string, timeout time.Duration) ([]string, map[string][]string, string, error) {
+	msg := new(dns.Msg)
+	msg.SetQuestion(dns.Fqdn(base), dns.TypeNS)
+	msg.RecursionDesired = false
+	client := &dns.Client{Timeout: timeout}
+
+	lastErr := errors.New("no servers")
+	for _, server := range servers {
+		resp, _, err := client.ExchangeContext(ctx, msg, server)
+		if err != nil {
+			lastErr = err
+			continue
+		}
+		if resp.Rcode != dns.RcodeSuccess {
+			lastErr = fmt.Errorf("%s answered %s", server, dns.RcodeToString[resp.Rcode])
+			continue
+		}
+		var listed []string
+		for _, rr := range append(resp.Ns, resp.Answer...) {
+			if ns, ok := rr.(*dns.NS); ok && strings.EqualFold(ns.Hdr.Name, dns.Fqdn(base)) {
+				listed = append(listed, ns.Ns)
+			}
+		}
+		glue := map[string][]string{}
+		for _, rr := range resp.Extra {
+			switch a := rr.(type) {
+			case *dns.A:
+				glue[a.Hdr.Name] = append(glue[a.Hdr.Name], a.A.String())
+			case *dns.AAAA:
+				glue[a.Hdr.Name] = append(glue[a.Hdr.Name], a.AAAA.String())
+			}
+		}
+		return listed, glue, server, nil
+	}
+	return nil, nil, "", fmt.Errorf("no server answered for %s: %w", base, lastErr)
+}

spinifex/operator/cli/admin_dns_test.go

@@ -0,0 +1,172 @@
+package cli
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/miekg/dns"
+	dnsdomain "github.com/mulgadc/spinifex/spinifex/domains/dns"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCompareDelegation(t *testing.T) {
+	published := []dnsdomain.Nameserver{
+		{Host: "ns1", Node: "node1", IP: "192.0.2.1"},
+		{Host: "ns2", Node: "node2", IP: "192.0.2.2"},
+		{Host: "ns3", Node: "node3", IP: "192.0.2.3"},
+		{Host: "ns4", Node: "node4", IP: "192.0.2.4"},
+	}
+	// The parent's answer: names are FQDNs in whatever case the registry
+	// stores, ns3's glue is stale, ns4 is missing and ns9 is a leftover.
+	listed := []string{"NS1.spx3.net.", "ns2.spx3.net.", "ns3.spx3.net.", "ns9.spx3.net."}
+	glue := map[string][]string{
+		"NS1.spx3.net.": {"192.0.2.1"},
+		"ns3.spx3.net.": {"198.51.100.3"},
+		"ns9.spx3.net.": {"198.51.100.9"},
+	}
+
+	got := map[string]delegationStatus{}
+	for _, r := range compareDelegation("spx3.net", published, listed, glue) {
+		got[r.Name] = r.Status
+	}
+	assert.Equal(t, map[string]delegationStatus{
+		"ns1.spx3.net": delegationOK,
+		"ns2.spx3.net": delegationNoGlue,
+		"ns3.spx3.net": delegationBadGlue,
+		"ns4.spx3.net": delegationMissing,
+		"ns9.spx3.net": delegationNotOurs,
+	}, got)
+}
+
+func dnsClusterToml(t *testing.T, nodes int, extra string) string {
+	t.Helper()
+	var b strings.Builder
+	b.WriteString("node = \"node1\"\n\n" + extra + "\n")
+	for i := 1; i <= nodes; i++ {
+		fmt.Fprintf(&b, "[nodes.node%d]\nregion = \"us-east-1\"\nadvertise = \"192.0.2.%d\"\n\n[nodes.node%d.northstar]\nconfig_path = \"/etc/spinifex/northstar/northstar.toml\"\n", i, i, i)
+		if i == 1 {
+			b.WriteString("default_domain = \"spx3.net\"\n")
+		}
+		b.WriteString("\n")
+	}
+	return writeSpinifexToml(t, b.String())
+}
+
+func runNameservers(t *testing.T, path string) (string, error) {
+	t.Helper()
+	resetGlobalViper(t)
+	// LoadConfig declares the process-wide cluster size; undo it so later
+	// tests in this package can declare their own.
+	clustersize.ResetForTest()
+	t.Cleanup(clustersize.ResetForTest)
+	viper.Set("config", path)
+	var out bytes.Buffer
+	cmd := &cobra.Command{}
+	cmd.Flags().Bool("check", false, "")
+	cmd.Flags().Duration("timeout", time.Second, "")
+	cmd.SetOut(&out)
+	cmd.SetContext(context.Background())
+	err := runDNSNameservers(cmd, nil)
+	return out.String(), err
+}
+
+func TestRunDNSNameservers_PublishesAtMostFour(t *testing.T) {
+	out, err := runNameservers(t, dnsClusterToml(t, 5, ""))
+	require.NoError(t, err)
+	for i := 1; i <= 4; i++ {
+		assert.Contains(t, out, fmt.Sprintf("ns%d.spx3.net  node%d  192.0.2.%d", i, i, i))
+	}
+	assert.NotContains(t, out, "ns5")
+	assert.NotContains(t, out, "node5")
+}
+
+func TestRunDNSNameservers_FollowsPin(t *testing.T) {
+	out, err := runNameservers(t, dnsClusterToml(t, 3, "[northstar]\nnameserver_count = 2\nnameserver_nodes = [\"node3\"]\n"))
+	require.NoError(t, err)
+	assert.Contains(t, out, "ns1.spx3.net  node3  192.0.2.3")
+	assert.Contains(t, out, "ns2.spx3.net  node1  192.0.2.1")
+	assert.NotContains(t, out, "ns3")
+}
+
+func TestRunDNSNameservers_RequiresBaseDomain(t *testing.T) {
+	path := writeSpinifexToml(t, "node = \"node1\"\n\n[nodes.node1]\nregion = \"us-east-1\"\n")
+	_, err := runNameservers(t, path)
+	require.ErrorContains(t, err, "base domain")
+}
+
+// startDNS serves handler on a loopback UDP port and returns its address.
+func startDNS(t *testing.T, handler dns.HandlerFunc) string {
+	t.Helper()
+	pc, err := net.ListenPacket("udp", "127.0.0.1:0")
+	require.NoError(t, err)
+	srv := &dns.Server{PacketConn: pc, Handler: handler}
+	go func() { _ = srv.ActivateAndServe() }()
+	t.Cleanup(func() { _ = srv.Shutdown() })
+	return pc.LocalAddr().String()
+}
+
+// A parent server that refuses is skipped; the first referral is read for the
+// listed names and their glue, and recursion is never asked for.
+func TestExchangeDelegation(t *testing.T) {
+	refused := startDNS(t, func(w dns.ResponseWriter, r *dns.Msg) {
+		m := new(dns.Msg)
+		m.SetRcode(r, dns.RcodeRefused)
+		_ = w.WriteMsg(m)
+	})
+	var sawRecursion atomic.Bool
+	referral := startDNS(t, func(w dns.ResponseWriter, r *dns.Msg) {
+		if r.RecursionDesired {
+			sawRecursion.Store(true)
+		}
+		m := new(dns.Msg)
+		m.SetReply(r)
+		for _, rr := range []string{
+			"spx3.net. 172800 IN NS ns1.spx3.net.",
+			"spx3.net. 172800 IN NS ns2.spx3.net.",
+			"other.net. 172800 IN NS ns.other.net.",
+		} {
+			m.Ns = append(m.Ns, mustRR(t, rr))
+		}
+		m.Extra = append(m.Extra, mustRR(t, "ns1.spx3.net. 172800 IN A 192.0.2.1"), mustRR(t, "ns2.spx3.net. 172800 IN AAAA 2001:db8::2"))
+		_ = w.WriteMsg(m)
+	})
+
+	listed, glue, server, err := exchangeDelegation(context.Background(), "spx3.net", []string{refused, referral}, time.Second)
+	require.NoError(t, err)
+	assert.False(t, sawRecursion.Load(), "the parent must be asked without recursion")
+	assert.Equal(t, referral, server)
+	assert.Equal(t, []string{"ns1.spx3.net.", "ns2.spx3.net."}, listed)
+	assert.Equal(t, map[string][]string{"ns1.spx3.net.": {"192.0.2.1"}, "ns2.spx3.net.": {"2001:db8::2"}}, glue)
+
+	_, _, _, err = exchangeDelegation(context.Background(), "spx3.net", []string{refused}, time.Second)
+	require.ErrorContains(t, err, "REFUSED")
+}
+
+func mustRR(t *testing.T, s string) dns.RR {
+	t.Helper()
+	rr, err := dns.NewRR(s)
+	require.NoError(t, err)
+	return rr
+}
+
+func TestPrintDelegation(t *testing.T) {
+	var out bytes.Buffer
+	printDelegation(&out, []delegationRow{
+		{Name: "ns1.spx3.net", Want: "192.0.2.1", Glue: []string{"192.0.2.1"}, Status: delegationOK},
+		{Name: "ns9.spx3.net", Status: delegationNotOurs},
+	})
+	lines := strings.Split(strings.TrimSpace(out.String()), "\n")
+	require.Len(t, lines, 3)
+	assert.Equal(t, []string{"ns1.spx3.net", "192.0.2.1", "192.0.2.1", "ok"}, strings.Fields(lines[1]))
+	assert.Equal(t, []string{"ns9.spx3.net", "-", "-", "not", "published", "here"}, strings.Fields(lines[2]))
+}

spinifex/operator/cli/admin_dns_zone.go

@@ -0,0 +1,226 @@
+package cli
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"text/tabwriter"
+	"time"
+
+	route53domain "github.com/mulgadc/spinifex/spinifex/domains/route53"
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/spf13/cobra"
+)
+
+// route53AdminTimeout covers a delegation check, which walks the public DNS
+// tree, and a delegate batch, which waits for the zone-writer election.
+const route53AdminTimeout = 60 * time.Second
+
+var dnsZoneCmd = &cobra.Command{
+	Use:   "zone",
+	Short: "Inspect and override the activation of customer hosted zones",
+}
+
+var dnsZoneStatusCmd = &cobra.Command{
+	Use:   "status <zone-id|name>",
+	Short: "Show a hosted zone's activation, in whichever account holds it",
+	Args:  cobra.ExactArgs(1),
+	RunE: func(cmd *cobra.Command, args []string) error {
+		q := route53domain.AdminZoneQuery{}
+		if looksLikeZoneID(args[0]) {
+			q.ZoneID = args[0]
+		} else {
+			q.Name = args[0]
+		}
+		return runRoute53Admin(cmd, "route53.admin.FindZones", q)
+	},
+}
+
+var dnsZoneVerifyCmd = &cobra.Command{
+	Use:   "verify <zone-id>",
+	Short: "Check a hosted zone's delegation now",
+	Args:  cobra.ExactArgs(1),
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runRoute53Admin(cmd, "route53.admin.Verify", route53domain.AdminZoneQuery{ZoneID: args[0]})
+	},
+}
+
+var dnsZoneActivateCmd = &cobra.Command{
+	Use:   "activate <zone-id> --reason <why>",
+	Short: "Serve a hosted zone whatever its delegation, for private roots and air-gapped sites",
+	Args:  cobra.ExactArgs(1),
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runZoneOverride(cmd, args[0], route53domain.StateActive)
+	},
+}
+
+var dnsZoneDeactivateCmd = &cobra.Command{
+	Use:   "deactivate <zone-id> --reason <why>",
+	Short: "Stop serving a hosted zone, for a contested domain",
+	Args:  cobra.ExactArgs(1),
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runZoneOverride(cmd, args[0], route53domain.StateSuspended)
+	},
+}
+
+var dnsZoneReleaseCmd = &cobra.Command{
+	Use:   "release <zone-id>",
+	Short: "Clear an override and hand the zone back to the delegation check",
+	Args:  cobra.ExactArgs(1),
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runZoneOverride(cmd, args[0], "")
+	},
+}
+
+var dnsDelegateCmd = &cobra.Command{
+	Use:   "delegate [name...]",
+	Short: "Delegate names below the base domain to this cluster's nameservers",
+	Long: `Write an NS set in the base zone delegating each name to the published nsN
+nameservers, so a customer hosted zone of that name passes its delegation
+check without any registrar. --remove withdraws the delegations; --file reads
+one name per line, for batches.`,
+	RunE: runDNSDelegate,
+}
+
+func init() {
+	dnsCmd.AddCommand(dnsZoneCmd, dnsDelegateCmd)
+	dnsZoneCmd.AddCommand(dnsZoneStatusCmd, dnsZoneVerifyCmd, dnsZoneActivateCmd, dnsZoneDeactivateCmd, dnsZoneReleaseCmd)
+	for _, c := range []*cobra.Command{dnsZoneActivateCmd, dnsZoneDeactivateCmd} {
+		c.Flags().String("reason", "", "Why, recorded on the zone (required)")
+	}
+	dnsDelegateCmd.Flags().Bool("remove", false, "Withdraw the delegations instead of writing them")
+	dnsDelegateCmd.Flags().String("file", "", "Read names from this file, one per line ('-' for stdin)")
+}
+
+// route53AdminFn sends one admin request and returns the raw response; tests
+// replace it.
+var route53AdminFn = func(ctx context.Context, subject string, in any) (json.RawMessage, error) {
+	_, nc, err := loadConfigAndConnectFn()
+	if err != nil {
+		return nil, err
+	}
+	defer nc.Close()
+	out, err := natsmsg.NATSRequest[json.RawMessage](ctx, nc, subject, in, route53AdminTimeout, awsidentifiers.GlobalAccountID)
+	if err != nil {
+		return nil, err
+	}
+	return *out, nil
+}
+
+func runZoneOverride(cmd *cobra.Command, id, override string) error {
+	reason := ""
+	if f := cmd.Flags().Lookup("reason"); f != nil {
+		reason = strings.TrimSpace(f.Value.String())
+		if reason == "" {
+			return errors.New("--reason is required")
+		}
+	}
+	return runRoute53Admin(cmd, "route53.admin.Override", route53domain.AdminOverrideInput{ZoneID: id, Override: override, Reason: reason})
+}
+
+func runRoute53Admin(cmd *cobra.Command, subject string, in any) error {
+	ctx, cancel := context.WithTimeout(cmd.Context(), route53AdminTimeout)
+	defer cancel()
+	raw, err := route53AdminFn(ctx, subject, in)
+	if err != nil {
+		return err
+	}
+	var out route53domain.AdminZones
+	if err := json.Unmarshal(raw, &out); err != nil {
+		return fmt.Errorf("decode response: %w", err)
+	}
+	if len(out.Zones) == 0 {
+		return errors.New("no hosted zone matches")
+	}
+	printZones(cmd.OutOrStdout(), out.Zones)
+	return nil
+}
+
+func printZones(out io.Writer, zones []route53domain.ZoneRecord) {
+	w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
+	fmt.Fprintln(w, "ZONE ID\tACCOUNT\tNAME\tSTATE\tCHECKED\tPARENT\tDELEGATED TO\tREASON")
+	for _, z := range zones {
+		checked := "-"
+		if !z.CheckedAt.IsZero() {
+			checked = z.CheckedAt.UTC().Format(time.RFC3339)
+		}
+		state, reason := z.State, z.StateReason
+		if z.Override != "" {
+			state += " (override)"
+			reason = z.OverrideReason
+		}
+		fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", z.ID, z.AccountID, strings.TrimSuffix(z.Name, "."), state, checked,
+			dashIfEmpty(z.DelegationSource), dashIfEmpty(strings.Join(z.Delegation, ",")), dashIfEmpty(reason))
+	}
+	_ = w.Flush()
+}
+
+func runDNSDelegate(cmd *cobra.Command, args []string) error {
+	names := append([]string(nil), args...)
+	if path, _ := cmd.Flags().GetString("file"); path != "" {
+		more, err := readNames(cmd.InOrStdin(), path)
+		if err != nil {
+			return err
+		}
+		names = append(names, more...)
+	}
+	if len(names) == 0 {
+		return errors.New("no names given")
+	}
+	remove, _ := cmd.Flags().GetBool("remove")
+	ctx, cancel := context.WithTimeout(cmd.Context(), route53AdminTimeout)
+	defer cancel()
+	raw, err := route53AdminFn(ctx, "route53.admin.Delegate", route53domain.AdminDelegateInput{Names: names, Remove: remove})
+	if err != nil {
+		return err
+	}
+	var out route53domain.AdminDelegateOutput
+	if err := json.Unmarshal(raw, &out); err != nil {
+		return fmt.Errorf("decode response: %w", err)
+	}
+	verb := "Delegated"
+	if remove {
+		verb = "Withdrew the delegation of"
+	}
+	fmt.Fprintf(cmd.OutOrStdout(), "%s %d name(s); %d record change(s) applied\n", verb, len(names), out.Applied)
+	return nil
+}
+
+func readNames(stdin io.Reader, path string) ([]string, error) {
+	r := stdin
+	if path != "-" {
+		f, err := os.Open(path)
+		if err != nil {
+			return nil, err
+		}
+		defer f.Close()
+		r = f
+	}
+	var names []string
+	sc := bufio.NewScanner(r)
+	for sc.Scan() {
+		if line := strings.TrimSpace(sc.Text()); line != "" && !strings.HasPrefix(line, "#") {
+			names = append(names, line)
+		}
+	}
+	return names, sc.Err()
+}
+
+// looksLikeZoneID tells a Route 53 zone ID from a domain name: an ID has no dot.
+func looksLikeZoneID(s string) bool {
+	s = strings.TrimPrefix(s, "/hostedzone/")
+	return s != "" && !strings.Contains(s, ".")
+}
+
+func dashIfEmpty(s string) string {
+	if s == "" {
+		return "-"
+	}
+	return s
+}

spinifex/operator/cli/admin_dns_zone_test.go

@@ -0,0 +1,113 @@
+package cli
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	route53domain "github.com/mulgadc/spinifex/spinifex/domains/route53"
+)
+
+type adminCall struct {
+	subject string
+	body    string
+}
+
+// fakeRoute53Admin records each request and answers with reply.
+func fakeRoute53Admin(t *testing.T, reply any) *[]adminCall {
+	t.Helper()
+	var calls []adminCall
+	prev := route53AdminFn
+	route53AdminFn = func(_ context.Context, subject string, in any) (json.RawMessage, error) {
+		body, err := json.Marshal(in)
+		require.NoError(t, err)
+		calls = append(calls, adminCall{subject, string(body)})
+		return json.Marshal(reply)
+	}
+	t.Cleanup(func() { route53AdminFn = prev })
+	return &calls
+}
+
+func runCmd(t *testing.T, cmd *cobra.Command, flags map[string]string, args ...string) (string, error) {
+	t.Helper()
+	for k, v := range flags {
+		require.NoError(t, cmd.Flags().Set(k, v))
+		t.Cleanup(func() { _ = cmd.Flags().Set(k, cmd.Flags().Lookup(k).DefValue) })
+	}
+	var out bytes.Buffer
+	cmd.SetOut(&out)
+	cmd.SetContext(t.Context())
+	err := cmd.RunE(cmd, args)
+	return out.String(), err
+}
+
+var sampleZone = route53domain.ZoneRecord{
+	ID: "Z123", AccountID: "000000000011", Name: "example.com.", State: route53domain.StatePendingNameservers,
+	StateReason: "the com name servers delegates example.com to name servers that are not this cluster's: dana.ns.cloudflare.com",
+	CheckedAt:   time.Date(2026, 10, 24, 1, 2, 3, 0, time.UTC), DelegationSource: "the com name servers",
+	Delegation: []string{"dana.ns.cloudflare.com."},
+}
+
+func TestZoneStatusQueriesByIDOrName(t *testing.T) {
+	calls := fakeRoute53Admin(t, route53domain.AdminZones{Zones: []route53domain.ZoneRecord{sampleZone}})
+	out, err := runCmd(t, dnsZoneStatusCmd, nil, "example.com")
+	require.NoError(t, err)
+	_, err = runCmd(t, dnsZoneStatusCmd, nil, "/hostedzone/Z123")
+	require.NoError(t, err)
+
+	assert.Equal(t, []adminCall{
+		{"route53.admin.FindZones", `{"name":"example.com"}`},
+		{"route53.admin.FindZones", `{"zone_id":"/hostedzone/Z123"}`},
+	}, *calls)
+	for _, want := range []string{"Z123", "000000000011", "example.com", "PENDING_NAMESERVERS", "2026-10-24T01:02:03Z", "dana.ns.cloudflare.com.", "not this cluster's"} {
+		assert.Contains(t, out, want)
+	}
+}
+
+func TestZoneStatusReportsNoMatch(t *testing.T) {
+	fakeRoute53Admin(t, route53domain.AdminZones{})
+	_, err := runCmd(t, dnsZoneStatusCmd, nil, "nothing.example")
+	require.ErrorContains(t, err, "no hosted zone matches")
+}
+
+func TestZoneOverridesNeedAReason(t *testing.T) {
+	calls := fakeRoute53Admin(t, route53domain.AdminZones{Zones: []route53domain.ZoneRecord{sampleZone}})
+	_, err := runCmd(t, dnsZoneActivateCmd, nil, "Z123")
+	require.ErrorContains(t, err, "--reason is required")
+	assert.Empty(t, *calls, "a refused override sends nothing")
+
+	_, err = runCmd(t, dnsZoneDeactivateCmd, map[string]string{"reason": "squatting"}, "Z123")
+	require.NoError(t, err)
+	_, err = runCmd(t, dnsZoneReleaseCmd, nil, "Z123")
+	require.NoError(t, err)
+	assert.Equal(t, []adminCall{
+		{"route53.admin.Override", `{"zone_id":"Z123","override":"SUSPENDED","reason":"squatting"}`},
+		{"route53.admin.Override", `{"zone_id":"Z123","override":"","reason":""}`},
+	}, *calls)
+}
+
+func TestDelegateReadsNamesFromArgsAndFile(t *testing.T) {
+	calls := fakeRoute53Admin(t, route53domain.AdminDelegateOutput{Applied: 3})
+	path := filepath.Join(t.TempDir(), "names")
+	require.NoError(t, os.WriteFile(path, []byte("# e2e\nb.r53e2e.spx3.net\n\n c.r53e2e.spx3.net \n"), 0o600))
+
+	out, err := runCmd(t, dnsDelegateCmd, map[string]string{"file": path, "remove": "true"}, "a.r53e2e.spx3.net")
+	require.NoError(t, err)
+	assert.Equal(t, []adminCall{{"route53.admin.Delegate", `{"names":["a.r53e2e.spx3.net","b.r53e2e.spx3.net","c.r53e2e.spx3.net"],"remove":true}`}}, *calls)
+	assert.Contains(t, out, "Withdrew the delegation of 3 name(s); 3 record change(s) applied")
+}
+
+func TestDelegateNeedsNames(t *testing.T) {
+	calls := fakeRoute53Admin(t, route53domain.AdminDelegateOutput{})
+	_, err := runCmd(t, dnsDelegateCmd, nil)
+	require.ErrorContains(t, err, "no names given")
+	assert.Empty(t, *calls)
+}

spinifex/operator/cli/service.go

@@ -914,6 +914,7 @@ func runNorthstarStart(options northstarStartOptions, deps northstarStartDepende
 	svc, err := deps.newService(&northstar.Config{
 		ConfigPath: configPath,
 		BasePath:   baseDir,
+		NodeName:   clusterConfig.Node,
 		NatsHost:   nodeConfig.NATS.Host,
 		NatsToken:  nodeConfig.NATS.ACL.Token,
 		NatsCACert: nodeConfig.NATS.CACert,

spinifex/operator/cli/service_test.go

@@ -106,6 +106,7 @@ func TestRunNorthstarStartActivation(t *testing.T) {
 			assert.Equal(t, 1, stub.starts)
 			require.NotNil(t, startedConfig)
 			assert.Equal(t, tt.wantPath, startedConfig.ConfigPath)
+			assert.Equal(t, "node1", startedConfig.NodeName)
 		})
 	}
 }

spinifex/runtime/roles/awsgw/awsgw.go

@@ -36,6 +36,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
 	"github.com/mulgadc/spinifex/spinifex/domains/ochre"
 	ochrevector "github.com/mulgadc/spinifex/spinifex/domains/ochre/vector"
+	route53awsapi "github.com/mulgadc/spinifex/spinifex/domains/route53/awsapi"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/reconciler"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
@@ -531,7 +532,8 @@ func launchService(config *config.ClusterConfig) error {
 		NATS:               natsConn,
 	})
 	acmControlPlane := acmawsapi.NewRegistration(acmawsapi.Deps{NATS: natsConn})
-	if err := wireServiceRegistry(&gw, ecrControlPlane, acmControlPlane); err != nil {
+	route53ControlPlane := route53awsapi.NewRegistration(route53awsapi.Deps{NATS: natsConn})
+	if err := wireServiceRegistry(&gw, ecrControlPlane, acmControlPlane, route53ControlPlane); err != nil {
 		return fmt.Errorf("awsgw: %w", err)
 	}
 

spinifex/runtime/roles/awsgw/awsgw_test.go

@@ -8,6 +8,7 @@ import (
 
 	acmawsapi "github.com/mulgadc/spinifex/spinifex/domains/acm/awsapi"
 	awsapi "github.com/mulgadc/spinifex/spinifex/domains/ecr/awsapi"
+	route53awsapi "github.com/mulgadc/spinifex/spinifex/domains/route53/awsapi"
 	"github.com/mulgadc/spinifex/spinifex/gateway"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
 	"github.com/stretchr/testify/assert"
@@ -48,15 +49,16 @@ func TestWireServiceRegistry(t *testing.T) {
 		require.Error(t, wireServiceRegistry(gw, reg, reg))
 	})
 
-	t.Run("ECR and ACM registrations wire together, as production does", func(t *testing.T) {
+	t.Run("ECR, ACM and Route 53 registrations wire together, as production does", func(t *testing.T) {
 		gw := &gateway.GatewayConfig{}
 		require.NoError(t, wireServiceRegistry(gw,
-			awsapi.NewRegistration(awsapi.Deps{}), acmawsapi.NewRegistration(acmawsapi.Deps{})))
+			awsapi.NewRegistration(awsapi.Deps{}), acmawsapi.NewRegistration(acmawsapi.Deps{}),
+			route53awsapi.NewRegistration(route53awsapi.Deps{})))
 		require.NotNil(t, gw.Services)
-		_, ecrOK := gw.Services.Lookup("ecr")
-		_, acmOK := gw.Services.Lookup("acm")
-		assert.True(t, ecrOK)
-		assert.True(t, acmOK)
+		for _, svc := range []string{"ecr", "acm", "route53"} {
+			_, ok := gw.Services.Lookup(svc)
+			assert.True(t, ok, svc)
+		}
 	})
 
 	t.Run("a registration claiming a legacy name fails startup", func(t *testing.T) {

spinifex/runtime/roles/northstar/northstar.go

@@ -28,7 +28,9 @@ type Config struct {
 	// ConfigPath is the path to northstar.toml (written by `spx admin init`).
 	ConfigPath string
 	// BasePath is the node base dir where the PID file is written.
-	BasePath   string
+	BasePath string
+	// NodeName identifies this server in zone reload acknowledgements.
+	NodeName   string
 	NatsHost   string
 	NatsToken  string
 	NatsCACert string
@@ -114,18 +116,42 @@ func (svc *Service) subscribeReload() {
 	svc.nc = nc
 
 	if _, err := nc.Subscribe(dns.SubjectZoneReload, func(msg *nats.Msg) {
-		var evt dns.ZoneReload
-		if err := json.Unmarshal(msg.Data, &evt); err != nil || evt.Zone == "" {
-			return
-		}
-		if err := svc.server.ReloadZone(evt.Zone); err != nil {
-			slog.Warn("northstar: reload zone", "zone", evt.Zone, "error", err)
-			return
-		}
-		slog.Info("northstar: zone reloaded via NATS", "zone", evt.Zone)
+		handleReload(svc.server, svc.Config.NodeName, msg)
 	}); err != nil {
 		slog.Warn("northstar: subscribe zone reload", "error", err)
 		return
 	}
 	slog.Info("northstar: subscribed to live zone reload", "subject", dns.SubjectZoneReload)
 }
+
+type zoneReloader interface {
+	ReloadZoneVersion(zone string) (string, error)
+}
+
+// handleReload reloads the zone a ZoneReload names and, when the sender asked
+// for a reply, acknowledges with the version now served or the failure.
+func handleReload(server zoneReloader, node string, msg *nats.Msg) {
+	var evt dns.ZoneReload
+	if err := json.Unmarshal(msg.Data, &evt); err != nil || evt.Zone == "" {
+		return
+	}
+	ack := dns.ZoneReloadAck{Zone: evt.Zone, Node: node}
+	etag, err := server.ReloadZoneVersion(evt.Zone)
+	if err != nil {
+		ack.Error = err.Error()
+		slog.Warn("northstar: reload zone", "zone", evt.Zone, "error", err)
+	} else {
+		ack.ETag = etag
+		slog.Info("northstar: zone reloaded via NATS", "zone", evt.Zone)
+	}
+	if msg.Reply == "" {
+		return
+	}
+	payload, err := json.Marshal(ack)
+	if err != nil {
+		return
+	}
+	if err := msg.Respond(payload); err != nil {
+		slog.Warn("northstar: acknowledge zone reload", "zone", evt.Zone, "error", err)
+	}
+}

spinifex/runtime/roles/northstar/northstar_test.go

@@ -1,10 +1,18 @@
 package northstar
 
 import (
+	"context"
+	"encoding/json"
+	"errors"
 	"testing"
+	"time"
 
+	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/dns"
 )
 
 func TestNewRejectsWrongConfigType(t *testing.T) {
@@ -31,3 +39,78 @@ func TestStartRequiresConfigPath(t *testing.T) {
 	_, err = svc.Start()
 	require.Error(t, err)
 }
+
+type fakeReloader struct {
+	etag  string
+	err   error
+	zones chan string
+}
+
+func (f *fakeReloader) ReloadZoneVersion(zone string) (string, error) {
+	f.zones <- zone
+	return f.etag, f.err
+}
+
+func subscribeHandler(t *testing.T, nc *nats.Conn, r zoneReloader) {
+	t.Helper()
+	sub, err := nc.Subscribe(dns.SubjectZoneReload, func(msg *nats.Msg) { handleReload(r, "node2", msg) })
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+}
+
+func TestHandleReloadAcknowledgesServedVersion(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	r := &fakeReloader{etag: "abc123", zones: make(chan string, 1)}
+	subscribeHandler(t, nc, r)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	acks, err := dns.RequestZoneReload(ctx, nc, "example.com", 1)
+	require.NoError(t, err)
+	require.Len(t, acks, 1)
+	assert.Equal(t, dns.ZoneReloadAck{Zone: "example.com", Node: "node2", ETag: "abc123"}, acks[0])
+	assert.Equal(t, "example.com", <-r.zones)
+}
+
+func TestHandleReloadAcknowledgesFailure(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	r := &fakeReloader{etag: "stale", err: errors.New("fetch zone: timeout"), zones: make(chan string, 1)}
+	subscribeHandler(t, nc, r)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	acks, err := dns.RequestZoneReload(ctx, nc, "example.com", 1)
+	require.NoError(t, err)
+	require.Len(t, acks, 1)
+	assert.Equal(t, "fetch zone: timeout", acks[0].Error)
+	assert.Empty(t, acks[0].ETag, "a failed reload must not report a version")
+}
+
+func TestHandleReloadWithoutReplyStillReloads(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	r := &fakeReloader{zones: make(chan string, 1)}
+	subscribeHandler(t, nc, r)
+
+	body, err := json.Marshal(dns.ZoneReload{Zone: "example.com"})
+	require.NoError(t, err)
+	require.NoError(t, nc.Publish(dns.SubjectZoneReload, body))
+	select {
+	case zone := <-r.zones:
+		assert.Equal(t, "example.com", zone)
+	case <-time.After(5 * time.Second):
+		t.Fatal("a fire-and-forget reload was not applied")
+	}
+}
+
+func TestHandleReloadIgnoresEventWithoutZone(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	r := &fakeReloader{zones: make(chan string, 1)}
+	subscribeHandler(t, nc, r)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
+	defer cancel()
+	acks, err := dns.RequestZoneReload(ctx, nc, "", 1)
+	require.NoError(t, err)
+	assert.Empty(t, acks)
+	assert.Empty(t, r.zones)
+}

tests/e2e/Makefile

@@ -18,9 +18,9 @@ SUITE_TAG   ?= $(notdir $(patsubst %/...,%,$(firstword $(PKG))))
 # BIN_DIR/<suite>.test so CI can ship binaries (not source + go toolchain) to
 # the remote VM.
 BIN_DIR      ?= $(CURDIR)/_bin
-SUITES_BUILD ?= cert lb eks ecs single iam ecr bedrock gpu natuplink multinode storagegrowth partialblock rds quota diskperf storagefault instancerecovery lbrecovery kvquorum
+SUITES_BUILD ?= cert lb eks ecs single iam ecr bedrock gpu natuplink multinode storagegrowth partialblock rds quota diskperf storagefault instancerecovery lbrecovery kvquorum route53
 
-.PHONY: e2e e2e-quota e2e-cert e2e-lb e2e-eks e2e-ecs e2e-single e2e-iam e2e-ecr e2e-bedrock e2e-multinode e2e-bm e2e-reboot e2e-tofu e2e-gpu e2e-natuplink e2e-storagegrowth e2e-partialblock e2e-diskperf e2e-rds e2e-storagefault e2e-instancerecovery e2e-lbrecovery e2e-kvquorum e2e-all clean e2e-build
+.PHONY: e2e e2e-quota e2e-route53 e2e-cert e2e-lb e2e-eks e2e-ecs e2e-single e2e-iam e2e-ecr e2e-bedrock e2e-multinode e2e-bm e2e-reboot e2e-tofu e2e-gpu e2e-natuplink e2e-storagegrowth e2e-partialblock e2e-diskperf e2e-rds e2e-storagefault e2e-instancerecovery e2e-lbrecovery e2e-kvquorum e2e-all clean e2e-build
 
 RUN_FLAG = $(if $(RUN),-run '$(RUN)',)
 
@@ -65,6 +65,7 @@ e2e-diskperf:      ; $(MAKE) e2e PKG=./diskperf/... TIMEOUT=180m
 # most of a 30m package timeout before the tests still running get to report.
 e2e-rds:        ; $(MAKE) e2e PKG=./rds/... TIMEOUT=45m
 e2e-quota:      ; $(MAKE) e2e PKG=./quota/...
+e2e-route53:    ; $(MAKE) e2e PKG=./route53/...
 # Freezes predastore, so its budget covers a deliberate outage plus the
 # recovery and re-verification that follow it, twice over for the two tests.
 e2e-storagefault: ; $(MAKE) e2e PKG=./storagefault/... TIMEOUT=45m

tests/e2e/harness/aws.go

@@ -24,6 +24,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/elbv2"
 	"github.com/aws/aws-sdk-go/service/iam"
 	"github.com/aws/aws-sdk-go/service/rds"
+	"github.com/aws/aws-sdk-go/service/route53"
 	"github.com/aws/aws-sdk-go/service/sts"
 )
 
@@ -43,6 +44,7 @@ type AWSClient struct {
 	ECR            *ecr.ECR
 	ECS            *ecs.ECS
 	RDS            *rds.RDS
+	Route53        *route53.Route53
 	Bedrock        *bedrock.Bedrock
 	BedrockRuntime *bedrockruntime.BedrockRuntime
 }
@@ -144,6 +146,7 @@ func newAWSClient(t *testing.T, env *Env, accessKey, secretKey, sessionToken str
 		ECR:            ecr.New(sess),
 		ECS:            ecs.New(sess),
 		RDS:            rds.New(sess),
+		Route53:        route53.New(sess),
 		Bedrock:        bedrock.New(sess),
 		BedrockRuntime: bedrockruntime.New(sess),
 	}

tests/e2e/harness/northstar_zones.go

@@ -0,0 +1,144 @@
+//go:build e2e
+
+package harness
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	nsconfig "github.com/mulgadc/northstar/pkg/config"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+)
+
+const northstarServerConfigPath = "/etc/spinifex/northstar/northstar.toml"
+
+// DNSTargets returns the nodes whose northstar the harness can query on :53:
+// every cluster node in multinode mode, else the single local node.
+func DNSTargets(env *Env) []Node {
+	if env.Mode == ModeMultinode {
+		if cluster, err := ClusterFromEnv(); err == nil && len(cluster.Nodes) > 0 {
+			return cluster.Nodes
+		}
+	}
+	return []Node{{Index: 1, Name: "local", Addr: gatewayDialHost(env)}}
+}
+
+// NorthstarZoneStore returns write credentials for the bucket northstar loads
+// zones from: the bucket and region from northstar.toml and the node's system
+// predastore key, which is what the control-plane zone writer uses. The
+// endpoint host is replaced by the node's address so it is reachable from the
+// runner; certificate verification is skipped, as elsewhere in the harness.
+func NorthstarZoneStore(t *testing.T, env *Env, node Node) *nsconfig.S3Config {
+	t.Helper()
+	host := node.Addr
+	if env.Mode != ModeMultinode {
+		host = ""
+	}
+
+	body, err := readNodeFile(host, northstarServerConfigPath)
+	if err != nil {
+		t.Fatalf("read %s on %s: %v", northstarServerConfigPath, hostLabel(host), err)
+	}
+	path := filepath.Join(t.TempDir(), "northstar.toml")
+	if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
+		t.Fatalf("stage northstar.toml: %v", err)
+	}
+	server, err := nsconfig.LoadServerConfig(path)
+	if err != nil {
+		t.Fatalf("parse northstar.toml from %s: %v", hostLabel(host), err)
+	}
+	if server.S3.Bucket == "" {
+		t.Skipf("northstar on %s serves zones from %q, not S3", hostLabel(host), server.ZoneDir)
+	}
+
+	cc := nodeClusterConfig(t, host)
+	creds := cc.Nodes[cc.Node].Predastore
+	if creds.AccessKey == "" || creds.SecretKey == "" {
+		t.Fatalf("no system predastore credentials in spinifex.toml on %s", hostLabel(host))
+	}
+
+	endpoint := server.S3.Endpoint
+	if env.Mode == ModeMultinode {
+		endpoint = withHost(endpoint, node.Addr)
+	}
+	return &nsconfig.S3Config{
+		Endpoint:  endpoint,
+		Region:    server.S3.Region,
+		Bucket:    server.S3.Bucket,
+		AccessKey: creds.AccessKey,
+		SecretKey: creds.SecretKey,
+		Insecure:  true,
+	}
+}
+
+// WriteTestZone stores a minimal zone for domain whose apex A record is addr.
+func WriteTestZone(s3cfg *nsconfig.S3Config, domain, addr string) error {
+	zone := nsconfig.NewZoneConfig(nsconfig.BaseZoneSeed{
+		Domain:      domain,
+		Nameservers: []nsconfig.NameserverSeed{{Host: "ns1", IP: addr}},
+	})
+	zone.UpsertRecord("", nsconfig.TypeA, nsconfig.ClassIN, addr, 0)
+	body, err := nsconfig.RenderZone(zone)
+	if err != nil {
+		return fmt.Errorf("render %s: %w", domain, err)
+	}
+	return nsconfig.WriteZoneFile(s3cfg, domain, body)
+}
+
+// DeleteTestZone removes domain's zone object.
+func DeleteTestZone(ctx context.Context, s3cfg *nsconfig.S3Config, domain string) error {
+	_, err := zoneStoreClient(s3cfg).DeleteObject(ctx, &s3.DeleteObjectInput{
+		Bucket: aws.String(s3cfg.Bucket),
+		Key:    aws.String(strings.TrimSuffix(domain, ".") + ".toml"),
+	})
+	return err
+}
+
+func zoneStoreClient(s3cfg *nsconfig.S3Config) *s3.Client {
+	endpoint := s3cfg.Endpoint
+	if !strings.Contains(endpoint, "://") {
+		endpoint = "https://" + endpoint
+	}
+	return s3.New(s3.Options{
+		Region:       s3cfg.Region,
+		BaseEndpoint: aws.String(endpoint),
+		UsePathStyle: true,
+		Credentials:  credentials.NewStaticCredentialsProvider(s3cfg.AccessKey, s3cfg.SecretKey, ""),
+		HTTPClient:   insecureHTTPClient(30 * time.Second),
+	})
+}
+
+// withHost replaces the host of an endpoint, keeping its scheme and port.
+func withHost(endpoint, host string) string {
+	scheme := ""
+	rest := endpoint
+	if i := strings.Index(endpoint, "://"); i >= 0 {
+		scheme, rest = endpoint[:i+3], endpoint[i+3:]
+	}
+	if _, port, err := net.SplitHostPort(rest); err == nil {
+		return scheme + net.JoinHostPort(host, port)
+	}
+	return scheme + host
+}
+
+func nodeClusterConfig(t *testing.T, host string) *config.ClusterConfig {
+	t.Helper()
+	body, err := readNodeFile(host, "/etc/spinifex/spinifex.toml")
+	if err != nil {
+		t.Fatalf("read spinifex.toml on %s: %v", hostLabel(host), err)
+	}
+	cc, err := loadClusterConfigBytes([]byte(body))
+	if err != nil {
+		t.Fatalf("decode spinifex.toml from %s: %v", hostLabel(host), err)
+	}
+	return cc
+}

tests/e2e/multinode/dns_test.go

@@ -62,6 +62,7 @@ func runMultinodeDNS(t *testing.T, fix *Fixture) {
 	firstNode := fix.Cluster.Nodes[0]
 	baseDomain, internalDomain := harness.PeerNorthstarDomains(t, firstNode)
 	firstConfig := configs[firstNode.Name]
+	require.Zerof(t, firstConfig.Northstar, "expected nameservers assume no [northstar] pin or count on %s", firstNode.Name)
 	expectedSeeds := make([]nsconfig.NameserverSeed, 0, len(fix.Cluster.Nodes))
 	expectedResolvers := make([]string, 0, len(fix.Cluster.Nodes))
 	for i, node := range fix.Cluster.Nodes {
@@ -69,7 +70,9 @@ func runMultinodeDNS(t *testing.T, fix *Fixture) {
 		require.NotNilf(t, ip, "cluster node %s address %q is not an IP", node.Name, node.Addr)
 		require.Falsef(t, ip.IsLoopback() || ip.IsUnspecified(), "cluster node %s has unusable DNS address %s", node.Name, ip)
 		expectedResolvers = append(expectedResolvers, ip.String())
-		expectedSeeds = append(expectedSeeds, nsconfig.NameserverSeed{Host: fmt.Sprintf("ns%d", i+1), IP: ip.String()})
+		if i < spinconfig.MaxNameservers {
+			expectedSeeds = append(expectedSeeds, nsconfig.NameserverSeed{Host: fmt.Sprintf("ns%d", i+1), IP: ip.String()})
+		}
 	}
 	expectedTopology := seedTopology(baseDomain, expectedSeeds)
 

tests/e2e/route53/hosted_zone_test.go

@@ -0,0 +1,223 @@
+//go:build e2e
+
+package route53
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/request"
+	r53 "github.com/aws/aws-sdk-go/service/route53"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/route53/ext"
+	"github.com/mulgadc/spinifex/tests/e2e/harness"
+)
+
+const (
+	wwwAddr = "192.0.2.80"
+	apiAddr = "192.0.2.81"
+	txtMark = "r53-e2e"
+
+	// A check, a publish and a reload on every node, with headroom.
+	activationTimeout = 3 * time.Minute
+	syncTimeout       = 2 * time.Minute
+)
+
+// A customer's hosted zone is served only once its parent delegates it to the
+// cluster, every node serves a change before it reports INSYNC, a second
+// account cannot take the same name, and deleting the zone stops it resolving.
+// The zone is a delegated subdomain of the cluster's own domain, so the real
+// delegation check runs with no registrar and no internet.
+func TestHostedZoneServedOnceDelegated(t *testing.T) {
+	env := harness.LoadEnv(t)
+	targets := harness.DNSTargets(env)
+	base := harness.NorthstarBaseDomain(env)
+	if base == "" && env.Mode == harness.ModeMultinode {
+		base, _ = harness.PeerNorthstarDomains(t, targets[0])
+	}
+	require.NotEmpty(t, base, "northstar base domain is not configured")
+
+	owner := newTenant(t, env, "r53-owner")
+	c := owner.Client.Route53
+	name := fmt.Sprintf("z%d.r53e2e.%s", time.Now().UnixNano()%1_000_000_000, base)
+	www, api := "www."+name, "api."+name
+
+	harness.Phase(t, "Route 53 — create %s", name)
+	created, err := c.CreateHostedZone(&r53.CreateHostedZoneInput{Name: aws.String(name), CallerReference: aws.String(name)})
+	require.NoError(t, err)
+	zoneID := aws.StringValue(created.HostedZone.Id)
+	t.Cleanup(func() { deleteZone(t, c, zoneID) })
+	nameservers := aws.StringValueSlice(created.DelegationSet.NameServers)
+	require.NotEmpty(t, nameservers)
+	for _, ns := range nameservers {
+		assert.True(t, strings.HasSuffix(ns, "."+base), "%s is not one of the cluster's name servers", ns)
+	}
+	waitInSync(t, c, created.ChangeInfo.Id)
+
+	change, err := c.ChangeResourceRecordSets(&r53.ChangeResourceRecordSetsInput{
+		HostedZoneId: aws.String(zoneID),
+		ChangeBatch: &r53.ChangeBatch{Changes: []*r53.Change{
+			rrset("CREATE", www, "A", wwwAddr),
+			rrset("CREATE", name, "TXT", `"`+txtMark+`"`),
+		}},
+	})
+	require.NoError(t, err)
+	waitInSync(t, c, change.ChangeInfo.Id)
+
+	harness.Phase(t, "Route 53 — an undelegated zone changes no answer")
+	assert.Equal(t, "PENDING_NAMESERVERS", activation(t, c, zoneID, false).State)
+	for _, node := range targets {
+		require.NoError(t, wantGone(node, www), "an inactive zone must not be served")
+	}
+
+	harness.Phase(t, "Route 53 — delegate %s in %s and activate", name, base)
+	harness.SpxRun(t, false, "admin", "dns", "delegate", name)
+	t.Cleanup(func() { harness.SpxRunBestEffort(t, "admin", "dns", "delegate", "--remove", name) })
+	harness.EventuallyErr(t, func() error {
+		if a := activation(t, c, zoneID, true); a.State != "ACTIVE" {
+			return fmt.Errorf("zone is %s: %s", a.State, a.Reason)
+		}
+		return nil
+	}, activationTimeout, 5*time.Second)
+
+	harness.Phase(t, "Route 53 — every node serves the zone")
+	for _, node := range targets {
+		harness.EventuallyErr(t, func() error { return wantAddr(node, www, wwwAddr) }, syncTimeout, 5*time.Second)
+		require.NoError(t, wantTXT(node, name, txtMark))
+	}
+
+	harness.Phase(t, "Route 53 — a change is INSYNC only once every node serves it")
+	change, err = c.ChangeResourceRecordSets(&r53.ChangeResourceRecordSetsInput{
+		HostedZoneId: aws.String(zoneID),
+		ChangeBatch:  &r53.ChangeBatch{Changes: []*r53.Change{rrset("UPSERT", api, "A", apiAddr)}},
+	})
+	require.NoError(t, err)
+	waitInSync(t, c, change.ChangeInfo.Id)
+	for _, node := range targets {
+		require.NoError(t, wantAddr(node, api, apiAddr), "INSYNC was reported before %s served the change", node.Name)
+	}
+
+	harness.Phase(t, "Route 53 — a second account cannot take an active name")
+	rival := newTenant(t, env, "r53-rival").Client.Route53
+	contested, err := rival.CreateHostedZone(&r53.CreateHostedZoneInput{Name: aws.String(name), CallerReference: aws.String("rival-" + name)})
+	require.NoError(t, err)
+	rivalID := aws.StringValue(contested.HostedZone.Id)
+	t.Cleanup(func() { deleteZone(t, rival, rivalID) })
+	a := activation(t, rival, rivalID, true)
+	assert.Equal(t, "PENDING_NAMESERVERS", a.State)
+	assert.Equal(t, "the domain is active in another account", a.Reason)
+	_, err = rival.GetHostedZone(&r53.GetHostedZoneInput{Id: aws.String(zoneID)})
+	require.Error(t, err, "one account must not see another's zone")
+
+	harness.Phase(t, "Route 53 — a deleted zone stops resolving")
+	deleteZone(t, c, zoneID)
+	for _, node := range targets {
+		harness.EventuallyErr(t, func() error { return wantGone(node, www) }, syncTimeout, 5*time.Second)
+	}
+}
+
+func rrset(action, name, rrType, value string) *r53.Change {
+	return &r53.Change{Action: aws.String(action), ResourceRecordSet: &r53.ResourceRecordSet{
+		Name: aws.String(name), Type: aws.String(rrType), TTL: aws.Int64(60),
+		ResourceRecords: []*r53.ResourceRecord{{Value: aws.String(value)}},
+	}}
+}
+
+func waitInSync(t *testing.T, c *r53.Route53, id *string) {
+	t.Helper()
+	harness.EventuallyErr(t, func() error {
+		out, err := c.GetChange(&r53.GetChangeInput{Id: id})
+		if err != nil {
+			return err
+		}
+		if s := aws.StringValue(out.ChangeInfo.Status); s != r53.ChangeStatusInsync {
+			return fmt.Errorf("change %s is %s", aws.StringValue(id), s)
+		}
+		return nil
+	}, syncTimeout, 2*time.Second)
+}
+
+// activation reads, or with verify checks now, the zone's activation through
+// the Spinifex extension, called as a custom SDK operation.
+func activation(t *testing.T, c *r53.Route53, zoneID string, verify bool) (a struct{ State, Reason string }) {
+	t.Helper()
+	id := aws.String(strings.TrimPrefix(zoneID, "/hostedzone/"))
+	path := "/2013-04-01/hostedzone/{Id}/activation"
+	var got *ext.Activation
+	if verify {
+		out := &ext.VerifyHostedZoneDelegationOutput{}
+		op := &request.Operation{Name: ext.ActionVerifyHostedZoneDelegation, HTTPMethod: http.MethodPost, HTTPPath: path}
+		require.NoError(t, c.NewRequest(op, &ext.VerifyHostedZoneDelegationInput{Id: id}, out).Send())
+		got = out.Activation
+	} else {
+		out := &ext.GetHostedZoneActivationOutput{}
+		op := &request.Operation{Name: ext.ActionGetHostedZoneActivation, HTTPMethod: http.MethodGet, HTTPPath: path}
+		require.NoError(t, c.NewRequest(op, &ext.GetHostedZoneActivationInput{Id: id}, out).Send())
+		got = out.Activation
+	}
+	require.NotNil(t, got)
+	a.State, a.Reason = aws.StringValue(got.State), aws.StringValue(got.Reason)
+	return a
+}
+
+// deleteZone removes every record set but the apex NS and SOA, then the zone.
+// A zone already gone is not an error, so it serves as a cleanup too.
+func deleteZone(t *testing.T, c *r53.Route53, zoneID string) {
+	t.Helper()
+	zone, err := c.GetHostedZone(&r53.GetHostedZoneInput{Id: aws.String(zoneID)})
+	if err != nil {
+		return
+	}
+	sets, err := c.ListResourceRecordSets(&r53.ListResourceRecordSetsInput{HostedZoneId: aws.String(zoneID)})
+	if err != nil {
+		t.Logf("list record sets of %s: %v", zoneID, err)
+		return
+	}
+	apex := aws.StringValue(zone.HostedZone.Name)
+	var changes []*r53.Change
+	for _, s := range sets.ResourceRecordSets {
+		if aws.StringValue(s.Name) == apex && slices.Contains([]string{"NS", "SOA"}, aws.StringValue(s.Type)) {
+			continue
+		}
+		changes = append(changes, &r53.Change{Action: aws.String("DELETE"), ResourceRecordSet: s})
+	}
+	if len(changes) > 0 {
+		if _, err := c.ChangeResourceRecordSets(&r53.ChangeResourceRecordSetsInput{
+			HostedZoneId: aws.String(zoneID), ChangeBatch: &r53.ChangeBatch{Changes: changes},
+		}); err != nil {
+			t.Logf("delete record sets of %s: %v", zoneID, err)
+		}
+	}
+	if _, err := c.DeleteHostedZone(&r53.DeleteHostedZoneInput{Id: aws.String(zoneID)}); err != nil {
+		t.Logf("delete hosted zone %s: %v", zoneID, err)
+	}
+}
+
+func wantTXT(node harness.Node, name, want string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	txt, err := harness.NodeResolver(node).LookupTXT(ctx, name)
+	if err != nil {
+		return fmt.Errorf("TXT %s via %s: %w", name, node.Name, err)
+	}
+	if !slices.Contains(txt, want) {
+		return fmt.Errorf("TXT %s via %s = %v, want %q", name, node.Name, txt, want)
+	}
+	return nil
+}
+
+// newTenant creates an account the way self-service signup does, with a
+// client scoped to it.
+func newTenant(t *testing.T, env *harness.Env, label string) *harness.Profile {
+	t.Helper()
+	info := harness.SpxAdminAccountCreate(t, fmt.Sprintf("E2E %s %d", label, time.Now().UnixNano()), "")
+	return harness.NewAccountCarousel().Add(t, env, label, info)
+}

tests/e2e/route53/main_test.go

@@ -0,0 +1,6 @@
+//go:build e2e
+
+// Package route53 is the hosted-zone E2E suite. It drives northstar, the
+// authoritative DNS server behind Route 53, in both single and multinode
+// topologies, because every node serves every zone from the same bucket.
+package route53

tests/e2e/route53/northstar_scale_test.go

@@ -0,0 +1,161 @@
+//go:build e2e
+
+package route53
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/tests/e2e/harness"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	// More zone objects than one ListObjectsV2 page holds (1,000), so a
+	// listing that stops at the first page leaves zones unloaded.
+	scaleZoneCount = 1100
+	scaleZoneAddr  = "192.0.2.53"
+	scaleWorkers   = 8
+
+	// Two 30 s sync cycles plus reading 1,100 objects, with headroom.
+	scaleSyncTimeout = 4 * time.Minute
+)
+
+// scaleZone sorts every zone ahead of the platform base zone, so a purge that
+// trusts a truncated listing would take the base zone with it.
+func scaleZone(i int) string {
+	return fmt.Sprintf("a%04d.zonescale.test", i)
+}
+
+// Northstar loads every zone in its bucket, not just the first listing page,
+// and a sync never purges a zone that is still there. The last zone written
+// must resolve on every node, the platform base zone must survive the sync,
+// and deleting the objects must stop them resolving.
+func TestNorthstarServesZonesBeyondFirstListPage(t *testing.T) {
+	env := harness.LoadEnv(t)
+	targets := harness.DNSTargets(env)
+	store := harness.NorthstarZoneStore(t, env, targets[0])
+	base := harness.NorthstarBaseDomain(env)
+	if base == "" && env.Mode == harness.ModeMultinode {
+		base, _ = harness.PeerNorthstarDomains(t, targets[0])
+	}
+	require.NotEmpty(t, base, "northstar base domain is not configured")
+
+	harness.Phase(t, "Northstar — write %d zone objects", scaleZoneCount)
+	t.Cleanup(func() {
+		if err := forEachZone(func(name string) error {
+			return harness.DeleteTestZone(context.Background(), store, name)
+		}); err != nil {
+			t.Logf("cleanup: %v", err)
+		}
+	})
+	require.NoError(t, forEachZone(func(name string) error {
+		return harness.WriteTestZone(store, name, scaleZoneAddr)
+	}))
+
+	first, last := scaleZone(1), scaleZone(scaleZoneCount)
+	harness.Phase(t, "Northstar — every node serves the first and last zone")
+	for _, node := range targets {
+		harness.EventuallyErr(t, func() error {
+			for _, name := range []string{first, last} {
+				if err := wantAddr(node, name, scaleZoneAddr); err != nil {
+					return err
+				}
+			}
+			return nil
+		}, scaleSyncTimeout, 5*time.Second)
+	}
+
+	harness.Phase(t, "Northstar — the platform base zone survives the sync")
+	for _, node := range targets {
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		ns, err := harness.NodeResolver(node).LookupNS(ctx, base)
+		cancel()
+		require.NoError(t, err, "base zone %s on %s", base, node.Name)
+		require.NotEmpty(t, ns, "base zone %s on %s has no NS records", base, node.Name)
+	}
+
+	harness.Phase(t, "Northstar — deleted zone objects stop resolving")
+	require.NoError(t, forEachZone(func(name string) error {
+		return harness.DeleteTestZone(context.Background(), store, name)
+	}))
+	for _, node := range targets {
+		harness.EventuallyErr(t, func() error {
+			for _, name := range []string{first, last} {
+				if err := wantGone(node, name); err != nil {
+					return err
+				}
+			}
+			return nil
+		}, scaleSyncTimeout, 5*time.Second)
+	}
+}
+
+// forEachZone runs fn over every scale zone with a small worker pool and
+// returns the first error.
+func forEachZone(fn func(name string) error) error {
+	names := make(chan string)
+	var (
+		wg       sync.WaitGroup
+		mu       sync.Mutex
+		firstErr error
+	)
+	for range scaleWorkers {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for name := range names {
+				if err := fn(name); err != nil {
+					mu.Lock()
+					if firstErr == nil {
+						firstErr = fmt.Errorf("%s: %w", name, err)
+					}
+					mu.Unlock()
+				}
+			}
+		}()
+	}
+	for i := 1; i <= scaleZoneCount; i++ {
+		names <- scaleZone(i)
+	}
+	close(names)
+	wg.Wait()
+	return firstErr
+}
+
+func wantAddr(node harness.Node, name, addr string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	addrs, err := harness.ResolveViaNode(ctx, node, name)
+	if err != nil {
+		return err
+	}
+	for _, a := range addrs {
+		if a == addr {
+			return nil
+		}
+	}
+	return fmt.Errorf("%s via %s = %v, want %s", name, node.Name, addrs, addr)
+}
+
+// wantGone accepts any definite answer other than an address: NXDOMAIN when
+// the name is recursed upstream, REFUSED when the listener does not recurse
+// for this client. A timeout proves nothing, so it is not accepted.
+func wantGone(node harness.Node, name string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	addrs, err := harness.NodeResolver(node).LookupHost(ctx, name)
+	if err == nil {
+		return fmt.Errorf("%s via %s still resolves to %v", name, node.Name, addrs)
+	}
+	var dnsErr *net.DNSError
+	if errors.As(err, &dnsErr) && dnsErr.IsTimeout {
+		return fmt.Errorf("%s via %s: %w", name, node.Name, err)
+	}
+	return nil
+}

tests/integration/harness.go

@@ -32,9 +32,11 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/ecr"
 	"github.com/aws/aws-sdk-go/service/iam"
+	"github.com/aws/aws-sdk-go/service/route53"
 	"github.com/aws/aws-sdk-go/service/sts"
 	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
 	acmawsapi "github.com/mulgadc/spinifex/spinifex/domains/acm/awsapi"
+	route53awsapi "github.com/mulgadc/spinifex/spinifex/domains/route53/awsapi"
 	ecrdomain "github.com/mulgadc/spinifex/spinifex/domains/ecr"
 	ecrauth "github.com/mulgadc/spinifex/spinifex/domains/ecr/auth"
 	awsapi "github.com/mulgadc/spinifex/spinifex/domains/ecr/awsapi"
@@ -183,6 +185,7 @@ func startGateway(t *testing.T, collector *conformanceCollector, opts ...Option)
 		NATS:               nc,
 	})))
 	require.NoError(t, services.Register(acmawsapi.NewRegistration(acmawsapi.Deps{NATS: nc})))
+	require.NoError(t, services.Register(route53awsapi.NewRegistration(route53awsapi.Deps{NATS: nc})))
 
 	cfg := &gateway.GatewayConfig{
 		DisableLogging: true,
@@ -285,6 +288,8 @@ type PrincipalClients struct {
 	IAM *iam.IAM
 	STS *sts.STS
 	ECR *ecr.ECR
+	// Route53 is global; the gateway accepts its signature in any region.
+	Route53 *route53.Route53
 }
 
 // ClientsWithCreds returns PrincipalClients signed with a static access
@@ -323,10 +328,11 @@ func (gw *Gateway) principalClients(t *testing.T, accessKeyID, secretAccessKey,
 	})
 	require.NoError(t, err)
 	return &PrincipalClients{
-		EC2: ec2.New(sess),
-		IAM: iam.New(sess),
-		STS: sts.New(sess),
-		ECR: ecr.New(sess),
+		EC2:     ec2.New(sess),
+		IAM:     iam.New(sess),
+		STS:     sts.New(sess),
+		ECR:     ecr.New(sess),
+		Route53: route53.New(sess),
 	}
 }
 

tests/integration/route53_test.go

@@ -0,0 +1,55 @@
+//go:build integration
+
+package integration
+
+import (
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/route53"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// A hosted zone is visible to its own account only: another tenant gets
+// NoSuchHostedZone for every call naming it, exactly as for a zone that does
+// not exist, and its own listing stays empty.
+func TestRoute53_HostedZonesAreAccountScoped(t *testing.T) {
+	gw := StartGateway(t)
+	StartServiceDaemonLite(t, gw)
+	owner := createTenantAccount(t, gw, "route53-owner").Route53
+	other := createTenantAccount(t, gw, "route53-other").Route53
+
+	created, err := owner.CreateHostedZone(&route53.CreateHostedZoneInput{
+		Name: aws.String("example.com"), CallerReference: aws.String("route53-scoping"),
+	})
+	require.NoError(t, err)
+	zoneID := created.HostedZone.Id
+	assert.Equal(t, []string{"ns1.spx3.net"}, aws.StringValueSlice(created.DelegationSet.NameServers))
+
+	_, err = owner.ChangeResourceRecordSets(&route53.ChangeResourceRecordSetsInput{
+		HostedZoneId: zoneID,
+		ChangeBatch: &route53.ChangeBatch{Changes: []*route53.Change{{
+			Action: aws.String("CREATE"),
+			ResourceRecordSet: &route53.ResourceRecordSet{
+				Name: aws.String("www.example.com"), Type: aws.String("A"), TTL: aws.Int64(300),
+				ResourceRecords: []*route53.ResourceRecord{{Value: aws.String("192.0.2.10")}},
+			},
+		}}},
+	})
+	require.NoError(t, err)
+
+	_, err = other.GetHostedZone(&route53.GetHostedZoneInput{Id: zoneID})
+	assertAWSErrorCode(t, err, "NoSuchHostedZone", "foreign GetHostedZone")
+	_, err = other.ListResourceRecordSets(&route53.ListResourceRecordSetsInput{HostedZoneId: zoneID})
+	assertAWSErrorCode(t, err, "NoSuchHostedZone", "foreign ListResourceRecordSets")
+	_, err = other.DeleteHostedZone(&route53.DeleteHostedZoneInput{Id: zoneID})
+	assertAWSErrorCode(t, err, "NoSuchHostedZone", "foreign DeleteHostedZone")
+	listed, err := other.ListHostedZones(&route53.ListHostedZonesInput{})
+	require.NoError(t, err)
+	assert.Empty(t, listed.HostedZones)
+
+	sets, err := owner.ListResourceRecordSets(&route53.ListResourceRecordSetsInput{HostedZoneId: zoneID})
+	require.NoError(t, err)
+	assert.Len(t, sets.ResourceRecordSets, 3, "NS, SOA and the A record")
+}

tests/integration/service_daemon_lite.go

@@ -8,15 +8,17 @@ import (
 
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	acmdomain "github.com/mulgadc/spinifex/spinifex/domains/acm"
+	route53domain "github.com/mulgadc/spinifex/spinifex/domains/route53"
 	handlers_ecs "github.com/mulgadc/spinifex/spinifex/handlers/ecs"
 	handlers_eks "github.com/mulgadc/spinifex/spinifex/handlers/eks"
 	handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2"
 	handlers_iam "github.com/mulgadc/spinifex/spinifex/handlers/iam"
 	handlers_rds "github.com/mulgadc/spinifex/spinifex/handlers/rds"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/require"
 )
 
-// StartServiceDaemonLite subscribes the real ACM, ECS, EKS, ELBv2 and RDS
+// StartServiceDaemonLite subscribes the real ACM, ECS, EKS, ELBv2, RDS and Route 53
 // services to their "<service>.<Method>" subjects. ECS and RDS run without
 // their provisioning dependencies, so checks that need them do not run.
 func StartServiceDaemonLite(t *testing.T, gw *Gateway) {
@@ -49,4 +51,10 @@ func StartServiceDaemonLite(t *testing.T, gw *Gateway) {
 
 	subscribeServiceMethods(t, nc, "ecs", handlers_ecs.NewService(nc, testRegion, ""))
 	subscribeServiceMethods(t, nc, "rds", handlers_rds.NewService(nc, testRegion))
+
+	js, err := jetstream.New(nc)
+	require.NoError(t, err, "open JetStream for Route 53")
+	route53 := route53domain.NewService(route53domain.NewStore(js))
+	route53.Nameservers = func() []string { return []string{"ns1.spx3.net"} }
+	subscribeServiceMethods(t, nc, "route53", route53)
 }