merge: replace the two-VM SQS candidate with the single-VM design

Bring the draft onto the reviewed single-VM implementation: the worker
and its dedicated SQS NATS server run in one system microVM on one
encrypted volume, and the separate store VM, its image and its cross-VM
bootstrap and authentication are removed. The tree is the reviewed
single-VM head on current dev.

.github/workflows/dev-release.yml

@@ -104,7 +104,7 @@ jobs:
 
       - name: Build microVM artifacts (lb-agent + kernel + initramfs + qboot + SQS store and worker images)
         if: steps.version.outputs.exists != 'true'
-        run: cd spinifex && make build-microvm-image build-sqs-store-microvm-image build-sqs-worker-microvm-image
+        run: cd spinifex && make build-microvm-image build-sqs-worker-microvm-image
 
       - name: Build amd64 Tarball
         if: steps.version.outputs.exists != 'true'

.github/workflows/release.yml

@@ -58,7 +58,7 @@ jobs:
           cache: false
 
       - name: Build microVM artifacts (lb-agent + kernel + initramfs + qboot + SQS store and worker images)
-        run: cd spinifex && make build-microvm-image build-sqs-store-microvm-image build-sqs-worker-microvm-image
+        run: cd spinifex && make build-microvm-image build-sqs-worker-microvm-image
 
       - name: Build amd64 Tarball
         env:

.gitignore

@@ -21,9 +21,6 @@ build/microvm/vmlinuz
 build/microvm/initramfs.cpio.gz
 build/microvm/qboot.rom
 build/microvm/qboot.LICENSE
-build/microvm/sqs-store/vmlinuz
-build/microvm/sqs-store/initramfs.cpio.gz
-build/microvm/sqs-store/.stage.*/
 build/microvm/sqs-worker/vmlinuz
 build/microvm/sqs-worker/initramfs.cpio.gz
 build/microvm/sqs-worker/.stage.*/

Makefile

@@ -153,37 +153,32 @@ $(MICROVM_ARTIFACTS) &: $(MICROVM_INPUTS)
 bin/lb-agent:
 	$(MAKE) build-lb-agent
 
-# The SQS store microVM image: a separate artifact from the LB image above, not
-# installed or released by any target yet.
-SQS_STORE_MICROVM_OUT_DIR := $(MICROVM_OUT_DIR)/sqs-store
-SQS_STORE_MICROVM_ARTIFACTS := $(SQS_STORE_MICROVM_OUT_DIR)/vmlinuz $(SQS_STORE_MICROVM_OUT_DIR)/initramfs.cpio.gz
-SQS_STORE_MICROVM_INPUTS := scripts/build-sqs-store-microvm-image.sh scripts/lib/sqs-store-microvm-publish.sh $(SQS_STORE_MICROVM_OUT_DIR)/init.sh $(SQS_STORE_MICROVM_OUT_DIR)/inittab $(addprefix $(SQS_STORE_MICROVM_OUT_DIR)/,sqs-store-lib.sh sqs-store-prepare sqs-store-run sqs-store-stop) bin/nats-server
-
-$(SQS_STORE_MICROVM_ARTIFACTS) &: $(SQS_STORE_MICROVM_INPUTS)
-	./scripts/build-sqs-store-microvm-image.sh
-
-# The dedicated SQS store server, built from the go.mod-pinned module so the
-# image ships exactly the reviewed version. Static for the musl guest.
+# The dedicated SQS server, built from the go.mod-pinned module so the image
+# ships exactly the reviewed version. Static for the musl guest.
 bin/nats-server: go.mod go.sum
 	@mkdir -p bin
 	GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags '-s -w' -o $@ github.com/nats-io/nats-server/v2
 
-build-sqs-store-microvm-image: $(SQS_STORE_MICROVM_ARTIFACTS) ## Build the SQS store microVM kernel and initramfs
+# go_local_srcs lists the main module package dirs and files guest package $(1)
+# links, scanned with the guest build settings; a dir's mtime catches a deleted
+# or renamed file. A failed scan names a missing prerequisite, so make stops.
+go_local_srcs = $(shell GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go list -deps -f '{{if and .Module .Module.Main}}{{.Dir}} {{range .GoFiles}}{{$$.Dir}}/{{.}} {{end}}{{range .EmbedFiles}}{{$$.Dir}}/{{.}} {{end}}{{end}}' $(1) || echo go-list-failed-for-$(1))
+.SECONDEXPANSION:
 
-# The shared SQS worker microVM image: the store image's /init and kernel with
-# the worker binary instead of nats-server. Not installed or released yet.
+# The SQS microVM image: one guest running its own SQS nats-server and the SQS
+# worker as separate processes. A separate artifact from the LB image above.
 SQS_WORKER_MICROVM_OUT_DIR := $(MICROVM_OUT_DIR)/sqs-worker
 SQS_WORKER_MICROVM_ARTIFACTS := $(SQS_WORKER_MICROVM_OUT_DIR)/vmlinuz $(SQS_WORKER_MICROVM_OUT_DIR)/initramfs.cpio.gz
-SQS_WORKER_MICROVM_INPUTS := scripts/build-sqs-store-microvm-image.sh scripts/lib/sqs-store-microvm-publish.sh $(SQS_STORE_MICROVM_OUT_DIR)/init.sh $(SQS_WORKER_MICROVM_OUT_DIR)/inittab $(addprefix $(SQS_WORKER_MICROVM_OUT_DIR)/,sqs-worker-lib.sh sqs-worker-prepare sqs-worker-ready sqs-worker-stop) bin/sqs-worker
+SQS_WORKER_MICROVM_INPUTS := scripts/build-sqs-store-microvm-image.sh scripts/lib/sqs-store-microvm-publish.sh $(SQS_WORKER_MICROVM_OUT_DIR)/init.sh $(SQS_WORKER_MICROVM_OUT_DIR)/inittab $(addprefix $(SQS_WORKER_MICROVM_OUT_DIR)/,sqs-worker-lib.sh sqs-worker-prepare sqs-worker-store sqs-worker-ready sqs-worker-stop) bin/nats-server bin/sqs-worker
 
 $(SQS_WORKER_MICROVM_ARTIFACTS) &: $(SQS_WORKER_MICROVM_INPUTS)
-	SQS_MICROVM_PROFILE=worker ./scripts/build-sqs-store-microvm-image.sh
+	./scripts/build-sqs-store-microvm-image.sh
 
-bin/sqs-worker: go.mod go.sum $(shell find cmd/sqs-worker spinifex/agents/sqs -name '*.go' ! -name '*_test.go')
+bin/sqs-worker: go.mod go.sum $$(call go_local_srcs,./cmd/sqs-worker)
 	@mkdir -p bin
 	GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags '-s -w' -o $@ ./cmd/sqs-worker
 
-build-sqs-worker-microvm-image: $(SQS_WORKER_MICROVM_ARTIFACTS) ## Build the shared SQS worker microVM kernel and initramfs
+build-sqs-worker-microvm-image: $(SQS_WORKER_MICROVM_ARTIFACTS) ## Build the SQS microVM kernel and initramfs
 
 # qboot is the microvm firmware. The source tarball ships with each release
 # because qboot is GPL-2.0.
@@ -196,13 +191,7 @@ $(QBOOT_ARTIFACTS) &: scripts/build-qboot.sh
 
 build-microvm-image: $(MICROVM_ARTIFACTS) $(QBOOT_ARTIFACTS) ## Build microVM kernel, initramfs and qboot firmware (incremental — skips when up to date)
 
-install-sqs-store-microvm: $(SQS_STORE_MICROVM_ARTIFACTS) ## Install the SQS store microVM image to /usr/share/spinifex/microvm/sqs-store/
-	sudo install -d /usr/share/spinifex/microvm/sqs-store
-	sudo rm -f /usr/share/spinifex/microvm/sqs-store/initramfs.cpio.gz
-	sudo install -m 0644 $(SQS_STORE_MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/sqs-store/vmlinuz
-	sudo install -m 0644 $(SQS_STORE_MICROVM_OUT_DIR)/initramfs.cpio.gz /usr/share/spinifex/microvm/sqs-store/initramfs.cpio.gz
-
-install-sqs-worker-microvm: $(SQS_WORKER_MICROVM_ARTIFACTS) ## Install the SQS worker microVM image to /usr/share/spinifex/microvm/sqs-worker/
+install-sqs-worker-microvm: $(SQS_WORKER_MICROVM_ARTIFACTS) ## Install the SQS microVM image to /usr/share/spinifex/microvm/sqs-worker/
 	sudo install -d /usr/share/spinifex/microvm/sqs-worker
 	sudo rm -f /usr/share/spinifex/microvm/sqs-worker/initramfs.cpio.gz
 	sudo install -m 0644 $(SQS_WORKER_MICROVM_OUT_DIR)/vmlinuz /usr/share/spinifex/microvm/sqs-worker/vmlinuz
@@ -514,7 +503,6 @@ distro-amd64:
 		echo "[distro-amd64] WARNING: microvm artifacts missing — tarball will not include them"; \
 		echo "[distro-amd64]          run 'make build-microvm-image' before 'make distro-amd64'"; \
 	fi
-	./scripts/lib/stage-sqs-store-microvm.sh $(SQS_STORE_MICROVM_OUT_DIR) dist/amd64
 	./scripts/lib/stage-sqs-store-microvm.sh $(SQS_WORKER_MICROVM_OUT_DIR) dist/amd64 sqs-worker
 	tar -czf dist/spinifex-$(VERSION)-linux-amd64.tar.gz -C dist/amd64 .
 	sha256sum dist/spinifex-$(VERSION)-linux-amd64.tar.gz > dist/spinifex-$(VERSION)-linux-amd64.tar.gz.sha256
@@ -534,7 +522,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: test-package-check build build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-store-microvm-image build-sqs-worker-microvm-image install-microvm install-sqs-store-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
+.PHONY: test-package-check build build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-worker-microvm-image install-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \

architecture/layering/allowlist_test.go

@@ -38,6 +38,7 @@ var allowlist = []allowance{
 	{From: "cmd/aws-model-coverage", To: "domains/ecs/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
 	{From: "cmd/aws-model-coverage", To: "domains/eks/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
 	{From: "cmd/aws-model-coverage", To: "domains/iam/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
+	{From: "cmd/aws-model-coverage", To: "domains/sqs/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
 	{From: "cmd/aws-model-coverage", To: "domains/sts/awsapi", Debt: "cmd/aws-model-coverage", Reason: "reads declared service inventories; named exception, must not construct a registration"},
 
 	{From: "runtime/daemon", To: "domains/ec2/awsapi/instance", Debt: "Instance adapter consumers outside EC2", Reason: "runtime/daemon/eks_cp_control.go and rds_deps.go call the instance adapter; needs an EC2 capability"},

build/microvm/sqs-store/inittab

@@ -1,10 +0,0 @@
-# The power button loads here, not in /init: busybox init has its SIGUSR2
-# (poweroff) handler installed before sysinit runs, so a press can never reach
-# the pre-exec shell PID 1. A press before this line is lost; the host re-sends.
-::sysinit:/sbin/modprobe tiny_power_button power_signal=12
-::sysinit:/usr/local/lib/sqs-store/sqs-store-prepare
-::respawn:/usr/local/lib/sqs-store/sqs-store-run
-::ctrlaltdel:/sbin/reboot
-# One action: the store stops and its disk unmounts first, and the generic
-# umount -a -r runs only if the store is gone.
-::shutdown:/usr/local/lib/sqs-store/sqs-store-stop

build/microvm/sqs-store/sqs-store-lib.sh

@@ -1,224 +0,0 @@
-# sqs-store-lib.sh — shared by the SQS store guest's prepare, run and stop
-# steps. Sourced, never executed. Every path and command is overridable so the
-# host test can drive the same code against a fake fw_cfg, sysfs and tools.
-#
-# Guest contract, read from fw_cfg under opt/spinifex/sqs-store/:
-#   data-serial  virtio-blk serial of the already-formatted ext4 data disk
-#   listen-addr  IPv4 address on a configured guest NIC; the only listener
-#   client-nkey  NATS user NKey public key allowed to connect; never a seed
-# Anything absent, malformed or ambiguous refuses the store. Nothing formats.
-
-SQS_FWCFG=${SQS_FWCFG:-/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/sqs-store}
-SQS_SYSBLOCK=${SQS_SYSBLOCK:-/sys/block}
-SQS_DEVDIR=${SQS_DEVDIR:-/dev}
-# The host test cannot create block nodes, so it relaxes this to -e.
-SQS_NODE_TEST=${SQS_NODE_TEST:--b}
-SQS_RUN=${SQS_RUN:-/run/sqs-store}
-SQS_DATA=${SQS_DATA:-/data}
-SQS_CONSOLE=${SQS_CONSOLE:-/dev/console}
-SQS_MOUNTS=${SQS_MOUNTS:-/proc/mounts}
-SQS_PORT=${SQS_PORT:-4222}
-SQS_STOP_BUDGET_TENTHS=${SQS_STOP_BUDGET_TENTHS:-120}
-NATS_BIN=${NATS_BIN:-/usr/local/bin/nats-server}
-E2FSCK=${E2FSCK:-e2fsck}
-MOUNT=${MOUNT:-mount}
-UMOUNT=${UMOUNT:-umount}
-IP=${IP:-ip}
-
-sqs_log() { echo "[sqs-store] $*" >> "$SQS_CONSOLE"; }
-sqs_fatal() { sqs_log "FATAL: $*"; return 1; }
-
-# sqs_fwcfg <name>: print one fw_cfg value, refusing a missing, empty, oversized
-# or multi-token file. The value is not echoed on failure.
-sqs_fwcfg() {
-    f="$SQS_FWCFG/$1/raw"
-    [ -f "$f" ] || { sqs_fatal "fw_cfg $1 not supplied"; return 1; }
-    [ "$(wc -c < "$f")" -le 128 ] || { sqs_fatal "fw_cfg $1 is oversized"; return 1; }
-    v=$(cat "$f")
-    case "$v" in
-        "" | *[[:space:]]*) sqs_fatal "fw_cfg $1 must be exactly one value"; return 1 ;;
-    esac
-    printf '%s' "$v"
-}
-
-# A volume serial is the volume ID with dashes stripped: vol + 17 hex.
-sqs_valid_serial() {
-    [ "${#1}" -eq 20 ] || return 1
-    case "$1" in vol*) ;; *) return 1 ;; esac
-    case "${1#vol}" in *[!0-9a-f]*) return 1 ;; esac
-}
-
-# A NATS user public NKey: "U" then 55 base32 characters. The checksum is
-# left to the pinned server's own config test.
-sqs_valid_user_nkey() {
-    [ "${#1}" -eq 56 ] || return 1
-    case "$1" in U*) ;; *) return 1 ;; esac
-    case "$1" in *[!A-Z2-7]*) return 1 ;; esac
-}
-
-sqs_valid_ipv4() {
-    case "$1" in *[!0-9.]* | .* | *. | *..*) return 1 ;; esac
-    old_ifs=$IFS; IFS=.
-    # shellcheck disable=SC2086 # split on dots on purpose
-    set -- $1
-    IFS=$old_ifs
-    [ "$#" -eq 4 ] || return 1
-    for o in "$@"; do
-        [ "${#o}" -le 3 ] && [ "$o" -le 255 ] || return 1
-        case "$o" in 0?*) return 1 ;; esac
-    done
-}
-
-# sqs_find_disk <serial>: print the /dev node of the one block device whose
-# sysfs serial matches exactly. Zero or several matches refuse.
-sqs_find_disk() {
-    found="" n=0
-    for b in "$SQS_SYSBLOCK"/*; do
-        [ -r "$b/serial" ] || continue
-        [ "$(cat "$b/serial")" = "$1" ] || continue
-        found=${b##*/}
-        n=$((n + 1))
-    done
-    [ "$n" -eq 1 ] || { sqs_fatal "expected one disk with serial $1, found $n"; return 1; }
-    test "$SQS_NODE_TEST" "$SQS_DEVDIR/$found" || { sqs_fatal "disk $found has no block device node"; return 1; }
-    printf '%s' "$SQS_DEVDIR/$found"
-}
-
-# sqs_has_addr <ipv4>: the address is assigned to a non-loopback interface.
-sqs_has_addr() {
-    "$IP" -o -4 addr show 2>/dev/null \
-        | awk -v a="$1" '$2 != "lo" { split($4, p, "/"); if (p[1] == a) f = 1 } END { exit !f }'
-}
-
-# sqs_alive <pid>: running or stopped, not exited (a zombie counts as exited).
-sqs_alive() { [ -r "/proc/$1/status" ] && ! grep -q '^State:[[:space:]]*Z' "/proc/$1/status" 2>/dev/null; }
-
-# sqs_starttime <pid>: kernel start time (stat field 22). It survives exec, and
-# pairing it with the pid reduces PID-reuse risk; without a pidfd the check and
-# the signal stay separate steps, so it is not an absolute guarantee.
-sqs_starttime() { sed 's/.*) //' "/proc/$1/stat" 2>/dev/null | cut -d' ' -f20; }
-
-# sqs_same <pid> <starttime>: the recorded process is still alive.
-sqs_same() { [ -n "$2" ] && sqs_alive "$1" && [ "$(sqs_starttime "$1")" = "$2" ]; }
-
-# sqs_write_config <addr> <nkey> <file>: the dedicated server's config. Its
-# max_payload admits a 1 MiB SQS message with its headers (ADR-0011:S4); the
-# 1MB NATS default does not.
-sqs_write_config() {
-    cat > "$3" <<EOF
-listen: "$1:$SQS_PORT"
-server_name: "sqs-store"
-max_payload: 2MiB
-jetstream {
-  store_dir: "$SQS_DATA"
-  sync_interval: always
-}
-authorization {
-  users = [ { nkey: "$2" } ]
-}
-EOF
-}
-
-# sqs_prepare: validate the contract, check and mount the data disk, write the
-# server config and mark the store ready. Returns non-zero, unready, on any refusal.
-sqs_prepare() {
-    rm -f "$SQS_RUN/ready"
-    mkdir -p "$SQS_RUN"
-    serial=$(sqs_fwcfg data-serial) || return 1
-    sqs_valid_serial "$serial" || { sqs_fatal "data-serial is not a volume serial"; return 1; }
-    addr=$(sqs_fwcfg listen-addr) || return 1
-    sqs_valid_ipv4 "$addr" || { sqs_fatal "listen-addr is not an IPv4 address"; return 1; }
-    nkey=$(sqs_fwcfg client-nkey) || return 1
-    sqs_valid_user_nkey "$nkey" || { sqs_fatal "client-nkey is not a user public NKey"; return 1; }
-    sqs_has_addr "$addr" || { sqs_fatal "listen-addr $addr is not on any guest NIC"; return 1; }
-
-    # The server checks the config, including the NKey checksum, before the
-    # disk is touched.
-    umask 077
-    sqs_write_config "$addr" "$nkey" "$SQS_RUN/nats.conf" || return 1
-    if ! "$NATS_BIN" -t -c "$SQS_RUN/nats.conf" > "$SQS_RUN/config-test.out" 2>&1; then
-        sqs_fatal "server config test failed: $(sed 's/U[A-Z2-7]\{55\}/<nkey>/g' "$SQS_RUN/config-test.out" | tr '\n' ' ')"
-        return 1
-    fi
-
-    dev=$(sqs_find_disk "$serial") || return 1
-    rc=0
-    "$E2FSCK" -p "$dev" > "$SQS_RUN/fsck.out" 2>&1 || rc=$?
-    sqs_log "e2fsck -p $dev rc=$rc"
-    # 0 clean, 1 corrected: proceed. Anything higher, including a disk that is
-    # not ext4 at all, refuses; the guest never formats.
-    [ "$rc" -le 1 ] || { sqs_fatal "e2fsck refused $dev: $(tr '\n' ' ' < "$SQS_RUN/fsck.out")"; return 1; }
-    mkdir -p "$SQS_DATA"
-    "$MOUNT" -t ext4 -o nodev,nosuid,noexec "$dev" "$SQS_DATA" || { sqs_fatal "mount $dev failed"; return 1; }
-    : > "$SQS_RUN/ready"
-    sqs_log "prepared: $dev mounted at $SQS_DATA, config written for $addr:$SQS_PORT; server not started yet"
-}
-
-# sqs_run: the respawned supervisor body. Its identity is recorded before the
-# stopping check: a stop that misses it set stopping first, so this parks; a
-# stop that reads it signals this process whether or not it has exec'd yet.
-sqs_run() {
-    mkdir -p "$SQS_RUN"
-    echo "$$ $(sqs_starttime $$)" > "$SQS_RUN/pid"
-    if [ -e "$SQS_RUN/stopping" ]; then
-        sqs_log "run: shutdown in progress, not starting"
-        return 2
-    fi
-    if [ ! -e "$SQS_RUN/ready" ]; then
-        sqs_log "run: store not prepared, not starting"
-        return 2
-    fi
-    sqs_log "run: starting pid=$$"
-    # ADR-0011:S11 bounds the store's Go heap; the guest must be larger.
-    GOMEMLIMIT=512MiB exec "$NATS_BIN" -c "$SQS_RUN/nats.conf"
-}
-
-# sqs_stop: first shutdown action. Block any respawn, SIGTERM the recorded run
-# process (a server, or a start not yet exec'd), SIGKILL only after the budget,
-# then sync and unmount. Each signal rechecks the identity; a survivor keeps the disk mounted.
-sqs_stop() {
-    mkdir -p "$SQS_RUN"
-    : > "$SQS_RUN/stopping"
-    sqs_log "stop: begin"
-    rec=$(cat "$SQS_RUN/pid" 2>/dev/null || true)
-    pid=${rec%% *} st=${rec#* }
-    [ "$st" != "$rec" ] || st=""
-    if [ -n "$pid" ] && sqs_same "$pid" "$st"; then
-        kill -TERM "$pid" 2>/dev/null || true
-        sqs_log "stop: SIGTERM sent to run pid=$pid"
-        i=0
-        while sqs_same "$pid" "$st" && [ "$i" -lt "$SQS_STOP_BUDGET_TENTHS" ]; do sleep 0.1; i=$((i + 1)); done
-        if sqs_same "$pid" "$st"; then
-            sqs_log "stop: run pid=$pid still running after $((SQS_STOP_BUDGET_TENTHS / 10))s, SIGKILL"
-            kill -KILL "$pid" 2>/dev/null || true
-            j=0
-            while sqs_same "$pid" "$st" && [ "$j" -lt 30 ]; do sleep 0.1; j=$((j + 1)); done
-        fi
-        if sqs_same "$pid" "$st"; then
-            sqs_log "stop: run pid=$pid STILL ALIVE, leaving $SQS_DATA mounted"
-            return 1
-        fi
-        sqs_log "stop: store exited after $((i / 10)).$((i % 10))s"
-    else
-        sqs_log "stop: no running store"
-    fi
-    sync
-    if grep -q " $SQS_DATA " "$SQS_MOUNTS" 2>/dev/null; then
-        rc=0
-        "$UMOUNT" "$SQS_DATA" || rc=$?
-        sqs_log "stop: umount $SQS_DATA rc=$rc"
-    fi
-}
-
-# sqs_shutdown: the only shutdown action. The generic read-only remount runs
-# only after the store stopped; a surviving writer keeps every mount as it is.
-sqs_shutdown() {
-    if ! sqs_stop; then
-        sync
-        sqs_log "shutdown: store survived, skipping umount -a -r"
-        return 1
-    fi
-    rc=0
-    "$UMOUNT" -a -r || rc=$?
-    echo "[init] shutdown: umount -a -r rc=$rc" >> "$SQS_CONSOLE"
-}

build/microvm/sqs-store/sqs-store-prepare

@@ -1,9 +0,0 @@
-#!/bin/sh
-# sysinit: load the disk drivers, then validate the guest contract, check and
-# mount the data disk and write the server config. A refusal leaves the store
-# unprepared, so the run step never starts it.
-. /usr/local/lib/sqs-store/sqs-store-lib.sh
-modprobe virtio_blk 2>/dev/null || true
-modprobe ext4 2>/dev/null || true
-sqs_prepare || sqs_log "store refused; it will not start"
-exit 0

build/microvm/sqs-store/sqs-store-run

@@ -1,6 +0,0 @@
-#!/bin/sh
-# Respawned by init. A store that must not run parks here instead of exiting,
-# so init does not respawn it in a tight loop.
-. /usr/local/lib/sqs-store/sqs-store-lib.sh
-sqs_run
-exec sleep 2147483647

build/microvm/sqs-store/sqs-store-stop

@@ -1,5 +0,0 @@
-#!/bin/sh
-# The only shutdown action: stop the store and unmount its disk, then run the
-# generic read-only remount only if the store is gone.
-. /usr/local/lib/sqs-store/sqs-store-lib.sh
-sqs_shutdown

build/microvm/sqs-worker/inittab

@@ -3,6 +3,7 @@
 # the pre-exec shell PID 1. A press before this line is lost; the host re-sends.
 ::sysinit:/sbin/modprobe tiny_power_button power_signal=12
 ::sysinit:/usr/local/lib/sqs-worker/sqs-worker-prepare
+::respawn:/usr/local/lib/sqs-worker/sqs-worker-store
 ::respawn:/usr/local/lib/sqs-worker/sqs-worker-ready
 ::ctrlaltdel:/sbin/reboot
 ::shutdown:/usr/local/lib/sqs-worker/sqs-worker-stop

build/microvm/sqs-worker/sqs-worker-lib.sh

@@ -1,19 +1,23 @@
-# sqs-worker-lib.sh — shared by the SQS worker guest's readiness and stop
+# sqs-worker-lib.sh — shared by the SQS guest's store, readiness and stop
 # steps. Sourced, never executed. Every path and command is overridable so the
 # host test can drive the same code against test-owned processes and stub tools.
 #
-# store-ready exists only while the running watcher's last check passed. A
-# check proves the store accepted the worker key, not the store server's
-# identity, and a store failure is noticed up to one interval plus call time late.
+# The guest runs its own SQS nats-server on loopback and the worker beside it.
+# The server starts only after boot settled the receipt key; store-ready exists
+# only while the running watcher's last check of that server passed.
 
 SQSW_RUN=${SQSW_RUN:-/run/sqs-worker}
 SQSW_DATA=${SQSW_DATA:-/data}
 SQSW_CONSOLE=${SQSW_CONSOLE:-/dev/console}
 SQSW_MOUNTS=${SQSW_MOUNTS:-/proc/mounts}
 SQSW_BIN=${SQSW_BIN:-/usr/local/bin/sqs-worker}
+SQSW_LIB=${SQSW_LIB:-/usr/local/lib/sqs-worker/sqs-worker-lib.sh}
+SQSW_NATS_BIN=${SQSW_NATS_BIN:-/usr/local/bin/nats-server}
+SQSW_STORE_PORT=${SQSW_STORE_PORT:-4222}
 SQSW_STOP_BUDGET_TENTHS=${SQSW_STOP_BUDGET_TENTHS:-100}
 SQSW_RESPAWN_GAP=${SQSW_RESPAWN_GAP:-5}
-SQSW_PARK_INTERVAL=${SQSW_PARK_INTERVAL:-30}
+SQSW_PARK=${SQSW_PARK:-sleep 2147483647}
+SQSW_BOOT_RETRY=${SQSW_BOOT_RETRY:-30}
 UMOUNT=${UMOUNT:-umount}
 
 sqsw_log() { echo "[sqs-worker] $*" >> "$SQSW_CONSOLE"; }
@@ -28,28 +32,102 @@ sqsw_starttime() { sed 's/.*) //' "/proc/$1/stat" 2>/dev/null | cut -d' ' -f20;
 # sqsw_same <pid> <starttime>: the recorded process is still alive.
 sqsw_same() { [ -n "$2" ] && sqsw_alive "$1" && [ "$(sqsw_starttime "$1")" = "$2" ]; }
 
+# sqsw_write_config <file>: the guest's own server. It listens only on
+# loopback, with no monitor, route or cluster listener. Its max_payload admits
+# a 1 MiB SQS message with its headers (ADR-0011:S4); the 1MB default does not.
+sqsw_write_config() {
+    cat > "$1" <<CONF
+listen: "127.0.0.1:$SQSW_STORE_PORT"
+server_name: "sqs-store"
+max_payload: 2MiB
+jetstream {
+  store_dir: "$SQSW_DATA/jetstream"
+  sync_interval: always
+}
+CONF
+}
+
+# sqsw_prepare_store: write and test the server config once boot has settled
+# the receipt key. Returns non-zero, leaving the store unstarted, otherwise.
+sqsw_prepare_store() {
+    rm -f "$SQSW_RUN/store.conf"
+    [ -s "$SQSW_RUN/receipt-ready" ] || { sqsw_log "store: receipt key not settled, store not prepared"; return 1; }
+    umask 077
+    sqsw_write_config "$SQSW_RUN/store.conf.new" || return 1
+    if ! "$SQSW_NATS_BIN" -t -c "$SQSW_RUN/store.conf.new" > "$SQSW_RUN/config-test.out" 2>&1; then
+        sqsw_log "store: FATAL: server config test failed: $(tr '\n' ' ' < "$SQSW_RUN/config-test.out")"
+        return 1
+    fi
+    mv "$SQSW_RUN/store.conf.new" "$SQSW_RUN/store.conf"
+    sqsw_log "store: config written for 127.0.0.1:$SQSW_STORE_PORT; server not started yet"
+}
+
+# sqsw_park <why>: init respawns a step that exits, so a step that must not run
+# logs once and waits for shutdown instead of looping.
+sqsw_park() {
+    sqsw_log "$1"
+    exec $SQSW_PARK
+}
+
+# sqsw_record <name>: record this process's identity before the stopping
+# check, so a stop that misses the record has already set stopping.
+sqsw_record() { echo "$$ $(sqsw_starttime $$)" > "$SQSW_RUN/$1.pid"; }
+
+# sqsw_boot: settle the disk and receipt key once per guest boot. A transient
+# failure (exit 75) waits and exits so init runs it again; boot never remounts
+# a disk already mounted. Any other failure is final for this guest boot.
+sqsw_boot() {
+    [ ! -e "$SQSW_RUN/boot-refused" ] || sqsw_park "store: boot refused, not starting"
+    [ -s "$SQSW_RUN/receipt-ready" ] && return 0
+    # The boot child records itself before its stopping check, as the other
+    # steps do, so shutdown either sees its record or it never starts.
+    SQSW_RUN="$SQSW_RUN" SQSW_BIN="$SQSW_BIN" sh -c '. "$1"; sqsw_record boot; [ ! -e "$SQSW_RUN/stopping" ] || exit 75; exec "$SQSW_BIN" boot' \
+        sh "$SQSW_LIB" >> "$SQSW_CONSOLE" 2>&1 &
+    child=$!
+    rc=0
+    wait "$child" || rc=$?
+    rm -f "$SQSW_RUN/boot.pid"
+    [ ! -e "$SQSW_RUN/stopping" ] || sqsw_park "store: shutdown during boot, not starting"
+    case "$rc" in
+        0) return 0 ;;
+        75) sqsw_log "store: boot not settled, retrying in ${SQSW_BOOT_RETRY}s"
+            sleep "$SQSW_BOOT_RETRY"
+            exit 1 ;;
+        *) : > "$SQSW_RUN/boot-refused"
+           sqsw_park "store: boot refused (rc=$rc), not starting" ;;
+    esac
+}
+
+# sqsw_store: the respawned server body. It starts only on a prepared config.
+sqsw_store() {
+    mkdir -p "$SQSW_RUN"
+    sqsw_record store
+    [ ! -e "$SQSW_RUN/stopping" ] || sqsw_park "store: shutdown in progress, not starting"
+    sqsw_boot
+    if [ ! -s "$SQSW_RUN/store.conf" ] && ! sqsw_prepare_store; then
+        : > "$SQSW_RUN/boot-refused"
+        sqsw_park "store: not prepared, not starting"
+    fi
+    sqsw_log "store: starting pid=$$"
+    # ADR-0011:S11 bounds the store's Go heap; the guest must be larger.
+    GOMEMLIMIT=512MiB exec "$SQSW_NATS_BIN" -c "$SQSW_RUN/store.conf"
+}
+
 # sqsw_run: the respawned readiness body. A predecessor that was killed or
-# crashed could not remove its marker, so every start removes it first. Its
-# identity is recorded before the stopping check, as the store's run step does.
+# crashed could not remove its marker, so every start removes it first.
 sqsw_run() {
     mkdir -p "$SQSW_RUN"
-    echo "$$ $(sqsw_starttime $$)" > "$SQSW_RUN/ready.pid"
+    sqsw_record ready
     if [ -e "$SQSW_RUN/store-ready" ]; then
         rm -f "$SQSW_RUN/store-ready"
         sqsw_log "ready: removed a store-ready marker left by an earlier watcher"
     fi
-    # Returning here would make init respawn and log at once, forever. Park
-    # instead, logging once: shutdown stops this recorded process, and a key
-    # that appears later lets the watcher start.
-    if [ -e "$SQSW_RUN/stopping" ] || [ ! -s "$SQSW_RUN/public-key" ]; then
-        if [ -e "$SQSW_RUN/stopping" ]; then
-            sqsw_log "ready: shutdown in progress, not starting"
-        else
-            sqsw_log "ready: no worker key; readiness not started, waiting"
-        fi
-        while [ -e "$SQSW_RUN/stopping" ] || [ ! -s "$SQSW_RUN/public-key" ]; do
-            sleep "$SQSW_PARK_INTERVAL"
-        done
+    [ ! -e "$SQSW_RUN/stopping" ] || sqsw_park "ready: shutdown in progress, not starting"
+    [ ! -e "$SQSW_RUN/boot-refused" ] || sqsw_park "ready: boot refused, readiness not started"
+    # Boot may still be retrying; check again after a gap rather than parking.
+    if [ ! -s "$SQSW_RUN/store.conf" ]; then
+        sleep "$SQSW_RESPAWN_GAP"
+        exit 0
     fi
     # Init respawns at once; a watcher that keeps exiting restarts at most
     # once per gap instead of in a tight loop.
@@ -62,37 +140,47 @@ sqsw_run() {
     exec "$SQSW_BIN" ready >> "$SQSW_CONSOLE" 2>&1
 }
 
-# sqsw_stop: block any respawn, drop the marker, SIGTERM the recorded watcher,
-# SIGKILL only after the budget, and unmount the worker disk only once the
-# watcher is confirmed gone. A survivor keeps the disk mounted.
+# sqsw_end <name>: SIGTERM the recorded process, SIGKILL only after the
+# budget. Returns non-zero while it is still alive.
+sqsw_end() {
+    rec=$(cat "$SQSW_RUN/$1.pid" 2>/dev/null || true)
+    pid=${rec%% *} st=${rec#* }
+    [ "$st" != "$rec" ] || st=""
+    if [ -z "$pid" ] || ! sqsw_same "$pid" "$st"; then
+        sqsw_log "stop: no running $1"
+        return 0
+    fi
+    kill -TERM "$pid" 2>/dev/null || true
+    sqsw_log "stop: SIGTERM sent to $1 pid=$pid"
+    i=0
+    while sqsw_same "$pid" "$st" && [ "$i" -lt "$SQSW_STOP_BUDGET_TENTHS" ]; do sleep 0.1; i=$((i + 1)); done
+    if sqsw_same "$pid" "$st"; then
+        sqsw_log "stop: $1 pid=$pid still running after $((SQSW_STOP_BUDGET_TENTHS / 10))s, SIGKILL"
+        kill -KILL "$pid" 2>/dev/null || true
+        j=0
+        while sqsw_same "$pid" "$st" && [ "$j" -lt 30 ]; do sleep 0.1; j=$((j + 1)); done
+    fi
+    if sqsw_same "$pid" "$st"; then
+        sqsw_log "stop: $1 pid=$pid STILL ALIVE"
+        return 1
+    fi
+    sqsw_log "stop: $1 pid=$pid exited"
+    rm -f "$SQSW_RUN/$1.pid"
+}
+
+# sqsw_stop: block any respawn, drop the marker, stop the watcher, the server
+# and any boot in progress, and unmount the disk only once all are confirmed
+# gone. A survivor keeps the disk mounted.
 sqsw_stop() {
     mkdir -p "$SQSW_RUN"
     : > "$SQSW_RUN/stopping"
     rm -f "$SQSW_RUN/store-ready"
-    rec=$(cat "$SQSW_RUN/ready.pid" 2>/dev/null || true)
-    pid=${rec%% *} st=${rec#* }
-    [ "$st" != "$rec" ] || st=""
-    if [ -n "$pid" ] && sqsw_same "$pid" "$st"; then
-        kill -TERM "$pid" 2>/dev/null || true
-        sqsw_log "stop: SIGTERM sent to readiness watcher pid=$pid"
-        i=0
-        while sqsw_same "$pid" "$st" && [ "$i" -lt "$SQSW_STOP_BUDGET_TENTHS" ]; do sleep 0.1; i=$((i + 1)); done
-        if sqsw_same "$pid" "$st"; then
-            sqsw_log "stop: readiness watcher pid=$pid still running after $((SQSW_STOP_BUDGET_TENTHS / 10))s, SIGKILL"
-            kill -KILL "$pid" 2>/dev/null || true
-            j=0
-            while sqsw_same "$pid" "$st" && [ "$j" -lt 30 ]; do sleep 0.1; j=$((j + 1)); done
-        fi
-        if sqsw_same "$pid" "$st"; then
-            rm -f "$SQSW_RUN/store-ready"
-            sqsw_log "stop: readiness watcher pid=$pid STILL ALIVE, leaving $SQSW_DATA mounted"
-            return 1
-        fi
-        sqsw_log "stop: readiness watcher pid=$pid exited"
-    else
-        sqsw_log "stop: no running readiness watcher"
+    if ! sqsw_end ready || ! sqsw_end store || ! sqsw_end boot; then
+        rm -f "$SQSW_RUN/store-ready"
+        sqsw_log "stop: leaving $SQSW_DATA mounted"
+        return 1
     fi
-    rm -f "$SQSW_RUN/store-ready" "$SQSW_RUN/ready.pid"
+    rm -f "$SQSW_RUN/store-ready"
     sync
     if grep -q " $SQSW_DATA " "$SQSW_MOUNTS" 2>/dev/null; then
         rc=0
@@ -102,11 +190,11 @@ sqsw_stop() {
 }
 
 # sqsw_shutdown: the only shutdown action. The generic read-only remount runs
-# only after the watcher stopped; a survivor keeps every mount as it is.
+# only after both processes stopped; a survivor keeps every mount as it is.
 sqsw_shutdown() {
     if ! sqsw_stop; then
         sync
-        sqsw_log "shutdown: readiness watcher survived, skipping umount -a -r"
+        sqsw_log "shutdown: a guest process survived, skipping umount -a -r"
         return 1
     fi
     rc=0

build/microvm/sqs-worker/sqs-worker-prepare

@@ -1,8 +1,9 @@
 #!/bin/sh
-# sysinit: load the disk drivers, then select, check and mount the worker disk
-# and load or create the worker's store key on it. Any refusal is logged; the
-# key is never read or created on a disk that is not proven mounted.
+# sysinit: load the disk drivers and clear this boot's markers. The respawned
+# store step mounts the disk, settles the receipt key and starts the store.
+. /usr/local/lib/sqs-worker/sqs-worker-lib.sh
 modprobe virtio_blk 2>/dev/null || true
 modprobe ext4 2>/dev/null || true
-/usr/local/bin/sqs-worker boot >> /dev/console 2>&1 || echo "[sqs-worker] boot refused" >> /dev/console
+mkdir -p "$SQSW_RUN"
+rm -f "$SQSW_RUN/receipt-ready" "$SQSW_RUN/store.conf" "$SQSW_RUN/boot-refused"
 exit 0

build/microvm/sqs-worker/sqs-worker-ready

@@ -1,6 +1,5 @@
 #!/bin/sh
-# Respawned by init. A watcher that must not run parks here instead of exiting,
-# so init does not respawn it in a tight loop.
+# Respawned by init: the worker, reporting ready only while the local store
+# answers. A watcher that must not run parks instead of exiting.
 . /usr/local/lib/sqs-worker/sqs-worker-lib.sh
 sqsw_run
-exec sleep 2147483647

build/microvm/sqs-worker/sqs-worker-stop

@@ -1,5 +1,5 @@
 #!/bin/sh
-# The only shutdown action: stop the readiness watcher and unmount the worker
-# disk, then run the generic read-only remount only if the watcher is gone.
+# The only shutdown action: stop the worker, then the store, and unmount the
+# data disk, then run the generic read-only remount only if both are gone.
 . /usr/local/lib/sqs-worker/sqs-worker-lib.sh
 sqsw_shutdown

build/microvm/sqs-worker/sqs-worker-store

@@ -0,0 +1,5 @@
+#!/bin/sh
+# Respawned by init: the guest's own SQS nats-server, started only on a
+# prepared config. A store that must not run parks instead of exiting.
+. /usr/local/lib/sqs-worker/sqs-worker-lib.sh
+sqsw_store

cmd/aws-model-coverage/main.go

@@ -17,6 +17,7 @@ import (
 	iamawsapi "github.com/mulgadc/spinifex/spinifex/domains/iam/awsapi"
 	rdsawsapi "github.com/mulgadc/spinifex/spinifex/domains/rds/awsapi"
 	route53awsapi "github.com/mulgadc/spinifex/spinifex/domains/route53/awsapi"
+	sqsawsapi "github.com/mulgadc/spinifex/spinifex/domains/sqs/awsapi"
 	stsawsapi "github.com/mulgadc/spinifex/spinifex/domains/sts/awsapi"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
 
@@ -66,6 +67,7 @@ func compareAll() ([]awsmodel.OperationCoverage, error) {
 		ecsawsapi.ServiceName:     ecsawsapi.OperationInventory(),
 		eksawsapi.ServiceName:     eksawsapi.OperationInventory(),
 		iamawsapi.ServiceName:     iamawsapi.OperationInventory(),
+		sqsawsapi.ServiceName:     sqsawsapi.OperationInventory(),
 		stsawsapi.ServiceName:     stsawsapi.OperationInventory(),
 	})
 	// The loop below walks the embedded models, so a served service with no

cmd/eks-gateway-fetch/main.go

@@ -6,7 +6,6 @@
 //
 //	eks-gateway-fetch -resource addons   # GET /clusters/{cluster}/internal-addons/{accountId}
 //	eks-gateway-fetch -resource recovery -instance-id i-… # GET /clusters/{cluster}/internal-recovery/{accountId}/{instanceId}
-//	eks-gateway-fetch -resource workers  # GET /clusters/{cluster}/internal-workers/{accountId}
 //
 // Flags default to environment variables seeded by cloud-init:
 // EKS_GATEWAY_URL, EKS_GATEWAY_CA, EKS_ACCESS_KEY, EKS_SECRET_KEY, EKS_REGION,
@@ -34,7 +33,7 @@ func main() {
 	flag.StringVar(&cfg.AccountID, "account-id", os.Getenv("EKS_ACCOUNT_ID"), "Cluster account ID")
 	flag.StringVar(&cfg.ClusterName, "cluster", os.Getenv("EKS_CLUSTER_NAME"), "Cluster name")
 	flag.StringVar(&cfg.InstanceID, "instance-id", os.Getenv("EKS_INSTANCE_ID"), "This member's EC2 instance ID (recovery resource)")
-	flag.StringVar(&cfg.Resource, "resource", "addons", "Resource to fetch: addons | recovery | workers")
+	flag.StringVar(&cfg.Resource, "resource", "addons", "Resource to fetch: addons | recovery")
 	flag.Parse()
 
 	if err := gatewayfetch.Run(context.Background(), cfg); err != nil {

cmd/sqs-worker/main.go

@@ -1,17 +1,22 @@
-// sqs-worker runs inside the shared SQS worker microVM.
+// sqs-worker runs inside the SQS microVM, next to its own SQS NATS server.
 //
 // Usage:
 //
 //	sqs-worker boot
 //	sqs-worker ready
 //
-// boot selects and mounts the worker's data disk named by fw_cfg, asks the
-// gateway for its current first-store decision with its instance-role
-// credentials, then loads or creates the store key on that disk and
-// publishes only the public key. ready then keeps checking the bound store's
-// JetStream with that key until it is stopped, keeping a readiness marker only
-// while the check passes; it proves the store accepted the worker key, not
-// the store server's identity.
+// boot selects and mounts the guest's data disk named by fw_cfg, asks the
+// gateway for its receipt key decision with its instance-role credentials,
+// then loads the recorded key, or creates one on a new volume and records its
+// fingerprint, before the guest's SQS store first starts on that disk. ready
+// then keeps checking the guest's own loopback JetStream, keeping a readiness
+// marker only while it answers, and serves SQS owner calls against it, pulling
+// them from the gateway with the same credentials.
+//
+// Serving is refused unless the gateway reports a single-node host cluster.
+// A disk whose receipt key is missing or does not match the recorded one
+// refuses to serve; recovering it needs a procedure not yet designed, and
+// nothing deletes or replaces the volume.
 package main
 
 import (
@@ -23,10 +28,14 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/nats-io/nats.go/jetstream"
+
 	"github.com/mulgadc/spinifex/internal/gwsign"
 	"github.com/mulgadc/spinifex/internal/sqsworkergw"
 	"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerboot"
 	"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerkey"
+	"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerserve"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
 )
 
 // The cluster CA the daemon supplies; the only root the client trusts.
@@ -36,7 +45,11 @@ const caPath = "/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/ca-cert/raw"
 // no static-key fallback.
 func newClaimer() workerboot.NewClaimer {
 	return func(region, gatewayURL string) (workerkey.Claimer, error) {
-		return newClient(region, gatewayURL)
+		c, err := newClient(region, gatewayURL)
+		if err != nil {
+			return nil, err
+		}
+		return workerboot.GatewayClaimer{Client: c}, nil
 	}
 }
 
@@ -48,13 +61,6 @@ func newClient(region, gatewayURL string) (*sqsworkergw.Client, error) {
 	return sqsworkergw.New(gatewayURL, region, ca, gwsign.NewInstanceRole())
 }
 
-type describer struct{ c *sqsworkergw.Client }
-
-func (d describer) DescribeStore(ctx context.Context) (workerboot.Store, error) {
-	st, err := d.c.DescribeStore(ctx)
-	return workerboot.Store(st), err
-}
-
 func main() {
 	if len(os.Args) != 2 || (os.Args[1] != "boot" && os.Args[1] != "ready") {
 		fmt.Fprintln(os.Stderr, "usage: sqs-worker boot|ready")
@@ -66,11 +72,18 @@ func main() {
 		run = ready
 	}
 	if err := run(logf); err != nil {
+		if os.Args[1] == "boot" && workerboot.Transient(err) {
+			logf("boot not settled, will retry: " + err.Error())
+			os.Exit(exitTransient)
+		}
 		logf("FATAL: " + err.Error())
 		os.Exit(1)
 	}
 }
 
+// exitTransient tells the guest's supervisor that boot may be run again.
+const exitTransient = 75
+
 func config(logf func(string)) workerboot.Config {
 	return workerboot.Config{
 		FwCfgDir:      "/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/sqs-worker",
@@ -99,19 +112,26 @@ func boot(logf func(string)) error {
 func ready(logf func(string)) error {
 	ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
 	defer cancel()
+	// The store is one guest, so its streams and buckets are R1.
+	if err := clustersize.Declare(1); err != nil {
+		return err
+	}
 	err := workerboot.Watch(ctx, workerboot.ReadyConfig{
-		Config: config(logf),
-		NewDescriber: func(region, gatewayURL string) (workerboot.StoreDescriber, error) {
-			c, err := newClient(region, gatewayURL)
-			if err != nil {
-				return nil, err
-			}
-			return describer{c}, nil
-		},
+		Config:      config(logf),
 		Checker:     workerboot.NATSChecker{},
 		CallTimeout: 30 * time.Second,
 		Refresh:     30 * time.Second,
 		Refused:     5 * time.Minute,
+		Serve: &workerboot.ServeConfig{
+			Connector: workerboot.NATSConnector{},
+			Run: func(ctx context.Context, region, gatewayURL string, js jetstream.JetStream, receiptKey []byte) error {
+				c, err := newClient(region, gatewayURL)
+				if err != nil {
+					return err
+				}
+				return workerserve.Run(ctx, c, js, receiptKey, logf)
+			},
+		},
 	})
 	if errors.Is(err, context.Canceled) {
 		return nil

docs/COMMANDS.md

@@ -194,18 +194,32 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 
 ### SQS System Instances
 
-These commands start the first SQS system instances on one node. They do not serve the SQS API, and a bound VM is not a ready service.
+These commands enable SQS on a single-node cluster and start its one system VM.
+A bound VM is not a ready service: `spx admin sqs status` reports readiness.
 
-Both send a signed `spinifex` admin action (`EnsureSQSWorker`, `EnsureSQSStore`) to the AWS gateway using the standard AWS credential chain, for example `AWS_PROFILE=spinifex`. The caller must be in the super-admin account (`000000000001`) and hold `spinifex:<Action>`. The VMs, volumes, private VPC and ENIs belong to the system account (`000000000000`). `--endpoint`, `--region` and `--ca-bundle` default to this node's gateway, region and `config/ca.pem`; off a node, `--endpoint` and `--region` are required. Each call waits up to 11 minutes.
+The SQS VM runs two processes on one encrypted data volume: the Region's dedicated SQS NATS server, which listens only on the guest's loopback address and is separate from platform NATS, and the worker, which serves SQS owner calls from that local store. The worker starts serving only after the local store answers and the receipt-signing key on the volume matches the fingerprint the platform recorded.
+
+SQS supports only a fresh single-node install. A worker record without a supported launch contract, or any record of the separate SQS store VM from an earlier dev build, cannot be upgraded: `enable` refuses it, and any existing volume is kept, not inspected, emptied or replaced. An existing separate store VM is refused on relaunch with its volume kept, and an existing worker VM is not restarted into a supported mode; upgrading such an install is unsupported.
+
+**Do not put tenant production data in SQS.** SQS-managed per-message encryption is not implemented: `GetQueueAttributes` reports `SqsManagedSseEnabled=false`, and a request to enable SSE-SQS or SSE-KMS is rejected.
+The SQS data volume must be encrypted, which is a separate layer under every queue and not SSE.
+There is no `DeleteQueue` or `PurgeQueue`, so a queue and its unexpired messages cannot be removed once created.
+Retention is checked when a message is received, against the queue's retention at that moment, and nothing deletes an expired message: the store drops a body only at its age backstop of 14 days and 1 hour.
+So a message past a short retention can be delivered again if `MessageRetentionPeriod` is later raised while its body is still stored.
+Account termination does not yet remove SQS queues or messages, and there is no disable: the VM's volume, receipt key and data are kept.
+
+Each sends a signed `spinifex` admin action (`EnableSQS`, `DescribeSQS`, `EnsureSQSWorker`) to the AWS gateway using the standard AWS credential chain, for example `AWS_PROFILE=spinifex`. The caller must be in the super-admin account (`000000000001`) and hold `spinifex:<Action>`. The VM, volume, private VPC and ENI belong to the system account (`000000000000`). `--endpoint`, `--region` and `--ca-bundle` default to this node's gateway, region and `config/ca.pem`; off a node, `--endpoint` and `--region` are required. `ensure-worker` and `enable` wait up to 11 minutes, and `status` up to 1 minute.
 
 | Command | Flags | Description |
 |---------|-------|-------------|
-| `spx admin sqs ensure-worker` | `--node` (required), `--instance-type` (required), `--endpoint`, `--region`, `--ca-bundle` | Launches the Region's one SQS worker VM on `--node`, or reports the VM already bound there: node, region, instance, instance type and generation. The worker is pinned to its first node; another node is refused with `InvalidParameterValue`, and there is no takeover. It gets a private system VPC with one /24 subnet and no internet gateway, NAT or Elastic IP, and an encrypted 1 GiB system-account data volume on which the guest generates its own store key. The platform records only the public key. A different `--instance-type` is accepted only while no volume, network interface or launch step has been attempted; once any has, it is `IncorrectState`, whether or not the step completed. |
-| `spx admin sqs ensure-store` | `--node` (required, the worker's node), `--instance-type` (required, more than 512 MiB memory), `--volume-size-gib` (required, 1–16384), `--endpoint`, `--region`, `--ca-bundle` | Launches the Region's SQS store VM on the worker's node with an encrypted system-account data volume, or reports the VM already bound there: node, region, instance, instance type and address. Needs the worker bound with its public key recorded, else `IncorrectState`. The store accepts only that key. A different size, type or key than the first call is `IncorrectState`. |
+| `spx admin sqs enable` | `--node` (required), `--instance-type` (required, more than 512 MiB memory), `--volume-size-gib` (required, 1–16384), `--endpoint`, `--region`, `--ca-bundle` | Records SQS as enabled for the Region, pinned to `--node`, then launches or reports the SQS VM there. The desired state is recorded first, so rerunning the command resumes an interrupted enable. More than one node in the cluster is refused before anything is recorded. A worker record with a missing or unsupported launch contract, or a record of the separate store VM, is also refused before anything is recorded or launched; any existing volume is kept, and upgrading it is not supported. |
+| `spx admin sqs status` | `--endpoint`, `--region`, `--ca-bundle` | Shows the requested state, node, generation, VM state, receipt key state, whether a worker is polling this gateway, whether this gateway serves the SQS API, and `Ready`, which is true only when all of them hold. `Unsupported` names a record whose missing or unsupported launch contract blocks enable. |
+| `spx admin sqs ensure-worker` | `--node` (required), `--instance-type` (required, more than 512 MiB memory), `--volume-size-gib` (required, 1–16384), `--endpoint`, `--region`, `--ca-bundle` | Launches the Region's one SQS VM on `--node`, or reports the VM already bound there: node, region, instance, instance type and generation. The VM is pinned to its first node; another node is refused with `InvalidParameterValue`, and there is no takeover. It gets a private system VPC with one /24 subnet and no internet gateway, NAT or Elastic IP, and an encrypted system-account data volume of `--volume-size-gib` that holds the queue data and the receipt-signing key the guest creates on first boot. The platform records only the key's fingerprint. A different `--instance-type` is accepted only while no volume, network interface or launch step has been attempted; once any has, it is `IncorrectState`, whether or not the step completed. A different `--volume-size-gib` than the recorded one is `IncorrectState`. |
 
 `ServiceUnavailable` never proves that no SQS resources exist, because an earlier attempt may already have created some. Its message says what it does establish. When no node serves the subject, this request was not delivered. When a volume create was not sent, that particular create did not happen. When the node cannot boot the guest yet or its launcher lacks a capability, this request stopped before allocating or recording anything. When the gateway times out waiting for the node's reply, nothing is known: the launch may still be running, so never assume it created nothing. In every case, call the command again with the same arguments. A retry first checks the durable record of the attempt and continues only when that is safe; it can proceed to the first launch after a create that was proven unsent, and otherwise reports the recorded outcome as bound or `IncorrectState`, or returns `ServiceUnavailable` again. `IncorrectState` from a volume, network interface or launch step means that step was recorded as sent but its outcome was never observed. It is never resent, the message names the volume when its ID is known, and no tool reconciles it yet: recovery is manual.
 
-`bound` says only that the VM was launched for the current record. The worker decides store readiness itself, by checking that the store is bound to its own public key and that a NATS connection with that key succeeds; nothing reports that readiness to the operator. The worker-to-store connection does not authenticate the store server. Real VPC routing, IMDS and gateway TLS from the guest, and worker-to-store traffic on a real interface have not been tested.
+`bound` says only that the VM was launched for the current record. The worker is ready only when the guest's local store answers, and it serves only while its receipt key matches the recorded fingerprint; a missing or different key once one is recorded refuses serving and is never replaced.
+Real VPC routing, IMDS and gateway TLS from the guest have not been tested on a live host.
 
 ## Private Admin API
 

docs/DESIGN.md

@@ -278,8 +278,8 @@ storage-provider contract.
 ### SQS message store (internal)
 
 `domains/sqs/message` defines the store-neutral interface the SQS domain reaches its message store through (ADR-0011:S6), with a JetStream adapter in `domains/sqs/message/jetstream` and a shared conformance suite in `domains/sqs/message/conformance`.
-It is an internal primitive: nothing outside the SQS domain uses it, no daemon or service is wired to it, and it does not itself deliver an SQS API.
-The dedicated JetStream server ADR-0011 describes is not deployed by this code, and Spinifex ships no host service, unit, user or firewall rule for it; the guest that could run it is described under *SQS store system instance (not wired)*.
+It is used only by the SQS worker inside the SQS system instance and does not itself deliver an SQS API.
+Spinifex ships no host service, unit, user or firewall rule for the dedicated JetStream server; the system instance that runs it is described under *SQS system instance (operator-launched)*.
 
 The interface covers durable and idempotent append, body and metadata reads, conditional state writes and deletes, per-namespace counts, namespace enumeration, an ordered change feed, a bounded paginated scan, namespace deletion and a sweep of tombstoned bodies.
 A namespace carries an immutable message-shard and state-shard allocation chosen when the queue is created, so a later change in shard counts never moves an existing queue.
@@ -298,124 +298,36 @@ If that consumer delivers nothing for about two seconds while the stream holds a
 
 Coverage is unit and conformance tests on in-process R1 and R3 servers; there is no end-to-end or multi-host validation of this code, and earlier multi-host evidence predates the final feed.
 
-### SQS store system instance (not wired)
+### SQS system instance (operator-launched)
 
-Spinifex can build, install, launch, restart and give storage to a dedicated SQS store microVM, but nothing calls any of it.
-No SQS API, domain service, admin command or daemon start-up path creates a store, so none of this is in production use.
+One system-owned microVM per Region runs both the SQS store and the SQS worker as separate processes, on one encrypted data volume pinned to one node (ADR-0011 Amendment 5, ADR-0012 Amendment 6).
+Only an operator activates it with the super-admin `EnsureSQSWorker` or `EnableSQS` actions; nothing starts it at daemon startup, and platform NATS is neither used nor changed for SQS data.
+This first profile is fresh single-node only: a host cluster of more than one node is refused before anything is recorded or allocated, and there is no disable.
 
-The guest image is built by `make build-sqs-store-microvm-image`, separate from the load balancer image, and bundles the `nats-server` version pinned in `go.mod` plus the ext4 tools.
-Release tarballs carry it as `microvm-sqs-store/`, and `setup.sh` installs it to `/usr/share/spinifex/microvm/sqs-store/`.
-A release without a complete image removes any previously installed one, so a launch is then refused rather than booting a stale image.
+`make build-sqs-worker-microvm-image` builds the one `sqs-worker` image: the static `sqs-worker` binary, the pinned `nats-server`, the ext4 tools and a network `/init`; releases ship it as `microvm-sqs-worker`, which `setup.sh` installs to `/usr/share/spinifex/microvm/sqs-worker`.
+At boot the guest reads three firmware config values under `opt/spinifex/sqs-worker/` (the data disk's serial, its Region and the gateway URL on this node's mgmt bridge) and refuses before touching any disk if one is missing or malformed.
+It selects exactly one disk with that serial, runs `e2fsck -p`, mounts it at `/data` and confirms from `/proc/self/mountinfo` that the whole disk is what is mounted; the guest never formats.
 
-At boot the guest reads three firmware config values: the data disk's serial, an IPv4 address on one of its interfaces, and one NATS user NKey public key.
-It mounts exactly one matching disk at `/data` after `e2fsck -p`, refuses a missing, duplicate, unformatted or unrepairable disk, and never formats.
-JetStream keeps its data on `/data` with `sync_interval: always` (ADR-0011:S3), `max_payload: 2MiB` admits a full 1 MiB SQS message with its headers (ADR-0011:S4), and the server listens only on that address and accepts only that key.
-On shutdown the store is stopped before `/data` is unmounted; a store that survives SIGKILL leaves every mount in place.
+The store is a dedicated `nats-server` listening only on `127.0.0.1`, with its JetStream directory at `/data/jetstream` and `GOMEMLIMIT=512MiB` (ADR-0011:S11), so nothing outside the VM can reach it and it needs no network identity of its own.
+The store starts only after the receipt key is settled, and the worker connects only after the store answers JetStream account info, so an unavailable store never appears ready.
 
-A launch uses the versioned subject `system.LaunchSQSStore.v1`, which a daemon without this code does not serve, so it fails with no responder instead of booting another guest.
-The legacy `system.LaunchInstance` path refuses the store kind on both send and receive.
-Every input check runs before capacity is reserved, and the data volume must be an available system-account volume.
-It is mounted through the normal EBS path and detached, never deleted, on terminate, which is the protection standard EBS gives today and nothing stronger.
-CPU mitigations stay enabled for the store, unlike the load balancer.
-A successful launch means the VM is running, not that NATS is ready.
+The receipt-signing key lives on the data disk under `/data/sqs-worker-key`, written to a temporary file, synced, renamed and its directory synced before its fingerprint is recorded.
+`domains/sqs/workerident` holds the platform's receipt-key state on the worker's identity record, beside its generation, claim, pinned node and volume: `pending` is set only when the record first creates its volume, and `recorded` holds the fingerprint.
+Under `pending` the guest may create a key, or reuse a valid key it already wrote before a crash, only while no `/data/jetstream` directory exists; any store data under `pending` refuses, because it proves the volume is not new.
+Under `recorded` a missing, unreadable or different key refuses the boot and is never regenerated; a record with no receipt-key state is refused as not proven new.
+`RecordReceiptKey` writes the fingerprint by compare-and-swap that re-checks the current instance, generation, claim and volume, and a lost reply is answered by reading the record again, never by a new key.
+A replacement worker reuses the pinned volume and keeps its predecessor's receipt-key state.
 
-The instance record keeps the store's non-secret guest settings so a restart on the node that launched it rewrites the firmware config files.
-A relaunch on any other node is refused, and cross-node recovery never claims a store.
-Both checks exist only in daemons with this code: an older daemon can still claim and relaunch a store elsewhere, and an explicit start served by another node claims the instance before its relaunch is refused.
+The worker reaches the gateway over TLS 1.3 trusting only the cluster CA, signing with instance-role credentials from the metadata service only.
+The internal `spinifex-sqs` actions are `ClaimReceiptKey`, `RecordReceiptKey`, `PollWork` and `CompleteWork`; the gateway admits only an assumed-role session of `sqsWorkerRole` in the system account and resolves the instance through `workerident.Authorize`, a point-in-time check against the current bound record.
+The worker pulls tenant work from its own node's gateway relay and posts results back, so no SQS payload crosses platform NATS; the relay holds a bounded, in-memory set of waiting requests and answers `ServiceUnavailable` when it is full or no worker is polling.
 
-`domains/sqs/storevolume` prepares a fresh data volume, because the guest never formats.
-It checks for `mkfs.ext4` and `qemu-img` before allocating, creates the volume in `creating`, writes an empty ext4 image to it over NBD on the local node, unpublishes it, and only then makes it `available`.
-While a volume is `creating`, attach, delete, modify, force-detach and snapshot are refused.
-A volume is deleted only after a confirmed unpublish; any publish error, an unconfirmed unpublish or a failed release keeps it and reports its ID, with its state reported as unknown after a failed release.
+`EnsureSQSWorker` takes a `NodeId`, an `InstanceType` and a `VolumeSizeGiB`; the instance type's memory must exceed the store's 512 MiB limit, and a later request for a different type or size is refused as `IncorrectState`.
+`EnableSQS` records the Region's desired state, pinned to one node, and then ensures the worker; `status` reports it ready only when the worker is bound, polling and the tenant API is registered.
+A store launch record or VM from the earlier separate-store design, or a worker record without a supported launch contract, is refused with its volume kept: enable stops before recording anything, and a relaunch of an old store VM is marked failed rather than started.
 
-Evidence is local only: unit tests, real QEMU boots against disposable scratch disks, and one run of the preparer against the `qemunbdd` provider whose volume kept acknowledged writes across a reboot.
-Viperblock, volume encryption, the EC2 volume record path, multiple nodes and host reachability of the guest address are not tested.
-Credential issuance and rotation, store placement and NATS readiness checks do not exist, and there is no failover: a lost node leaves its store unavailable.
-
-### SQS worker guest (operator-launched)
-
-The shared SQS worker guest holds the SQS store's client credential, so that no platform component does (ADR-0011:S2).
-Only an operator activates it, with the super-admin `EnsureSQSWorker` action described below; nothing starts it at daemon startup, and no worker service runs after boot.
-
-`make build-sqs-worker-microvm-image` builds it with the store image's script and network `/init` under `SQS_MICROVM_PROFILE=worker`; it carries the static `sqs-worker` binary and the ext4 tools, no `nats-server`, and an empty `/data` mountpoint.
-Releases build it beside the store image and ship it as `microvm-sqs-worker`, which `setup.sh` installs to `/usr/share/spinifex/microvm/sqs-worker` with the store image's rules: a release without it removes an earlier one, and an incomplete one is refused.
-At boot `sqs-worker boot` reads three firmware config values under `opt/spinifex/sqs-worker/`: the worker disk's serial, its Region and the gateway URL; none is a first-store decision, because a static value survives a guest reset.
-A missing, empty or malformed value refuses before any disk is touched.
-
-It selects exactly one disk with that serial, runs `e2fsck -p`, mounts it at `/data` and then confirms from `/proc/self/mountinfo` that this disk, as a whole ext4 filesystem, is what is mounted there; an empty mountpoint, a foreign mount or a failed check or mount refuses, and the guest never formats.
-Only then does it ask the gateway for the current decision with `ClaimFirstStore`, signed by `internal/sqsworkergw` with credentials from the instance metadata service only (`gwsign.NewInstanceRole`, no environment, profile or static-key fallback), over TLS 1.3 that trusts only the cluster CA from `opt/spinifex/ca-cert` and never follows a redirect.
-Under that decision `workerkey.Acquire` creates the key on `/data/sqs-worker-key`, owner-only and synced, records only its public key and then uses it, or loads the recorded key; a missing or different key once one is recorded refuses and is never replaced.
-The VM can start before its launch binds the instance, so its first calls are legitimately `AccessDenied`; that denial creates nothing and is asked again, as are a lost reply and a `ServiceUnavailable` or `Throttling` refusal, all within one bound (60 attempts 5 seconds apart), after which a worker that was never bound, or was replaced, refuses the boot.
-A signature or other refusal, or an answer that is neither "create" nor one valid public key, refuses the boot at once.
-
-`domains/sqs/firststore` is the SQS domain's first-store decision: one record per Region in the `spinifex-sqs-first-store` KV bucket moves from `claimed` to `key-recorded` to `launch-intent` under compare-and-swap, bound to the worker's instance ID, claim ID and generation.
-A claim authorizes key creation only while no public key is recorded; once one is, every claim returns only that key, and a store launch intent needs it recorded first.
-An older generation, a different identity at the same generation or an unreadable record authorizes nothing, and the record holds no private key material.
-Reads are stream message gets answered by the stream leader, never direct gets a lagging replica can answer, and every authorizing answer is also confirmed by a compare-and-swap at the revision it read, so a stale or unavailable read authorizes nothing.
-The authenticated caller that will carry this decision must derive the worker's instance ID and generation from authoritative instance state; a generation supplied in a request cannot itself authorize a takeover.
-
-`domains/sqs/workerident` is that authoritative instance state: one record per Region in the `spinifex-sqs-worker-identity` KV bucket holds a monotonic generation, claim ID, pinned node and encrypted data volume, and moves from `launching` to `bound` to `stopped`, or from `launching` to `failed`, under compare-and-swap.
-A launch intent is refused while the previous attempt is `launching` or `bound`, including one whose outcome is unknown, so only one caller ever holds an intent to start a VM on the pinned volume.
-A replacement takes the next generation only after `FailLaunch` or `RecordStopped` records that nothing holds the volume, and each request names the generation it supersedes, so a replayed request cannot take a generation it never observed; there is no automatic takeover.
-The launch path then raises the first-store fence to the successor's generation with `firststore.Fence` before `BeginLaunch`, which refuses a replacement until a leader read shows that fence; the fence only rises, and `firststore` refuses every call from a generation below it.
-Only the current claim ID is recorded, so claim IDs are minted fresh per attempt and a VM launch must be idempotent on generation and claim together.
-Binding writes the instance's index entry first and then binds the record to that instance.
-The index is only a lookup: `Authorize` accepts an instance only while the Region's record is `bound` to exactly that instance, generation and claim, confirmed by a compare-and-swap at the leader-read revision, so an old, orphaned, rewritten or unbound entry authorizes nothing.
-A same-instance reboot keeps its instance ID and identity; a replacement takes a new generation, and the first cut refuses another node or data volume and an unencrypted volume, although the encryption flag is the caller's claim and the launch wiring must verify it from the volume's metadata.
-Both buckets are opened through `OpenDurableKV`, which refuses an existing bucket whose effective stream configuration could expire, evict, lose, lag or refuse records: non-limits retention, an age, per-message or delete-marker TTL, memory storage, asynchronous flush, a size, count or message-size cap, a discard policy other than discard-new, unacknowledged writes, a sealed stream, a mirror or sources, or fewer replicas than the cluster needs.
-`workerkey.Acquire` drives that decision: it syncs a new key to disk, records its public key and only then returns it, so a crash before recording reuses the key already on disk.
-
-`domains/sqs/awsapi/worker` carries the decision to the worker as `ClaimFirstStore` and `RecordWorkerPublicKey`, served under the internal `spinifex-sqs` signing service, which no AWS SDK signs for; the public AmazonSQS action inventory is untouched.
-The gateway admits only an assumed-role session of `sqsWorkerRole` in the system account, then the role's policy, and only then resolves the session name, the instance ID STS set, through `workerident.Authorize`; the request carries no identity, and any parameter beyond the action's own is refused.
-The first-store identity is the current record's instance, claim and generation, and a worker bound to another Region is refused.
-A request that passed `Authorize` just before a confirmed stop and replacement still carries the old generation; the fence was raised before the successor began, so `firststore` refuses it whether or not the successor has claimed yet.
-The adapter never logs a caller-chosen parameter name, because one could carry a secret.
-`RecordWorkerPublicKey` accepts only a valid public user key, so a seed is refused without being recorded, logged or echoed.
-The gateway binds both buckets per request and never creates them, so it provisions nothing and answers `ServiceUnavailable` until a worker launch has created them; the worker launch below ensures the role.
-The private key stays on the worker disk; the guest logs and publishes only the public key to `/run/sqs-worker/public-key`.
-
-`domains/sqs/workerlaunch` realizes the worker on the node it runs on, its only node: a bound record is returned as is; otherwise it takes the intent, then on the first launch marks the intent's one volume create, prepares the volume and records its ID, reads the volume's own record and refuses it unless encrypted, ensures the role, finds or creates the network for the claim and reads the recorded store key, and marks the intent dispatched before launching.
-A dispatched launch is never sent again: a lost reply or error leaves the intent `launching` and dispatched, and every later call returns `ErrOutcomeUnknown` until the outcome is reconciled, since the launch path is not idempotent.
-A failure before dispatch leaves an undispatched intent that the next call resumes with the same claim; a confirmed stop or failure is replaced only after `firststore.Fence` excludes the old generation, reusing the pinned volume.
-The launch passes no store decision or key: the guest asks the gateway at boot.
-The intent exists before any volume, so concurrent first launches allocate nothing for the losers. A Prepare proven to have sent no create withdraws the mark and returns `ServiceUnavailable`, or `UnsupportedOperation` when the cluster storage key is not configured, and a retry may prepare the volume; a create that may have applied is never sent again, and every later call returns `IncorrectState` until it is reconciled. A volume the call knows of but cannot use is reported with its ID in a `VolumeError` and left in place, never deleted. An intent that has sent no create may be re-typed; one that has keeps its type until replaced. The worker role is created with EC2-only trust, and an existing role is checked for that trust before any policy is granted; the instance profile must carry exactly the worker role, and one holding any other role is refused unchanged.
-The daemon's `LaunchSQSWorkerInstance` boots the `sqs-worker` image through the same direct-boot path as the store, with the data volume attached at boot and kept on terminate, a required instance profile, and a required mgmt NIC; it re-reads the volume's own record and refuses one that is not available and encrypted, before anything is allocated.
-Its firmware config adds the gateway URL on this node's mgmt bridge and the cluster CA from the daemon's own trust store; the caller cannot supply either, and no value written is secret.
-A relaunch rebuilds that config from the VM record on the node that launched it and is refused anywhere else, and membership takeover already skips SQS-managed instances.
-`sqsWorkerRole` is ensured in the system account granting exactly `spinifex-sqs:ClaimFirstStore` and `spinifex-sqs:RecordWorkerPublicKey`, from the `domains/sqs/workerrole` contract the gateway gate also reads; an existing role whose trust admits more than the EC2 service is refused.
-
-Evidence is local only: unit tests with a fake firmware config, sysfs and mount table, tests running the boot through the real client against the real `spinifex-sqs` registration and records, and QEMU boots of the built image against disposable scratch disks and a host-side test gateway and metadata service, showing the first boot creates the key, reboots reuse it, a key lost after recording is never replaced, and the seed reaches no log or request.
-The worker's network is a private SQS system VPC per Region, built by `systemvpc.EnsurePrivate` from the `[sqs]` supernet (default `10.240.0.0/14`): one VPC and one /24 subnet, with no internet gateway, NAT gateway, EIP, route table or daemon-to-store node port. Each launch claim gets one ENI in that subnet, in the VPC default security group, found again by its claim on a retry; two ENIs for one claim are refused. The launch intent is marked before the one create is sent, so a create whose ENI is not yet visible is never sent again: the launch fails closed with `IncorrectState` until the ENI appears or the outcome is reconciled. Worker-to-store TCP and the guest's route to the metadata service are not yet proven by any traffic test.
-The super-admin `EnsureSQSWorker` action on the internal `spinifex` operator service takes a `NodeId`, which must be a node of this cluster and a single subject token, and an `InstanceType`, and sends a request to `sqs.worker.Ensure.v1.<node>` with no queue group, so only that node answers and the worker stays pinned.
-That node's daemon re-checks the node the request names and the instance type, and refuses as `ServiceUnavailable` before allocating a volume or network or recording an intent unless the worker image's kernel and initramfs are non-empty files, its mgmt bridge exists and the cluster CA is readable; a call after the image is installed then launches.
-It then runs its one composed launcher for its own Region, one launch at a time, in the daemon's own context with a 30-minute bound, so a caller's timeout does not cancel it.
-A reply lost to that timeout is reported as `ServiceUnavailable`; calling again reports the bound worker, or `IncorrectState` for a dispatched launch whose outcome is unknown, which is never resent; a Region pinned to another node is refused as `InvalidParameterValue`.
-The reply names the bound VM's node, Region, instance ID and generation; bound is not ready, and it says nothing about the guest's boot, its key or the store.
-Stop/detach evidence for `RecordStopped` is not wired, the guest's metadata route and gateway TLS on a real ENI and worker-to-store TCP are unproven, and Viperblock's at-rest key is held by the platform.
-
-The worker identity record holds the instance type of its generation; a request for another type is refused as `IncorrectState`, before dispatch and after binding, the reply names the recorded type, and only a replacement after a confirmed stop may change it.
-Operators call both actions with `spx admin sqs ensure-worker --node --instance-type` and `spx admin sqs ensure-store --node --instance-type --volume-size-gib`, signed for the `spinifex` service with the standard AWS credential chain like the other `spx admin` gateway commands.
-Each command refuses missing inputs before sending anything and reports only the bound VM; a bound VM is not a ready worker or store.
-The super-admin `EnsureSQSStore` action takes a `NodeId`, an `InstanceType` and a `VolumeSizeGiB` and sends a request to `sqs.store.Ensure.v1.<node>` with no queue group; only the node the Region's worker is bound to can launch the store.
-That node refuses an instance type whose memory does not exceed 512 MiB, the store's `GOMEMLIMIT` under ADR-0011:S11; this is a floor, not a validated size, and no type is defaulted.
-It refuses before recording or allocating anything unless the store image and the cluster storage key are present, the worker is bound on this node, and that worker's identity has recorded its public key in the first-store decision, which it then advances to a launch intent.
-The store's own launch record in `spinifex-sqs-store-launch` holds only that public key, the claim, the node and the operator's instance type and size, and marks each step before its uncertain create: the volume, the ENI in the worker's private subnet, and the launch.
-A later request for a different instance type or size is refused as `IncorrectState`, before dispatch and after binding, and the reply names the type the store was launched with.
-A volume preparation that failed before sending any create, such as a missing `mkfs.ext4` or a scratch or format failure, withdraws its mark and returns `ServiceUnavailable`, so a retry may try again once the host is fixed.
-Any other marked step whose result is not visible is reported as `IncorrectState` and never resent; a volume that is not encrypted by its own record is refused as `UnsupportedOperation` with its ID kept, and the daemon re-checks the volume's encryption and the guest's size at launch.
-The daemon never receives the worker's seed; the store's NATS server accepts only that public key, and the reply's bound VM and address are not readiness, which only the worker can check.
-
-After boot, the worker's long-running `sqs-worker ready` step mounts and verifies its disk again and calls the internal `spinifex-sqs` `DescribeStore` action, which admits only the current worker and returns the bound store's address and recorded public key, or pending.
-It loads its key only if it matches that public key, then connects to the store's NATS on port 4222 with it and asks JetStream for account info, each pass bounded to 30 seconds.
-The step runs until shutdown with no overall deadline, so a store the operator launches at any time is picked up: it asks again every 5 seconds while the store is pending, not answering or the reply was lost, and re-checks a ready store every 30 seconds.
-A pass writes only the address and public key to `/run/sqs-worker/store-ready`; any failed pass removes that file, and so does the step's own exit.
-A start that finds shutdown in progress or no worker key logs once and waits in place rather than exiting, so init does not respawn it in a loop; shutdown stops it like a running watcher.
-Init respawns the step, at most once every 5 seconds, and every start removes a marker left by a watcher that was killed or crashed before the new watcher runs.
-A ready store that fails is noticed up to one re-check interval plus one bounded call later, so the marker can be up to about a minute stale.
-Shutdown blocks further respawns, removes the marker, and stops the watcher by its recorded PID and start time, escalating to SIGKILL after a budget; it unmounts the worker disk only once the watcher is confirmed gone, and a survivor leaves every mount in place.
-A mismatched key, a store that refuses the key, a key missing from the disk or an unclassified gateway refusal is asked again only every 5 minutes, so it never spins and still recovers if the operator fixes it; a missing key is never replaced.
-The NKey authenticates the worker to the store, but nothing authenticates the store to the worker: there is no TLS, so a pass means only that something at that address accepted the key.
-This is store readiness for the worker alone; there is no SQS API, and no production sizing or support is claimed.
+Evidence is local: unit tests, the guest shell test and an AWS SDK test that creates a queue and sends, receives and deletes a message through the real worker loop against a real local JetStream store and a restart.
+A real guest boot, its metadata route on a real ENI and the signed path on a live node are unproven, Viperblock's at-rest key is held by the platform, and SQS-managed message encryption is not implemented, so the profile must not hold tenant production data.
 
 ## Configuration
 

docs/PACKAGE_BOUNDARY_MIGRATION.md

@@ -60,7 +60,7 @@ intact; ADR-0001 explicitly defers that design.
 
 | Current path | Standing | Target ownership | Rule |
 |---|---|---|---|
-| `cmd/aws-model-coverage` | Qualification generator | Thin main; reusable code in `internal/tooling` | Produces the dispatch-versus-model inventory; never imported by product packages. |
+| `cmd/aws-model-coverage` | Qualification generator | Thin main; reusable code in `internal/tooling` | Produces the dispatch-versus-model inventory; never imported by product packages. Debt: it reads each domain's declared `OperationInventory` from `domains/<service>/awsapi` (ACM, EC2, ECR, ECS, EKS, ELBv2, IAM, RDS, Route 53, SQS, STS) and never constructs a registration. |
 | `cmd/dhcptest` | Development diagnostic | Thin main; reusable code in `internal/tooling` if extraction is useful | It is a DHCP probe, not a supported `spx admin` command. |
 | `cmd/spx-loadgen` | Qualification/load tool | Thin main; reusable code in `internal/testkit` | It drives test load and is not shipped to nodes. |
 | `tests/e2e/*/cmd/*`, `tests/bench/*/cmd/*` | Test and benchmark-private tools | Remain test-private | They are invoked by a test or benchmark harness, not a product process. |

docs/coverage/coverage.json

@@ -3978,6 +3978,86 @@
         "extra": []
       }
     },
+    {
+      "service": "sqs",
+      "api_version": "2012-11-05",
+      "opaque": false,
+      "note": "",
+      "operations": {
+        "modelled": [
+          "AddPermission",
+          "CancelMessageMoveTask",
+          "ChangeMessageVisibility",
+          "ChangeMessageVisibilityBatch",
+          "CreateQueue",
+          "DeleteMessage",
+          "DeleteMessageBatch",
+          "DeleteQueue",
+          "GetQueueAttributes",
+          "GetQueueUrl",
+          "ListDeadLetterSourceQueues",
+          "ListMessageMoveTasks",
+          "ListQueueTags",
+          "ListQueues",
+          "PurgeQueue",
+          "ReceiveMessage",
+          "RemovePermission",
+          "SendMessage",
+          "SendMessageBatch",
+          "SetQueueAttributes",
+          "StartMessageMoveTask",
+          "TagQueue",
+          "UntagQueue"
+        ],
+        "registered": [
+          "ChangeMessageVisibility",
+          "ChangeMessageVisibilityBatch",
+          "CreateQueue",
+          "DeleteMessage",
+          "DeleteMessageBatch",
+          "GetQueueAttributes",
+          "GetQueueUrl",
+          "ListQueueTags",
+          "ListQueues",
+          "ReceiveMessage",
+          "SendMessage",
+          "SendMessageBatch",
+          "SetQueueAttributes",
+          "TagQueue",
+          "UntagQueue"
+        ],
+        "implemented": [
+          "ChangeMessageVisibility",
+          "ChangeMessageVisibilityBatch",
+          "CreateQueue",
+          "DeleteMessage",
+          "DeleteMessageBatch",
+          "GetQueueAttributes",
+          "GetQueueUrl",
+          "ListQueueTags",
+          "ListQueues",
+          "ReceiveMessage",
+          "SendMessage",
+          "SendMessageBatch",
+          "SetQueueAttributes",
+          "TagQueue",
+          "UntagQueue"
+        ],
+        "stubbed": [],
+        "unsupported": [],
+        "missing": [
+          "AddPermission",
+          "CancelMessageMoveTask",
+          "DeleteQueue",
+          "ListDeadLetterSourceQueues",
+          "ListMessageMoveTasks",
+          "PurgeQueue",
+          "RemovePermission",
+          "StartMessageMoveTask"
+        ],
+        "extra": []
+      }
+    },
     {
       "service": "sts",
       "api_version": "2011-06-15",

docs/coverage/internal-report.md

@@ -1441,6 +1441,21 @@ request-payment: Requester Pays shifts transfer charges to the caller, which nee
 - `GetBucketRequestPayment`
 - `PutBucketRequestPayment`
 
+## sqs
+
+`2012-11-05` — 15 of 23 modelled operations implemented (65.2%), 0 stubbed, 0 not applicable, 0 registered outside the pinned model.
+
+### Not implemented
+
+- `AddPermission`
+- `CancelMessageMoveTask`
+- `DeleteQueue`
+- `ListDeadLetterSourceQueues`
+- `ListMessageMoveTasks`
+- `PurgeQueue`
+- `RemovePermission`
+- `StartMessageMoveTask`
+
 ## sts
 
 `2011-06-15` — 5 of 11 modelled operations implemented (45.5%), 0 stubbed, 0 not applicable, 0 registered outside the pinned model.

docs/install/install-oracle-linux/README.md

@@ -164,7 +164,7 @@ On Oracle Linux the installer makes these changes, which Debian hosts do not nee
 - **Repositories.** It enables `ol10_UEKR8` and `ol10_kvm_utils`, and adds `qemu-kvm*` and `qemu-img*` to the `excludepkgs` of `ol10_appstream`, keeping any excludes already set, so a later AppStream build can never replace the KVM Utilities QEMU.
 - **NFV SIG.** It writes `/etc/yum.repos.d/spinifex-nfv-sig.repo` with an `includepkgs=` list of the OVS and OVN packages, and imports the SIG key to `/etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-NFV` only when its fingerprint is `35154228174901BEFA8E69A621465E289D2A76A7`.
 - **OVS and OVN user.** Both run as root, as on Debian, through `OVS_USER_ID` and `OVN_USER_ID` in `/etc/sysconfig/openvswitch` and `/etc/sysconfig/ovn`.
-- **DNS.** Where systemd-resolved is installed but not running, it saves `/etc/resolv.conf` to `/var/lib/spinifex-reset/resolv.conf.orig`, starts resolved behind NetworkManager, tells NetworkManager in `/etc/NetworkManager/conf.d/90-spinifex-dns.conf` to leave `/etc/resolv.conf` alone, and links `/etc/resolv.conf` to the resolved stub. It then checks that names still resolve and stops, printing the commands that put DNS back, if they do not.
+- **DNS.** Where systemd-resolved is installed but not running, it saves `/etc/resolv.conf` to `/var/lib/spinifex-reset/resolv.conf.orig`, starts resolved behind NetworkManager and links `/etc/resolv.conf` to the resolved stub. It then checks that names still resolve and stops, printing the commands that put DNS back, if they do not.
 - **chrony.** It appends `confdir /etc/chrony.d` to `/etc/chrony.conf` once, and writes its step policy to `/etc/chrony.d/spinifex.conf`.
 - **`spx` under `sudo`.** Oracle Linux's `sudo` path leaves out `/usr/local/bin`, so the installer links `/usr/bin/spx` to `/usr/local/bin/spx`.
 

internal/awsmodel/embed.go

@@ -19,5 +19,6 @@ var modelFiles = map[Service]string{
 	RDS:                    "models/rds-2014-10-31.json.gz",
 	Route53:                "models/route-53-2013-04-01.json.gz",
 	S3:                     "models/s3-2006-03-01.json.gz",
+	SQS:                    "models/sqs-2012-11-05.json.gz",
 	STS:                    "models/sts-2011-06-15.json.gz",
 }

internal/awsmodel/model.go

@@ -36,6 +36,7 @@ const (
 	RDS                    Service = "rds"
 	Route53                Service = "route53"
 	S3                     Service = "s3"
+	SQS                    Service = "sqs"
 	STS                    Service = "sts"
 )
 

internal/awsmodel/model_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func TestServices(t *testing.T) {
-	want := []Service{ACM, EC2, ECR, ECS, EKS, ElasticLoadBalancingV2, IAM, RDS, Route53, S3, STS}
+	want := []Service{ACM, EC2, ECR, ECS, EKS, ElasticLoadBalancingV2, IAM, RDS, Route53, S3, SQS, STS}
 	if got := Services(); !reflect.DeepEqual(got, want) {
 		t.Fatalf("Services() = %v, want %v", got, want)
 	}

internal/sqsworkergw/client.go

@@ -1,6 +1,6 @@
 // Package sqsworkergw is the shared SQS worker guest's SigV4 client for the
 // gateway's internal spinifex-sqs service. It sends only public values: the
-// worker's identity is the signing session, and the store seed never leaves
+// worker's identity is the signing session, and the receipt key never leaves
 // the guest.
 package sqsworkergw
 
@@ -10,20 +10,21 @@ import (
 	"crypto/sha256"
 	"crypto/tls"
 	"crypto/x509"
+	"encoding/base64"
 	"encoding/hex"
+	"encoding/json"
 	"encoding/xml"
 	"errors"
 	"fmt"
 	"io"
 	"maps"
-	"net"
 	"net/http"
 	"net/url"
 	"strings"
 	"time"
 
 	"github.com/mulgadc/bluebottle/pkg/tlsconfig"
-	"github.com/nats-io/nkeys"
+	"github.com/mulgadc/spinifex/spinifex/domains/sqs/relay"
 )
 
 const (
@@ -33,9 +34,12 @@ const (
 	callTimeout    = 30 * time.Second
 	maxResponse    = 64 << 10
 
-	actionClaim    = "ClaimFirstStore"
-	actionDescribe = "DescribeStore"
-	actionRecord   = "RecordWorkerPublicKey"
+	actionClaim    = "ClaimReceiptKey"
+	actionRecord   = "RecordReceiptKey"
+	actionPoll     = "PollWork"
+	actionComplete = "CompleteWork"
+	// maxPollResponse fits the gateway's 16 MiB poll batch as base64 XML.
+	maxPollResponse = 32 << 20
 )
 
 // Signer signs a request in place for service and region.
@@ -43,7 +47,7 @@ type Signer interface {
 	Sign(r *http.Request, payloadHash, service, region string) error
 }
 
-// Client calls the internal first-store actions for this worker.
+// Client calls the internal SQS worker actions for this worker.
 type Client struct {
 	baseURL    string
 	region     string
@@ -114,80 +118,72 @@ func (e *TransientError) Error() string   { return e.Err.Error() }
 func (e *TransientError) Unwrap() error   { return e.Err }
 func (e *TransientError) Retryable() bool { return true }
 
-type claimResult struct {
-	CreateKey bool   `xml:"CreateKey"`
-	PublicKey string `xml:"PublicKey"`
+// ReceiptKey is the worker volume's receipt key initialization as the
+// gateway reports it: pending authorizes creating or resuming one, otherwise
+// Fingerprint names the key the disk must already hold.
+type ReceiptKey struct {
+	Pending     bool
+	Fingerprint string
+	// HostNodes is the gateway's declared cluster node count; a gateway
+	// that does not report one reads as zero.
+	HostNodes int
 }
 
-// Claim asks for this worker's current first-store decision. A decision
-// that is neither "create" nor one valid public key is refused here.
-func (c *Client) Claim(ctx context.Context) (bool, string, error) {
+type claimResult struct {
+	Pending     bool   `xml:"Pending"`
+	Fingerprint string `xml:"Fingerprint"`
+	HostNodes   int    `xml:"HostNodes"`
+}
+
+// ClaimReceiptKey asks for the receipt key decision. A reply that is neither
+// pending nor one valid fingerprint is refused here.
+func (c *Client) ClaimReceiptKey(ctx context.Context) (ReceiptKey, error) {
 	var res claimResult
 	if err := c.call(ctx, actionClaim, nil, &res); err != nil {
-		return false, "", err
+		return ReceiptKey{}, err
 	}
-	switch {
-	case res.CreateKey && res.PublicKey == "":
-		return true, "", nil
-	case !res.CreateKey && nkeys.IsValidPublicUserKey(res.PublicKey):
-		return false, res.PublicKey, nil
+	if res.Pending == (res.Fingerprint != "") || (res.Fingerprint != "" && !ValidFingerprint(res.Fingerprint)) {
+		return ReceiptKey{}, errors.New("spinifex-sqs ClaimReceiptKey: response is not a valid decision")
 	}
-	return false, "", errors.New("spinifex-sqs ClaimFirstStore: response is not a valid decision")
+	return ReceiptKey(res), nil
 }
 
-// Store is the Region's store launch as the gateway reports it. Bound means
-// a VM was launched for it, not that it is ready.
-type Store struct {
-	Bound     bool
-	Address   string
-	PublicKey string
-}
-
-type describeResult struct {
-	State     string `xml:"State"`
-	Address   string `xml:"Address"`
-	PublicKey string `xml:"PublicKey"`
-}
-
-// DescribeStore reports this worker's store. A bound reply must carry an
-// IPv4 address and a valid public key; a pending one must carry neither.
-func (c *Client) DescribeStore(ctx context.Context) (Store, error) {
-	var res describeResult
-	if err := c.call(ctx, actionDescribe, nil, &res); err != nil {
-		return Store{}, err
+// RecordReceiptKey records the key's fingerprint. Recording the same one
+// again succeeds, so a lost reply is safe to repeat.
+func (c *Client) RecordReceiptKey(ctx context.Context, fingerprint string) error {
+	if !ValidFingerprint(fingerprint) {
+		return errors.New("spinifex-sqs RecordReceiptKey: refusing to send a value that is not a fingerprint")
 	}
-	switch {
-	case res.State == "pending" && res.Address == "" && res.PublicKey == "":
-		return Store{}, nil
-	case res.State == "bound" && isIPv4(res.Address) && nkeys.IsValidPublicUserKey(res.PublicKey):
-		return Store{Bound: true, Address: res.Address, PublicKey: res.PublicKey}, nil
-	}
-	return Store{}, errors.New("spinifex-sqs DescribeStore: response is not a valid store state")
+	return c.call(ctx, actionRecord, url.Values{"Fingerprint": {fingerprint}}, nil)
 }
 
-func isIPv4(s string) bool {
-	ip := net.ParseIP(s)
-	return ip != nil && ip.To4() != nil && !strings.Contains(s, ":")
-}
-
-// RecordPublicKey records the worker's public key. Anything other than a
-// user public key is refused before it is sent.
-func (c *Client) RecordPublicKey(ctx context.Context, publicKey string) error {
-	if !nkeys.IsValidPublicUserKey(publicKey) {
-		return errors.New("spinifex-sqs RecordWorkerPublicKey: refusing to send a value that is not a user public key")
+// ValidFingerprint reports a lowercase hex SHA-256 fingerprint.
+func ValidFingerprint(s string) bool {
+	if len(s) != 2*sha256.Size || strings.ToLower(s) != s {
+		return false
 	}
-	return c.call(ctx, actionRecord, url.Values{"PublicKey": {publicKey}}, nil)
+	_, err := hex.DecodeString(s)
+	return err == nil
 }
 
 func (c *Client) call(ctx context.Context, action string, params url.Values, out any) error {
 	form := url.Values{"Action": {action}, "Version": {apiVersion}}
 	maps.Copy(form, params)
-	body := []byte(form.Encode())
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/", bytes.NewReader(body))
+	return c.do(ctx, action, "", []byte(form.Encode()), "application/x-www-form-urlencoded; charset=utf-8", maxResponse, out)
+}
+
+// do signs and sends one request. rawQuery, when set, carries the action and
+// its arguments; the body hash is signed either way.
+func (c *Client) do(ctx context.Context, action, rawQuery string, body []byte, contentType string, limit int64, out any) error {
+	target := c.baseURL + "/"
+	if rawQuery != "" {
+		target += "?" + rawQuery
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, target, bytes.NewReader(body))
 	if err != nil {
 		return fmt.Errorf("spinifex-sqs %s: build request: %w", action, err)
 	}
-	req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=utf-8")
+	req.Header.Set("Content-Type", contentType)
 	sum := sha256.Sum256(body)
 	if err := c.signer.Sign(req, hex.EncodeToString(sum[:]), signingService, c.region); err != nil {
 		return &TransientError{Err: fmt.Errorf("spinifex-sqs %s: sign: %w", action, err)}
@@ -197,7 +193,7 @@ func (c *Client) call(ctx context.Context, action string, params url.Values, out
 		return &TransientError{Err: fmt.Errorf("spinifex-sqs %s: send: %w", action, err)}
 	}
 	defer resp.Body.Close()
-	respBody, err := io.ReadAll(io.LimitReader(resp.Body, maxResponse))
+	respBody, err := io.ReadAll(io.LimitReader(resp.Body, limit))
 	if err != nil {
 		return &TransientError{Err: fmt.Errorf("spinifex-sqs %s: read response: %w", action, err)}
 	}
@@ -210,6 +206,44 @@ func (c *Client) call(ctx context.Context, action string, params url.Values, out
 	return decodeResult(action, respBody, out)
 }
 
+// PollWork long-polls the gateway for owner calls queued for this worker.
+// An empty list is a poll that found nothing.
+func (c *Client) PollWork(ctx context.Context) ([]relay.Work, error) {
+	var res struct {
+		Items []string `xml:"Items>member"`
+	}
+	if err := c.do(ctx, actionPoll, "", []byte(url.Values{"Action": {actionPoll}, "Version": {apiVersion}}.Encode()),
+		"application/x-www-form-urlencoded; charset=utf-8", maxPollResponse, &res); err != nil {
+		return nil, err
+	}
+	out := make([]relay.Work, 0, len(res.Items))
+	for _, item := range res.Items {
+		raw, err := base64.StdEncoding.DecodeString(item)
+		if err != nil {
+			return nil, errors.New("spinifex-sqs PollWork: item is not base64")
+		}
+		var w relay.Work
+		if err := json.Unmarshal(raw, &w); err != nil || w.ID == "" || w.Op == "" {
+			return nil, errors.New("spinifex-sqs PollWork: item is not a work item")
+		}
+		out = append(out, w)
+	}
+	return out, nil
+}
+
+// CompleteWork posts an encoded result for work item id and returns the
+// gateway's outcome name. Any error leaves delivery unknown.
+func (c *Client) CompleteWork(ctx context.Context, id string, result []byte) (string, error) {
+	q := url.Values{"Action": {actionComplete}, "Version": {apiVersion}, "WorkId": {id}}
+	var res struct {
+		Outcome string `xml:"Outcome"`
+	}
+	if err := c.do(ctx, actionComplete, q.Encode(), result, "application/json", maxResponse, &res); err != nil {
+		return "", err
+	}
+	return res.Outcome, nil
+}
+
 func decodeResult(action string, body []byte, out any) error {
 	want := action + "Result"
 	dec := xml.NewDecoder(bytes.NewReader(body))

internal/sqsworkergw/client_test.go

@@ -10,8 +10,6 @@ import (
 	"strings"
 	"sync/atomic"
 	"testing"
-
-	"github.com/nats-io/nkeys"
 )
 
 type nopSigner struct{ err error }
@@ -24,15 +22,10 @@ func (s nopSigner) Sign(r *http.Request, _, service, region string) error {
 	return nil
 }
 
-func userKey(t *testing.T) (pub, seed string) {
-	t.Helper()
-	kp, err := nkeys.CreateUser()
-	if err != nil {
-		t.Fatal(err)
-	}
-	p, _ := kp.PublicKey()
-	s, _ := kp.Seed()
-	return p, string(s)
+var fp = strings.Repeat("ab", 32)
+
+func claimBody(inner string) string {
+	return `<ClaimReceiptKeyResponse><ClaimReceiptKeyResult>` + inner + `</ClaimReceiptKeyResult></ClaimReceiptKeyResponse>`
 }
 
 // server answers with body and status, counting and keeping requests.
@@ -73,63 +66,56 @@ func TestNew_RefusesWithoutHTTPSRegionCAOrSigner(t *testing.T) {
 }
 
 func TestClaim_SignsForTheInternalServiceAndRegion(t *testing.T) {
-	c, _, seen := server(t, 200, `<ClaimFirstStoreResponse><ClaimFirstStoreResult><CreateKey>true</CreateKey></ClaimFirstStoreResult></ClaimFirstStoreResponse>`)
-	create, pub, err := c.Claim(context.Background())
-	if err != nil || !create || pub != "" {
-		t.Fatalf("claim = %v %q %v", create, pub, err)
+	c, _, seen := server(t, 200, claimBody(`<Pending>true</Pending><HostNodes>1</HostNodes>`))
+	got, err := c.ClaimReceiptKey(context.Background())
+	if err != nil || !got.Pending || got.Fingerprint != "" || got.HostNodes != 1 {
+		t.Fatalf("claim = %+v %v", got, err)
 	}
-	if got := (*seen)[0]; !strings.HasPrefix(got, "test spinifex-sqs/ap-southeast-2|") || !strings.Contains(got, "Action=ClaimFirstStore") {
-		t.Fatalf("request = %q", got)
+	if req := (*seen)[0]; !strings.HasPrefix(req, "test spinifex-sqs/ap-southeast-2|") || !strings.Contains(req, "Action=ClaimReceiptKey") {
+		t.Fatalf("request = %q", req)
 	}
 }
 
-// Only "create" or exactly one valid public key is a decision.
+// Only pending alone, or exactly one valid fingerprint, is a decision.
 func TestClaim_RefusesAnythingButOneDecision(t *testing.T) {
-	pub, seed := userKey(t)
-	cases := map[string]string{
-		"create with a key": `<CreateKey>true</CreateKey><PublicKey>` + pub + `</PublicKey>`,
-		"neither":           `<CreateKey>false</CreateKey>`,
-		"seed as key":       `<CreateKey>false</CreateKey><PublicKey>` + seed + `</PublicKey>`,
-		"no result":         ``,
-	}
-	for name, inner := range cases {
+	for name, inner := range map[string]string{
+		"pending with a key":  `<Pending>true</Pending><Fingerprint>` + fp + `</Fingerprint>`,
+		"neither":             `<Pending>false</Pending>`,
+		"uppercase key":       `<Pending>false</Pending><Fingerprint>` + strings.ToUpper(fp) + `</Fingerprint>`,
+		"short key":           `<Pending>false</Pending><Fingerprint>` + fp[:62] + `</Fingerprint>`,
+		"not the claim reply": "",
+	} {
 		t.Run(name, func(t *testing.T) {
-			body := `<ClaimFirstStoreResponse><ClaimFirstStoreResult>` + inner + `</ClaimFirstStoreResult></ClaimFirstStoreResponse>`
+			body := claimBody(inner)
 			if inner == "" {
 				body = `<Other/>`
 			}
 			c, _, _ := server(t, 200, body)
-			_, _, err := c.Claim(context.Background())
-			if err == nil {
+			if _, err := c.ClaimReceiptKey(context.Background()); err == nil {
 				t.Fatal("accepted a non-decision")
 			}
-			if strings.Contains(err.Error(), seed) {
-				t.Fatal("the error echoes a seed")
-			}
 		})
 	}
-	t.Run("existing", func(t *testing.T) {
-		c, _, _ := server(t, 200, `<ClaimFirstStoreResponse><ClaimFirstStoreResult><CreateKey>false</CreateKey><PublicKey>`+pub+`</PublicKey></ClaimFirstStoreResult></ClaimFirstStoreResponse>`)
-		create, got, err := c.Claim(context.Background())
-		if err != nil || create || got != pub {
-			t.Fatalf("claim = %v %q %v", create, got, err)
-		}
-	})
+	c, _, _ := server(t, 200, claimBody(`<Pending>false</Pending><Fingerprint>`+fp+`</Fingerprint>`))
+	got, err := c.ClaimReceiptKey(context.Background())
+	if err != nil || got.Pending || got.Fingerprint != fp {
+		t.Fatalf("recorded claim = %+v %v", got, err)
+	}
 }
 
-func TestRecordPublicKey_SendsOnlyAPublicKey(t *testing.T) {
-	pub, seed := userKey(t)
-	c, n, seen := server(t, 200, `<RecordWorkerPublicKeyResponse/>`)
-	if err := c.RecordPublicKey(context.Background(), seed); err == nil || strings.Contains(err.Error(), seed) {
-		t.Fatalf("seed err = %v", err)
+func TestRecordReceiptKey_SendsOnlyAFingerprint(t *testing.T) {
+	c, n, seen := server(t, 200, `<RecordReceiptKeyResponse/>`)
+	const raw = "not-a-fingerprint-and-maybe-key-material"
+	if err := c.RecordReceiptKey(context.Background(), raw); err == nil || strings.Contains(err.Error(), raw) {
+		t.Fatalf("invalid err = %v", err)
 	}
 	if n.Load() != 0 {
-		t.Fatal("a seed was sent")
+		t.Fatal("a non-fingerprint was sent")
 	}
-	if err := c.RecordPublicKey(context.Background(), pub); err != nil {
+	if err := c.RecordReceiptKey(context.Background(), fp); err != nil {
 		t.Fatal(err)
 	}
-	if !strings.Contains((*seen)[0], "PublicKey="+pub) {
+	if !strings.Contains((*seen)[0], "Fingerprint="+fp) {
 		t.Fatalf("request = %q", (*seen)[0])
 	}
 }
@@ -151,7 +137,7 @@ func TestErrors_ClassifiedWithoutEchoingTheBody(t *testing.T) {
 	}
 	for _, tc := range cases {
 		c, _, _ := server(t, tc.status, `<ErrorResponse><Error><Code>`+tc.code+`</Code><Message>secret-ish detail</Message></Error></ErrorResponse>`)
-		_, _, err := c.Claim(context.Background())
+		_, err := c.ClaimReceiptKey(context.Background())
 		api, ok := errors.AsType[*APIError](err)
 		if !ok || api.Code != tc.code || api.Retryable() != tc.retryable || api.Pending() != (tc.code == "AccessDenied") {
 			t.Fatalf("%s: err = %v", tc.code, err)
@@ -163,7 +149,7 @@ func TestErrors_ClassifiedWithoutEchoingTheBody(t *testing.T) {
 
 	c, _, _ := server(t, 200, "")
 	c.baseURL = "https://127.0.0.1:1"
-	_, _, err := c.Claim(context.Background())
+	_, err := c.ClaimReceiptKey(context.Background())
 	tr, ok := errors.AsType[*TransientError](err)
 	if !ok || !tr.Retryable() {
 		t.Fatalf("unreachable err = %v, want TransientError", err)
@@ -181,55 +167,10 @@ func TestCall_DoesNotFollowRedirects(t *testing.T) {
 	}))
 	t.Cleanup(srv.Close)
 	c.baseURL = srv.URL
-	if _, _, err := c.Claim(context.Background()); err == nil {
+	if _, err := c.ClaimReceiptKey(context.Background()); err == nil {
 		t.Fatal("a redirect was accepted as a decision")
 	}
 	if followed.Load() {
 		t.Fatal("the signed request followed a redirect")
 	}
 }
-
-// Only "pending" with nothing else, or "bound" with an IPv4 address and one
-// valid public key, is a store state.
-func TestDescribeStore_AcceptsOnlyAValidState(t *testing.T) {
-	pub, seed := userKey(t)
-	wrap := func(inner string) string {
-		return `<DescribeStoreResponse><DescribeStoreResult>` + inner + `</DescribeStoreResult></DescribeStoreResponse>`
-	}
-	t.Run("pending", func(t *testing.T) {
-		c, _, seen := server(t, 200, wrap(`<State>pending</State>`))
-		st, err := c.DescribeStore(context.Background())
-		if err != nil || st.Bound {
-			t.Fatalf("store = %+v %v", st, err)
-		}
-		if !strings.Contains((*seen)[0], "Action=DescribeStore") {
-			t.Fatalf("request = %q", (*seen)[0])
-		}
-	})
-	t.Run("bound", func(t *testing.T) {
-		c, _, _ := server(t, 200, wrap(`<State>bound</State><Address>10.240.0.9</Address><PublicKey>`+pub+`</PublicKey>`))
-		st, err := c.DescribeStore(context.Background())
-		if err != nil || !st.Bound || st.Address != "10.240.0.9" || st.PublicKey != pub {
-			t.Fatalf("store = %+v %v", st, err)
-		}
-	})
-	for name, inner := range map[string]string{
-		"pending with address": `<State>pending</State><Address>10.240.0.9</Address>`,
-		"bound without key":    `<State>bound</State><Address>10.240.0.9</Address>`,
-		"bound with seed":      `<State>bound</State><Address>10.240.0.9</Address><PublicKey>` + seed + `</PublicKey>`,
-		"bound with hostname":  `<State>bound</State><Address>store.example</Address><PublicKey>` + pub + `</PublicKey>`,
-		"bound with IPv6":      `<State>bound</State><Address>::ffff:10.240.0.9</Address><PublicKey>` + pub + `</PublicKey>`,
-		"unknown state":        `<State>ready</State>`,
-	} {
-		t.Run(name, func(t *testing.T) {
-			c, _, _ := server(t, 200, wrap(inner))
-			_, err := c.DescribeStore(context.Background())
-			if err == nil {
-				t.Fatal("accepted an invalid store state")
-			}
-			if strings.Contains(err.Error(), seed) {
-				t.Fatal("the error echoes a seed")
-			}
-		})
-	}
-}

scripts/README.md

@@ -6,7 +6,7 @@ Node installation, dev-environment lifecycle, guest image builds, and verificati
 
 | Script | What it does |
 |---|---|
-| `setup.sh` | The production binary installer, served from `install.mulgadc.com`. Installs deps, users, sudoers, the host firewall policy, files, systemd units, logrotate and udev rules, and loads `openvswitch`, `geneve`, `vhost_net` and `tun`, listing them in `/etc/modules-load.d/spinifex.conf` for later boots. It masks firewalld wherever it is installed, since the Spinifex nftables policy is the only supported node firewall. It writes a needrestart drop-in so an apt upgrade never restarts the Spinifex, OVS or OVN units; restarting them after an upgrade is left to the operator. An unset `--firewall` keeps the mode recorded in `/etc/spinifex/firewall/mode`, and otherwise arms the policy when firewalld was active. Where systemd-resolved is installed but not running, resolvconf is absent and NetworkManager runs, it hands host DNS to resolved: it saves `/etc/resolv.conf` once to `/var/lib/spinifex-reset/resolv.conf.orig`, writes `/etc/NetworkManager/conf.d/90-spinifex-dns.conf` (`dns=systemd-resolved`, `rc-manager=unmanaged`) so NetworkManager never rewrites `/etc/resolv.conf`, links `/etc/resolv.conf` to the resolved stub, and stops unless the link holds, resolved has upstream servers and the package repository host still resolves. A rerun after that handover adds a missing drop-in, reloads NetworkManager and relinks the stub. It links `/usr/bin/spx` to `/usr/local/bin/spx` where sudo's PATH leaves out `/usr/local/bin`, and takes the nbdkit plugin directory from `nbdkit --dump-config`, stopping when nbdkit cannot say. `SETUP_STAGES` runs a subset; `ISO_BUILD=1` runs it inside the ISO builder's chroot. |
+| `setup.sh` | The production binary installer, served from `install.mulgadc.com`. Installs deps, users, sudoers, the host firewall policy, files, systemd units, logrotate and udev rules, and loads `openvswitch`, `geneve`, `vhost_net` and `tun`, listing them in `/etc/modules-load.d/spinifex.conf` for later boots. It masks firewalld wherever it is installed, since the Spinifex nftables policy is the only supported node firewall. It writes a needrestart drop-in so an apt upgrade never restarts the Spinifex, OVS or OVN units; restarting them after an upgrade is left to the operator. An unset `--firewall` keeps the mode recorded in `/etc/spinifex/firewall/mode`, and otherwise arms the policy when firewalld was active. Where systemd-resolved is installed but not running, resolvconf is absent and NetworkManager runs, it hands host DNS to resolved: it saves `/etc/resolv.conf` once to `/var/lib/spinifex-reset/resolv.conf.orig`, links `/etc/resolv.conf` to the resolved stub, and stops unless resolved has upstream servers and the package repository host still resolves. It links `/usr/bin/spx` to `/usr/local/bin/spx` where sudo's PATH leaves out `/usr/local/bin`, and takes the nbdkit plugin directory from `nbdkit --dump-config`, stopping when nbdkit cannot say. `SETUP_STAGES` runs a subset; `ISO_BUILD=1` runs it inside the ISO builder's chroot. |
 | `setup-ovn.sh` | Bootstraps a compute node for OVN VPC networking: installs OVN/OVS, creates `br-int`, configures the WAN bridge (bridged, DHCP, or routed-NAT uplink), sets the chassis identity, and applies overlay sysctls. It runs OVS and OVN as root, setting `OVS_USER_ID` and `OVN_USER_ID` where `/etc/sysconfig` holds them, and hooks the socket-permission helper onto `ovs-vswitchd`, `ovn-controller`, `openvswitch-ipsec` and, on EL, `ovn-northd`. `--management` also starts the OVN central services, writes their `ovn-ctl` flags to `/etc/default/ovn-central` (Debian) or `/etc/sysconfig/ovn-northd` (EL), and records the role in `/etc/spinifex/ovn-role`. With clustered databases it returns only once this node's NB and SB members are back in their RAFT clusters under a known leader, and fails after `OVN_RAFT_REJOIN_WAIT` seconds (default 60) otherwise, so reruns one node at a time never take two members of a cluster down. A later run without `--management` refuses while that file, or the `OVN_CTL_OPTS` line older nodes carry in `/etc/default/ovn-central`, records the role. It adds `maxsize 100M` to the OVS and OVN logrotate files, plus `rotate 5` where the file sets no count, and rotates as root, matching the daemons. On a host whose IPsec stack is libreswan it enables, stops and masks `openvswitch-ipsec.service` and `ipsec.service` for the daemon to unmask when IPsec is on, unless `openvswitch-ipsec.service` is already running; elsewhere it enables and starts `openvswitch-ipsec.service`, unless the daemon has masked it to keep IPsec off. The links it creates persist through systemd-networkd where that is enabled, and otherwise through `/etc/spinifex/hostnet.json`, which `spinifex-hostnet.service` (`spx admin hostnet apply`) replays at boot and on every OVS start, starting `ovn-controller` again when an OVS stop took it down; the first run records the choice in `/etc/spinifex/hostnet-backend`. Where NetworkManager runs, it leaves the links Spinifex owns unmanaged. |
 | `install-node.sh` | Forms a multi-node cluster across hosts that are already running Spinifex: rebuilds the OVN database in clustered form, runs `spx admin init` on the first host and `spx admin join` on the rest, then verifies. Before touching any host it refuses hosts of different OS families (`distro_os_family`) or `spx` versions, and `--ipsec on` when any host uses libreswan. The scripted form of [install-multi-node](../docs/install/install-multi-node/README.md). Destroys the joining nodes' logical network state and crypto — see its `--help`. |
 | `dev-install.sh` | Full local dev setup through the production installer: builds from source, packs a tarball, runs `setup.sh`, initialises the cluster, and starts services under systemd. |

scripts/build-sqs-store-microvm-image.sh

@@ -1,22 +1,18 @@
 #!/bin/bash
-# build-sqs-store-microvm-image.sh — build the SQS store microVM kernel +
-# initramfs: the dedicated nats-server, ext4 support, the store's guest steps
-# and a clean power-down. The guest never formats a disk; build/microvm/
-# sqs-store/sqs-store-lib.sh states its fw_cfg contract. The LB image
-# (build-microvm-image.sh) is separate and unchanged.
-#
-# SQS_MICROVM_PROFILE=worker builds the shared SQS worker guest instead: the
-# same network /init and kernel, build/microvm/sqs-worker's steps and
-# $REPO_ROOT/bin/sqs-worker, with no nats-server.
+# build-sqs-store-microvm-image.sh — build the SQS microVM kernel + initramfs:
+# one guest running the dedicated SQS nats-server and the SQS worker as separate
+# processes, ext4 support, its guest steps and a clean power-down. The guest
+# never formats a disk. The LB image (build-microvm-image.sh) is separate.
 #
 # Inputs:
-#   $REPO_ROOT/build/microvm/sqs-store/init.sh, inittab
-#   $REPO_ROOT/build/microvm/sqs-store/sqs-store-{lib.sh,prepare,run,stop}
+#   $REPO_ROOT/build/microvm/sqs-worker/init.sh, inittab
+#   $REPO_ROOT/build/microvm/sqs-worker/sqs-worker-{lib.sh,prepare,ready,stop}
 #   $REPO_ROOT/bin/nats-server  (make bin/nats-server, the go.mod-pinned version)
+#   $REPO_ROOT/bin/sqs-worker
 #
 # Outputs:
-#   $REPO_ROOT/build/microvm/sqs-store/vmlinuz
-#   $REPO_ROOT/build/microvm/sqs-store/initramfs.cpio.gz
+#   $REPO_ROOT/build/microvm/sqs-worker/vmlinuz
+#   $REPO_ROOT/build/microvm/sqs-worker/initramfs.cpio.gz
 #
 # Rootfs strategy (first match wins):
 #   1. apk available on host → install directly into a temp chroot (Alpine CI/CD)
@@ -25,20 +21,16 @@
 set -euo pipefail
 
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
-BUILD_DIR="$REPO_ROOT/build/microvm/sqs-store"
+BUILD_DIR="$REPO_ROOT/build/microvm/sqs-worker"
 INIT_SH="$BUILD_DIR/init.sh"
 INITTAB="$BUILD_DIR/inittab"
 NATS_SERVER_BIN="$REPO_ROOT/bin/nats-server"
+SQS_WORKER_BIN="$REPO_ROOT/bin/sqs-worker"
 
-PROFILE="${SQS_MICROVM_PROFILE:-store}"
+PROFILE="${SQS_MICROVM_PROFILE:-worker}"
 case "$PROFILE" in
-    store) ;;
-    worker)
-        BUILD_DIR="$REPO_ROOT/build/microvm/sqs-worker"
-        INITTAB="$BUILD_DIR/inittab"
-        SQS_WORKER_BIN="$REPO_ROOT/bin/sqs-worker"
-        ;;
-    *) echo "ERROR: SQS_MICROVM_PROFILE must be store or worker, not $PROFILE" >&2; exit 1 ;;
+    worker) ;;
+    *) echo "ERROR: SQS_MICROVM_PROFILE must be worker, not $PROFILE" >&2; exit 1 ;;
 esac
 TAG="[build-sqs-$PROFILE-microvm-image]"
 
@@ -68,16 +60,14 @@ for tool in cpio gzip find fakeroot; do
     fi
 done
 
-if [ "$PROFILE" = worker ]; then
-    if [ ! -x "$SQS_WORKER_BIN" ]; then
-        echo "ERROR: $SQS_WORKER_BIN not found — run 'make bin/sqs-worker'" >&2
-        exit 1
-    fi
-    if ! file -b "$SQS_WORKER_BIN" | grep -q "statically linked"; then
-        echo "ERROR: $SQS_WORKER_BIN is not statically linked — it cannot run on the musl guest" >&2
-        exit 1
-    fi
-else
+if [ ! -x "$SQS_WORKER_BIN" ]; then
+    echo "ERROR: $SQS_WORKER_BIN not found — run 'make bin/sqs-worker'" >&2
+    exit 1
+fi
+if ! file -b "$SQS_WORKER_BIN" | grep -q "statically linked"; then
+    echo "ERROR: $SQS_WORKER_BIN is not statically linked — it cannot run on the musl guest" >&2
+    exit 1
+fi
 # --- Verify the dedicated nats-server is the go.mod-pinned build ---
 # A stale or host-installed binary would ship an unreviewed server version.
 NATS_SERVER_PIN=$(awk '$1 == "github.com/nats-io/nats-server/v2" { print $2; exit }' "$REPO_ROOT/go.mod")
@@ -99,7 +89,6 @@ if [ "$nats_server_version" != "$NATS_SERVER_PIN" ]; then
     exit 1
 fi
 echo "$TAG nats-server: $nats_server_version (matches go.mod)"
-fi
 
 # --- Create chroot ---
 CHROOT_DIR=$(mktemp -d)
@@ -213,39 +202,23 @@ fi
 echo "$TAG installing inittab..."
 install -m 0644 "$INITTAB" "$CHROOT_DIR/etc/inittab"
 
-if [ "$PROFILE" = worker ]; then
-    echo "$TAG installing worker guest steps..."
-    for step in sqs-worker-lib.sh sqs-worker-prepare sqs-worker-ready sqs-worker-stop; do
-        if [ ! -f "$BUILD_DIR/$step" ]; then
-            echo "ERROR: $BUILD_DIR/$step not found — cannot build initramfs" >&2
-            exit 1
-        fi
-    done
-    mkdir -p "$CHROOT_DIR/usr/local/lib/sqs-worker" "$CHROOT_DIR/usr/local/bin"
-    install -m 0755 "$BUILD_DIR/sqs-worker-lib.sh" "$BUILD_DIR/sqs-worker-prepare" "$BUILD_DIR/sqs-worker-ready" "$BUILD_DIR/sqs-worker-stop" \
-        "$CHROOT_DIR/usr/local/lib/sqs-worker/"
-    install -m 0755 "$SQS_WORKER_BIN" "$CHROOT_DIR/usr/local/bin/sqs-worker"
-    # The mountpoint ships empty; the worker proves its disk is mounted here.
-    mkdir -p "$CHROOT_DIR/data"
-else
-# --- Place the store's guest steps ---
-echo "$TAG installing store guest steps..."
-mkdir -p "$CHROOT_DIR/usr/local/lib/sqs-store"
-for step in sqs-store-lib.sh sqs-store-prepare sqs-store-run sqs-store-stop; do
+echo "$TAG installing worker guest steps..."
+for step in sqs-worker-lib.sh sqs-worker-prepare sqs-worker-store sqs-worker-ready sqs-worker-stop; do
     if [ ! -f "$BUILD_DIR/$step" ]; then
         echo "ERROR: $BUILD_DIR/$step not found — cannot build initramfs" >&2
         exit 1
     fi
 done
-install -m 0644 "$BUILD_DIR/sqs-store-lib.sh" "$CHROOT_DIR/usr/local/lib/sqs-store/"
-install -m 0755 "$BUILD_DIR/sqs-store-prepare" "$BUILD_DIR/sqs-store-run" "$BUILD_DIR/sqs-store-stop" \
-    "$CHROOT_DIR/usr/local/lib/sqs-store/"
-
+mkdir -p "$CHROOT_DIR/usr/local/lib/sqs-worker" "$CHROOT_DIR/usr/local/bin"
+install -m 0755 "$BUILD_DIR/sqs-worker-lib.sh" "$BUILD_DIR/sqs-worker-prepare" "$BUILD_DIR/sqs-worker-store" "$BUILD_DIR/sqs-worker-ready" "$BUILD_DIR/sqs-worker-stop" \
+    "$CHROOT_DIR/usr/local/lib/sqs-worker/"
+install -m 0755 "$SQS_WORKER_BIN" "$CHROOT_DIR/usr/local/bin/sqs-worker"
+# The mountpoint ships empty; the worker proves its disk is mounted here.
+mkdir -p "$CHROOT_DIR/data"
 # --- Copy nats-server binary ---
 echo "$TAG copying nats-server binary..."
 mkdir -p "$CHROOT_DIR/usr/local/bin"
 install -m 0755 "$NATS_SERVER_BIN" "$CHROOT_DIR/usr/local/bin/nats-server"
-fi
 
 # --- Create required directories for init ---
 mkdir -p \

scripts/images/eks-node/eks-node-role.sh

@@ -12,12 +12,11 @@ set -eu
 # once.
 #
 #   server      → eks-token-webhook, k3s (server), k3s-first-boot (bootstrap
-#                 publish), mulga-eks-state-report, mulga-eks-node-gc
+#                 publish), mulga-eks-state-report
 #   server-join → eks-token-webhook, k3s (server, joins the first server's etcd
-#                 quorum), mulga-eks-state-report, mulga-eks-node-gc. NO
-#                 k3s-first-boot: the first server already publishes the
-#                 cluster-identical bootstrap artifacts; a join re-publish only
-#                 races the bootstrap bus.
+#                 quorum), mulga-eks-state-report. NO k3s-first-boot: the first
+#                 server already publishes the cluster-identical bootstrap
+#                 artifacts; a join re-publish only races the bootstrap bus.
 #   agent       → k3s-agent
 #
 # Paths are overridable via env so the selection logic is unit-testable (bats)
@@ -141,7 +140,6 @@ case "${ROLE}" in
         # writer owns. HA multi-server addon delivery is tracked separately
         # Server-join nodes do not run this setup.
         svc_enable mulga-eks-addon-sync
-        svc_enable mulga-eks-node-gc
         svc_start eks-token-webhook
         svc_start mulga-eks-k3s-recovery
         svc_start k3s
@@ -149,7 +147,6 @@ case "${ROLE}" in
         svc_start k3s-first-boot
         svc_start mulga-eks-state-report
         svc_start mulga-eks-addon-sync
-        svc_start mulga-eks-node-gc
         ;;
     server-join)
         log "configuring server-join role"
@@ -161,13 +158,11 @@ case "${ROLE}" in
         svc_enable k3s
         svc_enable konnectivity-server
         svc_enable mulga-eks-state-report
-        svc_enable mulga-eks-node-gc
         svc_start eks-token-webhook
         svc_start mulga-eks-k3s-recovery
         svc_start k3s
         svc_start konnectivity-server
         svc_start mulga-eks-state-report
-        svc_start mulga-eks-node-gc
         ;;
     agent)
         log "configuring agent role"

scripts/images/eks-node/eks-node-role_test.sh

@@ -141,8 +141,6 @@ for INIT in openrc systemd; do
             END { print (rn > 0 && kn > 0 && rn < kn) ? "yes" : "no" }
         ' "${CALLS}"
     )"
-    check "server(${INIT}): enables node-gc" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-node-gc)")"
-    check "server(${INIT}): starts node-gc" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-node-gc)")"
     check "server(${INIT}): no k3s-agent" "no" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
     check "server(${INIT}): self-disables" "yes" "$(present "${CALLS}" "$(disable_cmd "${INIT}" eks-node-role)")"
 
@@ -155,8 +153,6 @@ for INIT in openrc systemd; do
     check "server-join(${INIT}): enables webhook" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" eks-token-webhook)")"
     check "server-join(${INIT}): enables k3s" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s)")"
     check "server-join(${INIT}): enables state-report" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-state-report)")"
-    check "server-join(${INIT}): enables node-gc" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-node-gc)")"
-    check "server-join(${INIT}): starts node-gc" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-node-gc)")"
     check "server-join(${INIT}): enables konnectivity-server" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" konnectivity-server)")"
     check "server-join(${INIT}): enables k3s-recovery" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-k3s-recovery)")"
     check "server-join(${INIT}): starts k3s-recovery" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-k3s-recovery)")"
@@ -174,7 +170,7 @@ for INIT in openrc systemd; do
     check "agent(${INIT}): exit 0" 0 "${rc}"
     check "agent(${INIT}): role file" "agent" "$(cat "${ROLE_FILE}")"
     check "agent(${INIT}): enables k3s-agent" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
-    check "agent(${INIT}): no server services" "yes" "$(absent_all "${CALLS}" "eks-token-webhook" "k3s-first-boot" "mulga-eks-node-gc" "$(enable_cmd "${INIT}" k3s)")"
+    check "agent(${INIT}): no server services" "yes" "$(absent_all "${CALLS}" "eks-token-webhook" "k3s-first-boot" "$(enable_cmd "${INIT}" k3s)")"
 
     # --- agent (inferred from agent.env, no explicit role) ---
     run_case "${INIT}" agent_inferred

scripts/images/eks-node/manifest.conf

@@ -42,8 +42,6 @@ scripts/images/eks-node/mulga-eks-state-report.initd:/etc/init.d/mulga-eks-state
 scripts/images/eks-node/mulga-eks-state-report.sh:/usr/local/sbin/mulga-eks-state-report \
 scripts/images/eks-node/mulga-eks-addon-sync.initd:/etc/init.d/mulga-eks-addon-sync \
 scripts/images/eks-node/mulga-eks-addon-sync.sh:/usr/local/sbin/mulga-eks-addon-sync \
-scripts/images/eks-node/mulga-eks-node-gc.initd:/etc/init.d/mulga-eks-node-gc \
-scripts/images/eks-node/mulga-eks-node-gc.sh:/usr/local/sbin/mulga-eks-node-gc \
 scripts/images/eks-node/konnectivity-server.initd:/etc/init.d/konnectivity-server \
 scripts/images/eks-node/konnectivity-server-prestart.sh:/usr/local/sbin/konnectivity-server-prestart \
 scripts/images/eks-node/konnectivity/konnectivity-agent.yaml:/usr/share/spinifex-eks/konnectivity/konnectivity-agent.yaml \

scripts/images/eks-node/mulga-eks-node-gc.initd

@@ -1,27 +0,0 @@
-#!/sbin/openrc-run
-# mulga-eks-node-gc — server-role service that deletes the Node of every
-# nodegroup worker the nodegroup has terminated, every NODE_GC_INTERVAL
-# seconds. Enabled on every server role by eks-node-role.sh.
-
-description="Mulga EKS terminated-worker Node cleanup"
-command="/usr/local/sbin/mulga-eks-node-gc"
-command_background="yes"
-pidfile="/run/mulga-eks-node-gc.pid"
-output_log="/var/log/mulga-eks-node-gc.log"
-error_log="/var/log/mulga-eks-node-gc.log"
-
-# A Node goes NotReady about 40s after its worker stops, so 30s deletes it
-# within a couple of minutes of the terminate.
-export NODE_GC_INTERVAL="${NODE_GC_INTERVAL:-30}"
-
-depend() {
-    need net
-    after k3s
-}
-
-start_pre() {
-    if [ ! -f /etc/spinifex-eks/first-boot.env ]; then
-        eerror "/etc/spinifex-eks/first-boot.env not present — cloud-init did not seed gateway creds"
-        return 1
-    fi
-}

scripts/images/eks-node/mulga-eks-node-gc.sh

@@ -1,89 +0,0 @@
-#!/bin/sh
-set -eu
-
-# mulga-eks-node-gc — deletes the Node of a nodegroup worker whose instance the
-# nodegroup has terminated (DeleteNodegroup, scale-down, reclaim), as the AWS
-# cloud controller deletes a Node whose instance no longer exists. K3s runs no
-# AWS cloud provider: its own reports every instance as existing, so without
-# this a terminated worker stays a NotReady Node forever, its pods stay bound to
-# it, and once most Nodes are NotReady the node lifecycle controller stops
-# evicting at all.
-#
-# Pull (host→VM): GET /clusters/{cluster}/internal-workers/{accountId}, the
-# instance ID of every worker the cluster's nodegroups still own.
-#
-# A Node is deleted only when all of these hold, so a live worker is never
-# touched: it carries the eks.amazonaws.com/nodegroup label, its providerID is
-# aws:///<az>/<instance-id>, it is not Ready, and its instance is not in the
-# fetched set. A failed fetch deletes nothing.
-#
-# Runs on every server; deletes are idempotent. Loops every NODE_GC_INTERVAL
-# seconds in service mode, one-shot otherwise.
-
-ENVFILE=${ENVFILE:-/etc/spinifex-eks/first-boot.env}
-[ -f "${ENVFILE}" ] || { logger -t mulga-eks-node-gc "${ENVFILE} missing — exiting"; exit 0; }
-# set -a so the sourced creds are exported to the eks-gateway-fetch child.
-set -a
-# shellcheck disable=SC1090
-. "${ENVFILE}"
-set +a
-
-: "${EKS_GATEWAY_URL:?}"
-: "${EKS_ACCOUNT_ID:?}"
-: "${EKS_CLUSTER_NAME:?}"
-
-KUBECONFIG=${KUBECONFIG:-/etc/rancher/k3s/k3s.yaml}
-export KUBECONFIG
-
-# One "name|providerID|nodegroup|Ready" row per Node.
-NODE_ROWS='jsonpath={range .items[*]}{.metadata.name}{"|"}{.spec.providerID}{"|"}{.metadata.labels.eks\.amazonaws\.com/nodegroup}{"|"}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}'
-
-log() {
-    logger -t mulga-eks-node-gc "$*"
-    echo "[mulga-eks-node-gc] $*" > "${CONSOLE:-/dev/console}" 2>/dev/null || true
-}
-
-# stale_nodes LIVE_FILE — reads Node rows on stdin and prints the name of each
-# Node to delete: a nodegroup worker, not Ready, whose instance is not listed
-# in LIVE_FILE.
-stale_nodes() {
-    awk -F'|' -v live="$1" '
-        BEGIN { while ((getline id < live) > 0) keep[id] = 1 }
-        $3 == "" || $4 == "True" { next }
-        $2 !~ /^aws:\/\/\/[^\/]+\/i-[0-9a-f]+$/ { next }
-        { n = split($2, p, "/"); if (!(p[n] in keep)) print $1 }
-    '
-}
-
-sync_once() {
-    # Nodes are listed before the workers are fetched. A worker is recorded on
-    # its nodegroup before it boots, so every Node listed here is already in a
-    # set fetched after.
-    if ! rows=$(kubectl get nodes -o "${NODE_ROWS}" 2>/dev/null); then
-        log "kubectl get nodes failed; skipping this pass"
-        return 0
-    fi
-    live=$(mktemp)
-    if ! eks-gateway-fetch -resource workers > "${live}" 2>/dev/null; then
-        log "fetch of the cluster's nodegroup workers failed; deleting nothing"
-        rm -f "${live}"
-        return 0
-    fi
-    printf '%s\n' "${rows}" | stale_nodes "${live}" | while read -r node; do
-        if kubectl delete node "${node}" --ignore-not-found --wait=false >/dev/null 2>&1; then
-            log "deleted Node ${node}: its instance is no longer a nodegroup worker"
-        else
-            log "delete of Node ${node} failed; retrying next pass"
-        fi
-    done
-    rm -f "${live}"
-}
-
-if [ -n "${NODE_GC_INTERVAL:-}" ] && [ "${NODE_GC_INTERVAL}" -gt 0 ] 2>/dev/null; then
-    while true; do
-        sync_once
-        sleep "${NODE_GC_INTERVAL}"
-    done
-else
-    sync_once
-fi

scripts/images/eks-node/mulga-eks-node-gc_test.sh

@@ -1,107 +0,0 @@
-#!/bin/sh
-# Self-contained POSIX test for mulga-eks-node-gc.sh. kubectl, eks-gateway-fetch
-# and logger are stubbed on PATH: kubectl serves canned Node rows and records
-# every delete, and eks-gateway-fetch serves a canned worker set or fails.
-#
-# Run: sh scripts/images/eks-node/mulga-eks-node-gc_test.sh
-set -eu
-
-SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
-AGENT="${SCRIPT_DIR}/mulga-eks-node-gc.sh"
-WORK=$(mktemp -d)
-trap 'rm -rf "${WORK}"' EXIT
-
-STUBBIN="${WORK}/bin"
-mkdir -p "${STUBBIN}"
-
-cat > "${STUBBIN}/kubectl" <<'EOF'
-#!/bin/sh
-case "$1 $2" in
-    "get nodes")
-        [ "${NODES_RC:-0}" -eq 0 ] || exit "${NODES_RC}"
-        printf '%s\n' "${NODES_BODY:-}" ;;
-    "delete node") echo "$3" >> "${DELETED}" ;;
-esac
-EOF
-
-cat > "${STUBBIN}/eks-gateway-fetch" <<'EOF'
-#!/bin/sh
-echo "$*" >> "${FETCHED}"
-[ "${FETCH_RC:-0}" -eq 0 ] || exit "${FETCH_RC}"
-printf '%s' "${LIVE_BODY:-}"
-EOF
-
-cat > "${STUBBIN}/logger" <<'EOF'
-#!/bin/sh
-:
-EOF
-
-chmod +x "${STUBBIN}"/*
-PATH="${STUBBIN}:${PATH}"
-export PATH
-
-ENVFILE="${WORK}/first-boot.env"
-cat > "${ENVFILE}" <<'EOF'
-EKS_GATEWAY_URL=https://gw.invalid:9999
-EKS_ACCOUNT_ID=000000000001
-EKS_CLUSTER_NAME=demo3
-EOF
-CONSOLE="${WORK}/console"
-DELETED="${WORK}/deleted"
-FETCHED="${WORK}/fetched"
-export ENVFILE CONSOLE DELETED FETCHED
-
-PASS=0
-FAIL=0
-check() {
-    if [ "$2" = "$3" ]; then
-        PASS=$((PASS + 1))
-    else
-        FAIL=$((FAIL + 1))
-        printf 'FAIL: %s\n  want: %s\n  got:  %s\n' "$1" "$2" "$3"
-    fi
-}
-
-run_agent() {
-    : > "${DELETED}"
-    : > "${FETCHED}"
-    sh "${AGENT}"
-}
-
-deleted() { tr '\n' ' ' < "${DELETED}" | sed 's/ $//'; }
-
-# Every way a Node can look: only a NotReady nodegroup worker whose instance the
-# cluster's nodegroups no longer own is deleted.
-NODES_BODY='live|aws:///z1/i-0aaa|ng1|True
-gone|aws:///z1/i-0bbb|ng1|Unknown
-gone-never-reported|aws:///z1/i-0ddd|ng2|
-owned-notready|aws:///z1/i-0ccc|ng1|False
-gone-but-ready|aws:///z1/i-0eee|ng1|True
-self-managed|aws:///z1/i-0fff||Unknown
-k3s-provider|k3s://k3s-node|ng1|Unknown'
-LIVE_BODY='i-0aaa
-i-0ccc
-'
-export NODES_BODY LIVE_BODY
-
-run_agent
-check "deletes only NotReady nodegroup Nodes whose instance is gone" "gone gone-never-reported" "$(deleted)"
-check "fetches the workers resource" "-resource workers" "$(cat "${FETCHED}")"
-
-# A cluster with no nodegroups left: every NotReady nodegroup Node goes.
-LIVE_BODY='' run_agent
-check "empty worker set deletes every NotReady nodegroup Node" "gone gone-never-reported owned-notready" "$(deleted)"
-
-# A failed fetch is not an empty set: nothing is deleted.
-FETCH_RC=1 run_agent
-check "failed fetch deletes nothing" "" "$(deleted)"
-
-# Without a Node listing the workers are not fetched, so a set fetched before a
-# Node registered can never be compared against it.
-NODES_RC=1 run_agent
-check "failed node listing deletes nothing" "" "$(deleted)"
-check "failed node listing does not fetch" "" "$(cat "${FETCHED}")"
-
-echo "---"
-echo "PASS=${PASS} FAIL=${FAIL}"
-[ "${FAIL}" -eq 0 ]

scripts/images/eks-node/setup.sh

@@ -44,10 +44,9 @@ ln -sf /usr/local/bin/k3s /usr/local/bin/ctr
 # role's services are baked; the selector enables the right ones at first boot.
 chmod 0755 /etc/init.d/eks-node-role /etc/init.d/k3s /etc/init.d/k3s-agent \
     /etc/init.d/eks-token-webhook /etc/init.d/k3s-first-boot /etc/init.d/mulga-eks-state-report \
-    /etc/init.d/mulga-eks-addon-sync /etc/init.d/mulga-eks-node-gc /etc/init.d/konnectivity-server
+    /etc/init.d/mulga-eks-addon-sync /etc/init.d/konnectivity-server
 chmod 0755 /usr/local/sbin/eks-node-role /usr/local/sbin/k3s-first-boot \
-    /usr/local/sbin/mulga-eks-state-report /usr/local/sbin/mulga-eks-addon-sync \
-    /usr/local/sbin/mulga-eks-node-gc
+    /usr/local/sbin/mulga-eks-state-report /usr/local/sbin/mulga-eks-addon-sync
 # start_pre helpers shared with the systemd wrappers that will eventually
 # replace these runscripts.
 chmod 0755 /usr/local/sbin/k3s-prestart /usr/local/sbin/k3s-agent-prestart \

scripts/lib/sqs-store-guest_test.sh

@@ -1,293 +0,0 @@
-#!/bin/sh
-# sqs-store-guest_test.sh — the SQS store guest steps start the store only on
-# a valid contract and one checked, mounted disk, never format, and stop the
-# store before unmounting it.
-#
-# Drives build/microvm/sqs-store/sqs-store-lib.sh against a fake fw_cfg, sysfs
-# and stub tools. The stop cases signal only test-owned children.
-# The case settings below are read by the sourced library, not by this file.
-# shellcheck disable=SC2034
-set -eu
-
-ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/../.." && pwd)
-LIB=${SQS_GUEST_LIB:-"$ROOT/build/microvm/sqs-store/sqs-store-lib.sh"}
-TMP=$(mktemp -d)
-CHILDREN=""
-cleanup() {
-    for p in $CHILDREN; do kill -KILL "$p" 2>/dev/null || true; done
-    rm -rf "$TMP"
-}
-trap cleanup EXIT
-
-fails=0
-pass() { echo "  ok   - $1"; }
-fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
-check() { if eval "$2"; then pass "$1"; else fail "$1"; fi; }
-
-echo "-- build/microvm/sqs-store/sqs-store-lib.sh"
-
-SERIAL=vol0123456789abcdef0
-NKEY=UAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAB
-ADDR=10.11.12.13
-
-# Stub tools record each call to calls.log; their exit codes come from files.
-mkdir -p "$TMP/bin"
-cat > "$TMP/bin/e2fsck" <<'EOF'
-#!/bin/sh
-echo "e2fsck $*" >> "$T/calls.log"
-exit "$(cat "$T/fsck.rc")"
-EOF
-cat > "$TMP/bin/mount" <<'EOF'
-#!/bin/sh
-echo "mount $*" >> "$T/calls.log"
-exit "$(cat "$T/mount.rc")"
-EOF
-cat > "$TMP/bin/umount" <<'EOF'
-#!/bin/sh
-echo "umount $*" >> "$T/calls.log"
-EOF
-for f in mkfs mkfs.ext4 mke2fs; do
-    printf '#!/bin/sh\necho "mkfs %s $*" >> "$T/calls.log"\n' "$f" > "$TMP/bin/$f"
-done
-cat > "$TMP/bin/ip" <<'EOF'
-#!/bin/sh
-cat "$T/ip.out"
-EOF
-cat > "$TMP/bin/nats-server" <<'EOF'
-#!/bin/sh
-echo "nats-server $*" >> "$T/calls.log"
-[ "$1" = -t ] || echo "GOMEMLIMIT=${GOMEMLIMIT:-unset}" >> "$T/env.log"
-[ "$1" = -t ] && exit "$(cat "$T/configtest.rc")"
-exit 0
-EOF
-chmod +x "$TMP/bin"/*
-
-# fresh <name>: a case directory with a valid contract and one matching disk
-# among others. Callers then break one thing.
-fresh() {
-    T="$TMP/$1"
-    export T
-    mkdir -p "$T/fwcfg" "$T/sys" "$T/dev" "$T/run" "$T/data"
-    for k in data-serial listen-addr client-nkey; do mkdir -p "$T/fwcfg/$k"; done
-    printf '%s\n' "$SERIAL" > "$T/fwcfg/data-serial/raw"
-    printf '%s\n' "$ADDR" > "$T/fwcfg/listen-addr/raw"
-    printf '%s\n' "$NKEY" > "$T/fwcfg/client-nkey/raw"
-    disk vda vol00000000000000000
-    disk vdb "$SERIAL"
-    echo 0 > "$T/fsck.rc"
-    echo 0 > "$T/mount.rc"
-    echo 0 > "$T/configtest.rc"
-    : > "$T/calls.log"
-    : > "$T/console"
-    printf '1: lo    inet 127.0.0.1/8 scope host lo\n2: eth0    inet %s/24 brd 10.11.12.255 scope global eth0\n' "$ADDR" > "$T/ip.out"
-}
-disk() { mkdir -p "$T/sys/$1"; printf '%s\n' "$2" > "$T/sys/$1/serial"; : > "$T/dev/$1"; }
-
-# prepare: run sqs_prepare in a subshell with every path pointed at the case.
-prepare() {
-    (
-        SQS_FWCFG="$T/fwcfg" SQS_SYSBLOCK="$T/sys" SQS_DEVDIR="$T/dev" SQS_NODE_TEST=-e
-        SQS_RUN="$T/run" SQS_DATA="$T/data" SQS_CONSOLE="$T/console"
-        E2FSCK="$TMP/bin/e2fsck" MOUNT="$TMP/bin/mount" IP="$TMP/bin/ip" NATS_BIN="$TMP/bin/nats-server"
-        PATH="$TMP/bin:$PATH"
-        export PATH
-        # shellcheck source=../../build/microvm/sqs-store/sqs-store-lib.sh
-        . "$LIB"
-        sqs_prepare
-    ) > /dev/null 2>&1
-}
-refused() { [ ! -e "$T/run/ready" ] && grep -q FATAL "$T/console"; }
-never_mounted() { ! grep -q '^mount ' "$T/calls.log"; }
-never_formatted() { ! grep -q '^mkfs ' "$T/calls.log"; }
-never_checked() { ! grep -q '^e2fsck ' "$T/calls.log"; }
-
-# --- A valid contract prepares the store ---
-fresh valid
-check "valid contract is prepared" 'prepare && [ -e "$T/run/ready" ]'
-check "the matching disk is checked, then mounted at the data dir" \
-    'grep -qx "e2fsck -p $T/dev/vdb" "$T/calls.log" && grep -qx "mount -t ext4 -o nodev,nosuid,noexec $T/dev/vdb $T/data" "$T/calls.log"'
-check "config listens only on the supplied address" \
-    'grep -c "listen" "$T/run/nats.conf" | grep -qx 1 && grep -qx "listen: \"$ADDR:4222\"" "$T/run/nats.conf"'
-check "config admits only the supplied user NKey" 'grep -q "users = \[ { nkey: \"$NKEY\" } \]" "$T/run/nats.conf"'
-check "config syncs JetStream on the data disk" \
-    'grep -q "store_dir: \"$T/data\"" "$T/run/nats.conf" && grep -q "sync_interval: always" "$T/run/nats.conf"'
-check "config admits a full SQS message (2MiB max_payload)" \
-    'grep -c "max_payload" "$T/run/nats.conf" | grep -qx 1 && grep -qx "max_payload: 2MiB" "$T/run/nats.conf"'
-check "config is tested by the server before ready" 'grep -qx "nats-server -t -c $T/run/nats.conf" "$T/calls.log"'
-check "no auth material on the console" '! grep -q "$NKEY" "$T/console"'
-check "prepare reports the disk and config, not a listener" 'grep -q "prepared: $T/dev/vdb mounted" "$T/console" && ! grep -qi "listening" "$T/console"'
-
-fresh fsck-corrected; echo 1 > "$T/fsck.rc"
-check "e2fsck exit 1 (corrected) proceeds" 'prepare && [ -e "$T/run/ready" ]'
-
-# --- Disk refusals: nothing mounted, nothing formatted ---
-fresh no-disk; rm -rf "$T/sys/vdb"
-check "missing disk is refused before any check or mount" '! prepare && refused && never_checked && never_mounted'
-
-fresh wrong-disk; printf 'vol0fffffffffffffff0\n' > "$T/fwcfg/data-serial/raw"
-check "disk with another serial is refused" '! prepare && refused && never_checked && never_mounted'
-
-fresh two-disks; disk vdc "$SERIAL"
-check "two disks with the serial are refused" '! prepare && refused && never_checked && never_mounted'
-
-fresh bad-serial; printf 'vol-0123456789abcdef\n' > "$T/fwcfg/data-serial/raw"
-check "malformed serial is refused" '! prepare && refused && never_mounted'
-
-fresh no-node; rm -f "$T/dev/vdb"
-check "disk without a device node is refused" '! prepare && refused && never_mounted'
-
-for rc in 2 4 8; do
-    fresh "fsck-$rc"; echo "$rc" > "$T/fsck.rc"
-    check "e2fsck exit $rc is refused without mount or format" '! prepare && refused && never_mounted && never_formatted'
-done
-
-fresh mount-fail; echo 32 > "$T/mount.rc"
-check "failed mount is refused" '! prepare && refused'
-
-# --- Auth and listener refusals happen before the disk is touched ---
-fresh no-nkey; rm -rf "$T/fwcfg/client-nkey"
-check "missing NKey is refused before the disk" '! prepare && refused && never_checked && never_mounted'
-
-for bad in seed lower short two empty; do
-    fresh "nkey-$bad"
-    case $bad in
-        seed) v=SUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA ;;
-        lower) v=uaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab ;;
-        short) v=UAAAA ;;
-        two) v="$NKEY
-$NKEY" ;;
-        empty) v="" ;;
-    esac
-    printf '%s\n' "$v" > "$T/fwcfg/client-nkey/raw"
-    check "NKey ($bad) is refused before the disk, without echoing it" \
-        '! prepare && refused && never_checked && { [ -z "$v" ] || ! grep -qF "$v" "$T/console"; }'
-done
-
-fresh nkey-checksum; echo 1 > "$T/configtest.rc"
-check "NKey the server rejects is refused before the disk" '! prepare && refused && never_checked && never_mounted'
-
-fresh no-addr; rm -rf "$T/fwcfg/listen-addr"
-check "missing listen address is refused" '! prepare && refused && never_mounted'
-
-fresh addr-absent; printf '1: lo    inet 127.0.0.1/8 scope host lo\n2: eth0    inet 10.11.12.99/24 scope global eth0\n' > "$T/ip.out"
-check "listen address not on a guest NIC is refused" '! prepare && refused && never_mounted'
-
-fresh addr-loopback; printf '127.0.0.1\n' > "$T/fwcfg/listen-addr/raw"
-check "loopback listen address is refused" '! prepare && refused && never_mounted'
-
-for bad in 0.0.0.0x 1.2.3 1.2.3.256 01.2.3.4; do
-    fresh "addr-$bad"; printf '%s\n' "$bad" > "$T/fwcfg/listen-addr/raw"
-    check "malformed listen address $bad is refused" '! prepare && refused && never_mounted'
-done
-
-# --- Run: the store starts only when prepared and not stopping ---
-run_step() {
-    (
-        SQS_RUN="$T/run" SQS_CONSOLE="$T/console" NATS_BIN="$TMP/bin/nats-server"
-        # shellcheck source=../../build/microvm/sqs-store/sqs-store-lib.sh
-        . "$LIB"
-        sqs_run
-    ) > /dev/null 2>&1
-}
-fresh run-ready; prepare; : > "$T/calls.log"
-check "a prepared store starts the server" 'run_step && grep -qx "nats-server -c $T/run/nats.conf" "$T/calls.log"'
-fresh run-memlimit; prepare; : > "$T/env.log"
-check "the server runs under the ADR-0011:S11 heap limit" 'run_step && grep -qx "GOMEMLIMIT=512MiB" "$T/env.log"'
-fresh run-unready
-check "an unprepared store never starts the server" '! run_step && ! grep -q "^nats-server" "$T/calls.log"'
-fresh run-stopping; prepare; : > "$T/calls.log"; : > "$T/run/stopping"
-check "a stopping store never starts the server" '! run_step && ! grep -q "^nats-server" "$T/calls.log"'
-
-# --- Stop: SIGTERM the verified store, then unmount ---
-STORE="$TMP/bin/fake-store"
-cp "$(command -v sleep)" "$STORE"
-stop_step() {
-    (
-        SQS_RUN="$T/run" SQS_DATA="$T/data" SQS_CONSOLE="$T/console" SQS_MOUNTS="$T/mounts"
-        UMOUNT="$TMP/bin/umount" NATS_BIN="$STORE" SQS_STOP_BUDGET_TENTHS=10
-        # shellcheck source=../../build/microvm/sqs-store/sqs-store-lib.sh
-        . "$LIB"
-        sqs_stop
-    ) > /dev/null 2>&1
-}
-gone() { [ ! -r "/proc/$1/status" ] || grep -q '^State:[[:space:]]*Z' "/proc/$1/status"; }
-# record <pid>: write the identity sqs_run would record for that process.
-# shellcheck source=../../build/microvm/sqs-store/sqs-store-lib.sh
-record() { echo "$1 $( . "$LIB"; sqs_starttime "$1" )" > "$T/run/pid"; }
-
-fresh stop-orderly; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
-"$STORE" 60 & pid=$!; CHILDREN="$CHILDREN $pid"; record "$pid"
-stop_step
-check "stop sends SIGTERM and the store exits" 'gone "$pid" && grep -q "SIGTERM sent" "$T/console" && ! grep -q SIGKILL "$T/console"'
-check "stop blocks respawn and unmounts after the store exits" \
-    '[ -e "$T/run/stopping" ] && grep -qx "umount $T/data" "$T/calls.log"'
-
-fresh stop-hung; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
-(trap '' TERM; exec "$STORE" 60) & pid=$!; CHILDREN="$CHILDREN $pid"; record "$pid"
-stop_step
-check "store ignoring SIGTERM is killed after the budget, then unmounted" \
-    'gone "$pid" && grep -q SIGKILL "$T/console" && grep -qx "umount $T/data" "$T/calls.log"'
-
-fresh stop-reused; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
-"$STORE" 60 & pid=$!; CHILDREN="$CHILDREN $pid"; echo "$pid 1" > "$T/run/pid"
-stop_step
-check "a recorded PID with another start time is never signalled" '! gone "$pid" && grep -q "no running store" "$T/console"'
-kill -KILL "$pid"
-
-# A start held after its stopping check and before its exec: the run step
-# blocks writing "run: starting" to a FIFO console until the test reads it.
-fresh stop-pending-start; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
-: > "$T/run/ready"
-mkfifo "$T/run-console"
-printf '#!/bin/sh\n: > "%s/started"\nexec "%s" 60\n' "$T" "$STORE" > "$TMP/bin/pending-nats"
-chmod +x "$TMP/bin/pending-nats"
-SQS_RUN="$T/run" SQS_CONSOLE="$T/run-console" NATS_BIN="$TMP/bin/pending-nats" \
-    sh -c '. "$1"; sqs_run' sh "$LIB" & pid=$!; CHILDREN="$CHILDREN $pid"
-i=0
-while [ "$i" -lt 100 ] && ! grep -q "^$pid " "$T/run/pid" 2>/dev/null; do i=$((i + 1)); sleep 0.05; done
-check "pending start is held before its exec" 'grep -q "^$pid " "$T/run/pid" && ! gone "$pid" && [ ! -e "$T/started" ]'
-stop_step
-# Release the FIFO: a start that survived the stop would exec the server now.
-cat "$T/run-console" > /dev/null 2>&1 &
-reader=$!; CHILDREN="$CHILDREN $reader"
-i=0
-while [ "$i" -lt 20 ] && ! gone "$pid"; do i=$((i + 1)); sleep 0.05; done
-check "stop ends a pending start before it can exec the server" 'gone "$pid" && [ ! -e "$T/started" ]'
-check "stop unmounts only after the pending start is gone" \
-    'grep -q "store exited" "$T/console" && grep -qx "umount $T/data" "$T/calls.log"'
-kill -KILL "$reader" 2>/dev/null || true
-
-# --- Shutdown: the generic remount runs only once the store is gone ---
-# shutdown_step [survivor-pid]: with a pid, that process is reported alive
-# whatever happens to it, as a writer stuck past SIGKILL would be.
-shutdown_step() {
-    (
-        SQS_RUN="$T/run" SQS_DATA="$T/data" SQS_CONSOLE="$T/console" SQS_MOUNTS="$T/mounts"
-        UMOUNT="$TMP/bin/umount" SQS_STOP_BUDGET_TENTHS=5
-        # shellcheck source=../../build/microvm/sqs-store/sqs-store-lib.sh
-        . "$LIB"
-        if [ -n "${1:-}" ]; then sqs_same() { [ "$1" = "$survivor" ]; }; survivor=$1; fi
-        sqs_shutdown
-    ) > /dev/null 2>&1
-}
-
-fresh shutdown-orderly; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
-"$STORE" 60 & pid=$!; CHILDREN="$CHILDREN $pid"; record "$pid"
-check "orderly shutdown succeeds" 'shutdown_step'
-check "orderly shutdown unmounts the data disk, then runs umount -a -r" \
-    '[ "$(grep "^umount" "$T/calls.log" | tr "\n" "|")" = "umount $T/data|umount -a -r|" ]'
-
-fresh shutdown-survivor; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
-"$STORE" 60 & pid=$!; CHILDREN="$CHILDREN $pid"; record "$pid"
-check "shutdown with a surviving store reports failure" '! shutdown_step "$pid"'
-check "a surviving store leaves the data disk and every other mount alone" \
-    '! grep -q "^umount" "$T/calls.log" && grep -q "STILL ALIVE" "$T/console" && grep -q "skipping umount -a -r" "$T/console"'
-
-fresh stop-unmounted; : > "$T/mounts"
-stop_step
-check "stop with no data mount does not unmount" '! grep -q "^umount" "$T/calls.log"'
-
-if [ "$fails" -ne 0 ]; then
-    echo "  $fails failure(s)"
-    exit 1
-fi

scripts/lib/sqs-store-microvm-publish.sh

@@ -11,13 +11,11 @@ STAGE_DIR="${1:?usage: sqs-store-microvm-publish.sh STAGE_DIR OUT_DIR}"
 OUT_DIR="${2:?usage: sqs-store-microvm-publish.sh STAGE_DIR OUT_DIR}"
 VMLINUZ="$STAGE_DIR/vmlinuz"
 INITRAMFS="$STAGE_DIR/initramfs.cpio.gz"
-PROFILE="${3:-store}"
+PROFILE="${3:-worker}"
 case "$PROFILE" in
-    store)
-        payload="usr/local/bin/nats-server usr/local/lib/sqs-store/sqs-store-lib.sh usr/local/lib/sqs-store/sqs-store-prepare usr/local/lib/sqs-store/sqs-store-run usr/local/lib/sqs-store/sqs-store-stop" ;;
     worker)
-        payload="usr/local/bin/sqs-worker usr/local/lib/sqs-worker/sqs-worker-lib.sh usr/local/lib/sqs-worker/sqs-worker-prepare usr/local/lib/sqs-worker/sqs-worker-ready usr/local/lib/sqs-worker/sqs-worker-stop data" ;;
-    *) echo "ERROR: profile must be store or worker, not $PROFILE" >&2; exit 1 ;;
+        payload="usr/local/bin/nats-server usr/local/bin/sqs-worker usr/local/lib/sqs-worker/sqs-worker-lib.sh usr/local/lib/sqs-worker/sqs-worker-prepare usr/local/lib/sqs-worker/sqs-worker-store usr/local/lib/sqs-worker/sqs-worker-ready usr/local/lib/sqs-worker/sqs-worker-stop data" ;;
+    *) echo "ERROR: profile must be worker, not $PROFILE" >&2; exit 1 ;;
 esac
 tag="[build-sqs-$PROFILE-microvm-image]"
 
@@ -56,12 +54,7 @@ for want_ko in ext4 tiny_power_button; do
     fi
 done
 if grep -qE '(^|/)(lb-agent|haproxy|nginx)$' <<< "$archive_list"; then
-    echo "ERROR: LB data-plane binaries found in the SQS store image" >&2
-    exit 1
-fi
-# The worker holds the store client key; it must never carry the store server.
-if [ "$PROFILE" = worker ] && grep -qE '(^|/)nats-server$' <<< "$archive_list"; then
-    echo "ERROR: nats-server found in the SQS worker image" >&2
+    echo "ERROR: LB data-plane binaries found in the SQS image" >&2
     exit 1
 fi
 echo "$tag content gate: PASS"

scripts/lib/sqs-store-microvm-publish_test.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-# sqs-store-microvm-publish_test.sh — a staged SQS store microVM pair reaches
+# sqs-store-microvm-publish_test.sh — a staged SQS microVM pair reaches
 # the output directory only when every gate passes, and a failed build cannot
 # leave a pair that make then takes as up to date.
 #
@@ -28,9 +28,10 @@ done
 stage() {
     dir=$1 omit=${2:-}
     root="$TMP/root.$$.$(date +%s%N)"
-    for p in init etc/inittab usr/local/bin/nats-server sbin/mkfs.ext4 sbin/e2fsck \
-        usr/local/lib/sqs-store/sqs-store-lib.sh usr/local/lib/sqs-store/sqs-store-prepare \
-        usr/local/lib/sqs-store/sqs-store-run usr/local/lib/sqs-store/sqs-store-stop \
+    for p in init etc/inittab usr/local/bin/nats-server usr/local/bin/sqs-worker data sbin/mkfs.ext4 sbin/e2fsck \
+        usr/local/lib/sqs-worker/sqs-worker-lib.sh usr/local/lib/sqs-worker/sqs-worker-prepare \
+        usr/local/lib/sqs-worker/sqs-worker-store usr/local/lib/sqs-worker/sqs-worker-ready \
+        usr/local/lib/sqs-worker/sqs-worker-stop \
         dev/console dev/null lib/modules/6.12-virt/kernel/fs/ext4/ext4.ko \
         lib/modules/6.12-virt/kernel/drivers/acpi/tiny-power-button.ko; do
         [ "$p" = "$omit" ] && continue
@@ -78,7 +79,7 @@ else
 fi
 
 # --- Each failing gate keeps the previous valid pair exactly ---
-for case in nats-server store-stop ext4 size; do
+for case in nats-server worker-store ext4 size; do
     d="$TMP/keep-$case"
     stage "$d/prev"
     bash "$PUBLISH" "$d/prev" "$d/out" >/dev/null 2>&1
@@ -86,7 +87,7 @@ for case in nats-server store-stop ext4 size; do
     max=50
     case $case in
         nats-server) stage "$d/stage" usr/local/bin/nats-server ;;
-        store-stop) stage "$d/stage" usr/local/lib/sqs-store/sqs-store-stop ;;
+        worker-store) stage "$d/stage" usr/local/lib/sqs-worker/sqs-worker-store ;;
         ext4) stage "$d/stage" lib/modules/6.12-virt/kernel/fs/ext4/ext4.ko ;;
         size) stage "$d/stage"; max=0 ;;
     esac

scripts/lib/sqs-worker-guest_test.sh

@@ -1,7 +1,8 @@
 #!/bin/sh
-# sqs-worker-guest_test.sh — the SQS worker guest's readiness watcher is
-# supervised by init, never leaves a store-ready marker behind a dead watcher,
-# and shutdown unmounts the worker disk only once the watcher is gone.
+# sqs-worker-guest_test.sh — the SQS guest's store starts only on a config
+# prepared after the receipt key settled, its readiness watcher never leaves a
+# store-ready marker behind a dead watcher, and shutdown unmounts the data disk
+# only once both the watcher and the store are gone.
 #
 # Drives build/microvm/sqs-worker/sqs-worker-lib.sh with test-owned processes
 # and a stub umount. Every signal goes only to this test's own children.
@@ -41,26 +42,56 @@ echo "$$" >> "$T/watchers"
 echo "10.0.0.5 UPUB" > "$T/run/store-ready"
 exec sleep 60
 EOF
+# The fake server checks its config, or notes how it was started and stays.
+cat > "$TMP/bin/nats" <<'EOF'
+#!/bin/sh
+if [ "$1" = -t ]; then [ -z "${NATS_BAD:-}" ]; exit; fi
+echo "$GOMEMLIMIT $*" >> "$T/servers"
+exec sleep 60
+EOF
+# The fake boot takes its exit codes in order from $T/boot-rcs, and like the
+# real one writes the receipt marker only on success; "hold" blocks like a slow call.
+cat > "$TMP/bin/boot" <<'EOF'
+#!/bin/sh
+echo "$*" >> "$T/boots"
+rc=$(head -n1 "$T/boot-rcs"); sed -i 1d "$T/boot-rcs"
+if [ "$rc" = hold ]; then echo "$$" > "$T/boot-child"; exec sleep 60; fi
+[ "$rc" != 0 ] || echo FP > "$T/run/receipt-ready"
+exit "$rc"
+EOF
 chmod +x "$TMP/bin"/*
 
 fresh() {
     T="$TMP/$1"
     export T
     mkdir -p "$T/run" "$T/data"
-    echo UPUBLICKEY > "$T/run/public-key"
+    echo FP > "$T/run/receipt-ready"
+    echo "listen: 127.0.0.1:4222" > "$T/run/store.conf"
     for f in calls.log console mounts; do : > "$T/$f"; done
 }
 gone() { [ ! -r "/proc/$1/status" ] || grep -q '^State:[[:space:]]*Z' "/proc/$1/status" 2>/dev/null; }
 waitfor() { i=0; while [ "$i" -lt 100 ] && ! eval "$1"; do i=$((i + 1)); sleep 0.05; done; eval "$1"; }
 # start_run: one init respawn of the readiness step, in the background.
 start_run() {
-    SQSW_RUN="$T/run" SQSW_CONSOLE="$T/console" SQSW_BIN="$TMP/bin/watcher" SQSW_RESPAWN_GAP="${GAP:-0}" SQSW_PARK_INTERVAL="${PARK:-30}" \
+    SQSW_RUN="$T/run" SQSW_CONSOLE="$T/console" SQSW_BIN="$TMP/bin/watcher" SQSW_RESPAWN_GAP="${GAP:-0}" SQSW_PARK="sleep 60" \
         sh -c '. "$1"; sqsw_run' sh "$LIB" & pid=$!
     CHILDREN="$CHILDREN $pid"
 }
+# start_store: one init respawn of the store step, in the background.
+start_store() {
+    SQSW_RUN="$T/run" SQSW_DATA="$T/data" SQSW_CONSOLE="$T/console" SQSW_NATS_BIN="$TMP/bin/nats" \
+        SQSW_BIN="$TMP/bin/boot" SQSW_BOOT_RETRY=0 SQSW_PARK="sleep 60" SQSW_LIB="$LIB" \
+        sh -c '. "$1"; sqsw_store' sh "$LIB" & spid=$!
+    CHILDREN="$CHILDREN $spid"
+}
+# prepare_store: the sysinit step's store preparation, after boot.
+prepare_store() {
+    SQSW_RUN="$T/run" SQSW_DATA="$T/data" SQSW_CONSOLE="$T/console" SQSW_NATS_BIN="$TMP/bin/nats" \
+        sh -c '. "$1"; sqsw_prepare_store' sh "$LIB"
+}
 # record <pid>: write the identity sqs_run would record for that process.
 # shellcheck source=../../build/microvm/sqs-worker/sqs-worker-lib.sh
-record() { echo "$1 $( . "$LIB"; sqsw_starttime "$1" )" > "$T/run/ready.pid"; }
+record() { echo "$1 $( . "$LIB"; sqsw_starttime "$1" )" > "$T/run/${2:-ready}.pid"; }
 # shutdown_step [survivor-pid]: with a pid, that process is reported alive
 # whatever happens to it, as a watcher stuck past SIGKILL would be.
 shutdown_step() {
@@ -74,8 +105,81 @@ shutdown_step() {
     ) > /dev/null 2>&1
 }
 
-check "init respawns the readiness step rather than running it once" \
-    'grep -qx "::respawn:/usr/local/lib/sqs-worker/sqs-worker-ready" "$INITTAB" && ! grep -q "::once:" "$INITTAB"'
+check "init respawns the store and readiness steps rather than running them once" \
+    'grep -qx "::respawn:/usr/local/lib/sqs-worker/sqs-worker-store" "$INITTAB" &&
+     grep -qx "::respawn:/usr/local/lib/sqs-worker/sqs-worker-ready" "$INITTAB" && ! grep -q "::once:" "$INITTAB"'
+
+# --- The store is prepared only after boot settled the receipt key ---
+fresh prepare-ok; rm -f "$T/run/store.conf"
+check "a settled receipt key prepares the store" 'prepare_store && [ -s "$T/run/store.conf" ]'
+check "the store listens only on loopback, with no auth, monitor, route or cluster listener" \
+    'grep -qx "listen: \"127.0.0.1:4222\"" "$T/run/store.conf" && grep -qx "  store_dir: \"$T/data/jetstream\"" "$T/run/store.conf" &&
+     ! grep -Eq "authorization|http|monitor|cluster|routes|leafnodes|websocket|mqtt" "$T/run/store.conf"'
+fresh prepare-no-key; rm -f "$T/run/store.conf" "$T/run/receipt-ready"
+check "a receipt key boot did not settle leaves the store unprepared" '! prepare_store && [ ! -e "$T/run/store.conf" ]'
+fresh prepare-bad; rm -f "$T/run/store.conf"
+check "a config the server rejects leaves the store unprepared" '! NATS_BAD=1 prepare_store && [ ! -e "$T/run/store.conf" ]'
+
+# --- The store step starts the server only on a prepared config ---
+fresh store-run
+start_store
+check "a prepared store starts the dedicated server with its heap bound" \
+    'waitfor "[ -s \"\$T/servers\" ]" && grep -qx "512MiB -c $T/run/store.conf" "$T/servers"'
+kill -KILL "$spid" 2>/dev/null || true
+fresh store-refused; : > "$T/run/boot-refused"
+start_store; waitfor "grep -q 'store: boot refused' \"\$T/console\"" || true
+check "a store whose boot was refused parks instead of starting or respawning" \
+    '! gone "$spid" && [ ! -e "$T/servers" ] && [ ! -e "$T/boots" ]'
+kill -KILL "$spid" 2>/dev/null || true
+
+# --- Boot is supervised: a transient failure is retried, a refusal is final ---
+fresh boot-transient; rm -f "$T/run/store.conf" "$T/run/receipt-ready"; printf '75\n0\n' > "$T/boot-rcs"
+start_store
+check "a transient boot failure exits so init runs the step again, starting nothing" \
+    'waitfor "gone $spid" && [ ! -e "$T/servers" ] && [ ! -e "$T/run/boot-refused" ]'
+start_store
+check "the respawn after recovery boots again and starts the store" \
+    'waitfor "[ -s \"\$T/servers\" ]" && [ "$(wc -l < "$T/boots")" -eq 2 ] && [ -s "$T/run/store.conf" ]'
+kill -KILL "$spid" 2>/dev/null || true
+fresh boot-final; rm -f "$T/run/store.conf" "$T/run/receipt-ready"; printf '1\n0\n' > "$T/boot-rcs"
+start_store; waitfor "grep -q 'boot refused (rc=1)' \"\$T/console\"" || true
+check "a final boot refusal parks the store and records the refusal" \
+    '! gone "$spid" && [ -e "$T/run/boot-refused" ] && [ ! -e "$T/servers" ]'
+kill -KILL "$spid" 2>/dev/null || true
+start_store; waitfor "grep -q 'store: boot refused, not starting' \"\$T/console\"" || true
+check "a later respawn never boots again after a final refusal" '[ "$(wc -l < "$T/boots")" -eq 1 ]'
+kill -KILL "$spid" 2>/dev/null || true
+fresh boot-settled; rm -f "$T/run/store.conf"; : > "$T/boot-rcs"
+start_store
+check "a respawn after the key settled starts the store without booting again" \
+    'waitfor "[ -s \"\$T/servers\" ]" && [ ! -e "$T/boots" ]'
+kill -KILL "$spid" 2>/dev/null || true
+fresh boot-held; rm -f "$T/run/store.conf" "$T/run/receipt-ready"; echo hold > "$T/boot-rcs"
+echo "/dev/vdb $T/data ext4 rw 0 0" > "$T/mounts"
+start_store; waitfor '[ -s "$T/boot-child" ]' || true
+bchild=$(cat "$T/boot-child" 2>/dev/null); CHILDREN="$CHILDREN $bchild"
+shutdown_step
+check "shutdown stops a boot still in progress before it unmounts the disk" \
+    'waitfor "gone $bchild" && grep -q "umount $T/data" "$T/calls.log"'
+kill -KILL "$spid" "$bchild" 2>/dev/null || true
+fresh boot-held-survivor; rm -f "$T/run/store.conf" "$T/run/receipt-ready"; echo hold > "$T/boot-rcs"
+echo "/dev/vdb $T/data ext4 rw 0 0" > "$T/mounts"
+start_store; waitfor '[ -s "$T/boot-child" ]' || true
+bchild=$(cat "$T/boot-child" 2>/dev/null); CHILDREN="$CHILDREN $bchild"
+check "a boot that survives SIGKILL fails shutdown and keeps the disk mounted" \
+    '! shutdown_step "$bchild" && ! grep -q umount "$T/calls.log" && grep -q "boot pid=$bchild STILL ALIVE" "$T/console"'
+kill -KILL "$spid" "$bchild" 2>/dev/null || true
+# Shutdown sets stopping after the store's own check but before the boot child
+# runs: the child must still record itself and never start sqs-worker boot.
+fresh boot-window; rm -f "$T/run/store.conf" "$T/run/receipt-ready"; echo 0 > "$T/boot-rcs"
+: > "$T/run/stopping"
+SQSW_RUN="$T/run" SQSW_CONSOLE="$T/console" SQSW_BIN="$TMP/bin/boot" SQSW_BOOT_RETRY=0 SQSW_PARK="sleep 60" SQSW_LIB="$LIB" \
+    sh -c '. "$1"; sqsw_boot' sh "$LIB" & wpid=$!
+CHILDREN="$CHILDREN $wpid"
+waitfor "grep -q 'shutdown during boot' \"\$T/console\"" || true
+check "a boot child started after shutdown began never runs boot" \
+    'grep -q "shutdown during boot" "$T/console" && [ ! -e "$T/boots" ] && [ ! -e "$T/run/boot-refused" ]'
+kill -KILL "$wpid" 2>/dev/null || true
 
 # --- A watcher killed after readiness: its marker must not outlive it ---
 fresh killed-after-ready
@@ -103,8 +207,7 @@ kill -KILL "$pid" 2>/dev/null || true
 # A run that cannot start parks rather than exiting, so init does not respawn
 # it in a loop; a second is many park intervals, and it must log only once.
 fresh run-stopping; : > "$T/run/stopping"; echo stale > "$T/run/store-ready"
-PARK=0.05 start_run; waitfor "grep -q 'shutdown in progress' \"\$T/console\"" || true
-sleep 1
+start_run; waitfor "grep -q 'shutdown in progress' \"\$T/console\"" || true
 check "a start during shutdown parks instead of exiting and respawning" '! gone "$pid"'
 check "a parked start during shutdown starts no watcher and removes the marker" \
     '[ ! -e "$T/watchers" ] && [ ! -e "$T/run/store-ready" ]'
@@ -112,26 +215,24 @@ check "a parked start during shutdown logs once" '[ "$(grep -c "shutdown in prog
 shutdown_step
 check "shutdown stops the parked start" 'waitfor "gone $pid"'
 
-fresh run-no-key; rm -f "$T/run/public-key"
-PARK=0.05 start_run; waitfor "grep -q 'no worker key' \"\$T/console\"" || true
-sleep 1
-check "a guest without a worker key parks instead of exiting and respawning" '! gone "$pid" && [ ! -e "$T/watchers" ]'
-check "a parked guest without a worker key logs once" '[ "$(grep -c "no worker key" "$T/console")" -eq 1 ]'
-echo UPUBLICKEY > "$T/run/public-key"
-check "a key that appears later starts the watcher" 'waitfor "[ -s \"\$T/watchers\" ]"'
-kill -KILL "$pid" 2>/dev/null || true
-
-fresh run-no-key-stop; rm -f "$T/run/public-key"
-PARK=0.05 start_run; waitfor "grep -q 'no worker key' \"\$T/console\"" || true
+fresh run-unprepared; rm -f "$T/run/store.conf"
+start_run
+check "a watcher whose store is not prepared yet exits to be respawned and never reports ready" \
+    'waitfor "gone $pid" && [ ! -e "$T/watchers" ] && [ ! -e "$T/run/store-ready" ]'
+fresh run-refused; rm -f "$T/run/store.conf"; : > "$T/run/boot-refused"
+start_run; waitfor "grep -q 'ready: boot refused' \"\$T/console\"" || true
+check "a guest whose boot was refused parks its watcher and never reports ready" \
+    '! gone "$pid" && [ ! -e "$T/watchers" ] && [ ! -e "$T/run/store-ready" ]'
 shutdown_step
-check "shutdown stops a start parked for a missing key" 'waitfor "gone $pid" && [ ! -e "$T/watchers" ]'
+check "shutdown stops a watcher parked after a refused boot" 'waitfor "gone $pid" && [ ! -e "$T/watchers" ]'
 
-# --- Shutdown: unmount only once the watcher is confirmed gone ---
+# --- Shutdown: unmount only once the watcher and the store are gone ---
 fresh shutdown-orderly; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
-start_run; waitfor "[ -s \"\$T/run/store-ready\" ]"
+start_store; start_run; waitfor "[ -s \"\$T/run/store-ready\" ]"; waitfor "[ -s \"\$T/servers\" ]"
 check "orderly shutdown succeeds" 'shutdown_step'
-check "orderly shutdown confirms the watcher exited, removes the marker, then unmounts" \
-    'gone "$pid" && grep -q "readiness watcher pid=$pid exited" "$T/console" && [ ! -e "$T/run/store-ready" ] &&
+check "orderly shutdown stops the watcher, then the store, removes the marker, then unmounts" \
+    'gone "$pid" && gone "$spid" && [ ! -e "$T/run/store-ready" ] &&
+     [ "$(grep "exited" "$T/console" | sed "s/.*stop: \([a-z]*\) pid.*/\1/" | tr "\n" "|")" = "ready|store|" ] &&
      [ "$(grep "^umount" "$T/calls.log" | tr "\n" "|")" = "umount $T/data|umount -a -r|" ]'
 check "shutdown blocks any later respawn" '[ -e "$T/run/stopping" ]'
 
@@ -151,10 +252,16 @@ check "a surviving watcher leaves every mount alone and is never reported stoppe
 check "a surviving watcher still leaves no readiness marker" '[ ! -e "$T/run/store-ready" ]'
 kill -KILL "$pid" 2>/dev/null || true
 
+fresh shutdown-store-survivor; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
+(trap '' TERM; exec sleep 60) & pid=$!; CHILDREN="$CHILDREN $pid"; record "$pid" store
+check "shutdown with a surviving store reports failure and leaves every mount alone" \
+    '! shutdown_step "$pid" && ! grep -q "^umount" "$T/calls.log" && grep -q "store pid=$pid STILL ALIVE" "$T/console"'
+kill -KILL "$pid" 2>/dev/null || true
+
 fresh shutdown-reused; echo "dev $T/data ext4 rw 0 0" > "$T/mounts"
 (exec sleep 60) & pid=$!; CHILDREN="$CHILDREN $pid"; echo "$pid 1" > "$T/run/ready.pid"
 shutdown_step
-check "a recorded PID with another start time is never signalled" '! gone "$pid" && grep -q "no running readiness watcher" "$T/console"'
+check "a recorded PID with another start time is never signalled" '! gone "$pid" && grep -q "no running ready" "$T/console"'
 kill -KILL "$pid" 2>/dev/null || true
 
 if [ "$fails" -ne 0 ]; then

scripts/lib/stage-sqs-store-microvm.sh

@@ -1,11 +1,10 @@
 #!/bin/sh
-# stage-sqs-store-microvm.sh <image-dir> <dist-root> [sqs-store|sqs-worker]:
-# copy a built SQS microVM image into a release tree as <dist-root>/microvm-<name>.
+# stage-sqs-store-microvm.sh <image-dir> <dist-root> [sqs-worker]: copy the
+# built SQS microVM image into a release tree as <dist-root>/microvm-sqs-worker.
 # Without a complete image it stages nothing and says so, as the LB image does.
 set -eu
-src=$1 name=${3:-sqs-store}
+src=$1 name=${3:-sqs-worker}
 case "$name" in
-    sqs-store) label="SQS store" ;;
     sqs-worker) label="SQS worker" ;;
     *) echo "[distro] unknown SQS microVM image '$name'" >&2; exit 2 ;;
 esac

scripts/setup.sh

@@ -1014,9 +1014,6 @@ RESOLV_CONF="${SPINIFEX_RESOLV_CONF:-/etc/resolv.conf}"
 RESOLVED_STUB=/run/systemd/resolve/stub-resolv.conf
 # node-reset.sh keeps /var/lib/spinifex-reset, so the copy survives until uninstall.
 RESOLV_CONF_SAVED="${SPINIFEX_RESOLV_CONF_SAVED:-/var/lib/spinifex-reset/resolv.conf.orig}"
-# NetworkManager started while resolv.conf is a plain file rewrites it on its
-# next DNS update, replacing the stub link; this drop-in stops that.
-NM_DNS_DROPIN="${SPINIFEX_NM_DNS_DROPIN:-/etc/NetworkManager/conf.d/90-spinifex-dns.conf}"
 
 # systemd-resolved ships resolvconf as a link to resolvectl, which only works
 # while resolved runs, so that one is not a resolvconf backend.
@@ -1069,52 +1066,10 @@ dns_handover_failed() {
     fatal "Host DNS through systemd-resolved does not work: $1.
     To put host DNS back as it was before this installer:
       sudo systemctl disable --now systemd-resolved
-      sudo rm -f $NM_DNS_DROPIN
       sudo cp -a --remove-destination $RESOLV_CONF_SAVED $RESOLV_CONF
       sudo systemctl restart NetworkManager"
 }
 
-# Prints the rc-manager NetworkManager would use if it is not unmanaged, which
-# means a file read after the Spinifex drop-in overrides it.
-nm_rc_manager_override() {
-    local config
-    config=$($SUDO NetworkManager --print-config) || return 1
-    printf '%s\n' "$config" | awk -F= '
-        /^\[/ { main = ($0 == "[main]"); next }
-        main && $1 == "rc-manager" { rc = $2 }
-        END { if (rc != "unmanaged") print "rc-manager=" (rc == "" ? "symlink" : rc) }'
-}
-
-# Writes the NetworkManager drop-in and checks nothing overrides it. Returns 1
-# when it was already in place.
-install_nm_dns_dropin() {
-    local want override changed=1
-    want=$(printf '[main]\ndns=systemd-resolved\nrc-manager=unmanaged')
-    if [ "$(cat "$NM_DNS_DROPIN" 2>/dev/null)" != "$want" ]; then
-        $SUDO mkdir -p "$(dirname "$NM_DNS_DROPIN")"
-        printf '%s\n' "$want" | $SUDO tee "$NM_DNS_DROPIN" > /dev/null || dns_handover_failed "cannot write $NM_DNS_DROPIN"
-        info "  $NM_DNS_DROPIN: NetworkManager leaves $RESOLV_CONF alone"
-        changed=0
-    fi
-    override=$(nm_rc_manager_override) || dns_handover_failed "cannot read NetworkManager's configuration with 'NetworkManager --print-config'"
-    if [ -n "$override" ]; then
-        dns_handover_failed "NetworkManager reads $override from a file after $NM_DNS_DROPIN, so it would replace the $RESOLV_CONF link. Remove that setting ('NetworkManager --print-config' shows the files it reads) and re-run this installer"
-    fi
-    return "$changed"
-}
-
-# Links the stub and checks the link is still there a second later, relinking a
-# bounded number of times in case a write NetworkManager had queued lands first.
-link_resolved_stub() {
-    local i
-    for i in 1 2 3 4 5; do
-        $SUDO ln -sfn "$RESOLVED_STUB" "$RESOLV_CONF"
-        sleep 1
-        [ "$(readlink "$RESOLV_CONF")" = "$RESOLVED_STUB" ] && return 0
-    done
-    dns_handover_failed "$RESOLV_CONF is not a link to $RESOLVED_STUB 5 seconds after linking it, so something keeps rewriting it"
-}
-
 # Checks resolved has upstream servers and still resolves the package
 # repository host, the one name this host is known to have resolved.
 check_resolved_dns() {
@@ -1148,13 +1103,8 @@ hand_dns_to_resolved() {
     [ "${ISO_BUILD:-0}" = "1" ] && return 0
 
     if systemctl is-active --quiet systemd-resolved.service; then
-        # A handover made by an earlier run is checked again, and repaired, on
-        # every run: installs before the drop-in could lose the link.
+        # A handover made by an earlier run is checked again on every run.
         if [ -e "$RESOLV_CONF_SAVED" ] || [ -L "$RESOLV_CONF_SAVED" ]; then
-            if install_nm_dns_dropin && systemctl is-active --quiet NetworkManager.service; then
-                $SUDO systemctl reload NetworkManager.service || dns_handover_failed "NetworkManager did not reload"
-            fi
-            [ "$(readlink "$RESOLV_CONF")" = "$RESOLVED_STUB" ] || link_resolved_stub
             check_resolved_dns
         fi
         return 0
@@ -1181,15 +1131,10 @@ hand_dns_to_resolved() {
         $SUDO cp -a "$RESOLV_CONF" "$RESOLV_CONF_SAVED" || fatal "Cannot save $RESOLV_CONF to $RESOLV_CONF_SAVED"
         info "  saved $RESOLV_CONF to $RESOLV_CONF_SAVED"
     fi
-    install_nm_dns_dropin || true
     $SUDO systemctl enable --now systemd-resolved.service > /dev/null 2>&1 \
         || dns_handover_failed "systemd-resolved did not start"
     $SUDO systemctl restart NetworkManager.service || dns_handover_failed "NetworkManager did not restart"
-    # Best effort: a link with no DHCP server can hold startup past the timeout.
-    if command -v nm-online > /dev/null; then
-        nm-online -s -q --timeout=15 || true
-    fi
-    link_resolved_stub
+    $SUDO ln -sfn "$RESOLVED_STUB" "$RESOLV_CONF"
     check_resolved_dns
     info "  $RESOLV_CONF -> $RESOLVED_STUB"
 }
@@ -1679,13 +1624,12 @@ install_files() {
         $SUDO cp "$EXTRACT_DIR/microvm/"* /usr/share/spinifex/microvm/
         info "  /usr/share/spinifex/microvm/*"
     fi
-    install_sqs_store_image
     install_sqs_worker_image
 }
 
-# The SQS images ship beside the load balancer's, not inside it: an older
-# installer copies microvm/* without -r and would stop at a subdirectory.
-install_sqs_store_image() { install_sqs_microvm_image sqs-store "SQS store"; }
+# The SQS image ships beside the load balancer's, not inside it: an older
+# installer copies microvm/* without -r and would stop at a subdirectory. A
+# store image left by an earlier install is not touched: nothing boots it.
 install_sqs_worker_image() { install_sqs_microvm_image sqs-worker "SQS worker"; }
 
 # install_sqs_microvm_image <name> <label>

scripts/setup_host_test.sh

@@ -20,10 +20,9 @@ fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
 echo "-- setup.sh host DNS, chrony, needrestart and nbdkit"
 
 mkdir -p "$TMP/sys"
-for tool in bash awk sed grep readlink basename dirname cp ln mkdir cat head tail wc tee rm chmod install mv; do
+for tool in bash awk sed grep readlink basename dirname cp ln mkdir cat head tail wc tee rm chmod install; do
     ln -s "$(command -v "$tool")" "$TMP/sys/$tool"
 done
-ln -s "$(command -v sleep)" "$TMP/sys/realsleep"
 
 # stub <name> <body> — a stub command; $S is the fixture's state directory.
 stub() {
@@ -31,17 +30,10 @@ stub() {
     chmod +x "$TMP/bin/$1"
 }
 
-# nm_settle — waits for the fixture's NetworkManager to finish writing.
-nm_settle() {
-    n=0
-    while [ -e "$TMP/state/nm-pending" ] && [ $n -lt 500 ]; do "$TMP/sys/realsleep" 0.01; n=$((n + 1)); done
-}
-
 # host — a fresh fixture: resolved installed and inactive, resolvconf the
 # resolvectl link, NetworkManager running with no DNS settings, a dnf host
 # whose first enabled repository is yum.oracle.com.
 host() {
-    nm_settle
     rm -rf "${TMP:?}/state" "${TMP:?}/root" "${TMP:?}/bin"
     mkdir -p "$TMP/state/active" "$TMP/root/etc" "$TMP/root/usr/lib/systemd/system" "$TMP/bin"
     : > "$TMP/root/usr/lib/systemd/system/systemd-resolved.service"
@@ -50,28 +42,11 @@ host() {
     printf '[main]\n# dns=default\nconfigure-and-quit=no\n' > "$TMP/state/nm-config"
     printf 'Global:\nLink 2 (br-wan): 192.168.1.1\n' > "$TMP/state/resolvectl-dns"
     printf '[ol10_UEKR8]\nbaseurl = https://mirror.example/uek/\nenabled = 0\n[ol10_baseos_latest]\nbaseurl = https://yum.oracle.com/repo/OracleLinux/OL10/baseos/latest/x86_64/\nenabled = 1\n' > "$TMP/state/dnf-dump"
-    # A (re)started NetworkManager whose rc-manager is not unmanaged writes
-    # resolv.conf as a plain file on its first DNS update, which can land just
-    # after the installer links the stub.
     stub systemctl '[ "$1" = is-active ] && { [ -e "$S/active/$3" ]; exit; }
 echo "$*" >> "$S/calls"
 [ "$1 $2" = "enable --now" ] && : > "$S/active/$3"
-case "$1 $2" in
-    "restart NetworkManager.service" | "reload NetworkManager.service")
-        NetworkManager --print-config | grep -qx rc-manager=unmanaged && exit 0
-        : > "$S/nm-pending"
-        (
-            n=0
-            while [ ! -L "$S/../root/etc/resolv.conf" ] && [ $n -lt 300 ]; do realsleep 0.01; n=$((n + 1)); done
-            printf "nameserver 192.168.1.1\n" > "$S/nm-resolv.conf"
-            mv -f "$S/nm-resolv.conf" "$S/../root/etc/resolv.conf"
-            rm -f "$S/nm-pending"
-        ) > /dev/null 2>&1 &
-        ;;
-esac
 exit 0'
-    stub NetworkManager 'cat "$S/nm-config"; cat "$S/../root/nm/90-spinifex-dns.conf" "$S/nm-config-late" 2>/dev/null; :'
-    stub nm-online ':'
+    stub NetworkManager 'cat "$S/nm-config"'
     stub resolvectl '[ "$1" = dns ] && cat "$S/resolvectl-dns"'
     stub resolvectl-real ':'
     ln -s resolvectl "$TMP/bin/resolvconf"
@@ -85,17 +60,10 @@ run() {
     RC=0
     OUT=$(PATH="$TMP/bin:$TMP/sys" DISTRO_ROOT="$TMP/root" \
         SPINIFEX_RESOLV_CONF="$TMP/root/etc/resolv.conf" SPINIFEX_RESOLV_CONF_SAVED="$TMP/root/saved/resolv.conf.orig" \
-        SPINIFEX_NM_DNS_DROPIN="$TMP/root/nm/90-spinifex-dns.conf" \
         INSTALL_SPINIFEX_LIB_ONLY=1 bash -c '. "$1"; shift; for f in "$@"; do $f; done' _ "$SETUP" "$@" 2>&1) || RC=$?
 }
 
 calls() { cat "$TMP/state/calls" 2>/dev/null || true; }
-# linked — whether resolv.conf is still the stub link once NetworkManager settles.
-linked() {
-    nm_settle
-    [ "$(readlink "$TMP/root/etc/resolv.conf")" = /run/systemd/resolve/stub-resolv.conf ]
-}
-dropin() { cat "$TMP/root/nm/90-spinifex-dns.conf" 2>/dev/null || echo "<none>"; }
 saved() { cat "$TMP/root/saved/resolv.conf.orig" 2>/dev/null || echo "<none>"; }
 expect_untouched() {
     if [ "$RC" = 0 ] && [ -z "$(calls)" ] && [ ! -L "$TMP/root/etc/resolv.conf" ] && [ "$(saved)" = "<none>" ]; then
@@ -118,7 +86,7 @@ expect_fatal() { # expect_fatal <desc> <text>...
 # --------------------------------------------------------------------------
 host
 run hand_dns_to_resolved
-if [ "$RC" = 0 ] && linked \
+if [ "$RC" = 0 ] && [ "$(readlink "$TMP/root/etc/resolv.conf")" = /run/systemd/resolve/stub-resolv.conf ] \
     && [ "$(saved)" = "$(printf 'nameserver 192.168.1.1\nsearch example.lan')" ] \
     && [ "$(calls)" = "$(printf 'enable --now systemd-resolved.service\nrestart NetworkManager.service')" ] \
     && [ "$(cat "$TMP/state/looked-up")" = yum.oracle.com ]; then
@@ -130,44 +98,12 @@ fi
 : > "$TMP/state/calls"
 rm -f "$TMP/state/looked-up"
 run hand_dns_to_resolved
-if [ "$RC" = 0 ] && [ -z "$(calls)" ] && linked && [ "$(cat "$TMP/state/looked-up")" = yum.oracle.com ]; then
+if [ "$RC" = 0 ] && [ -z "$(calls)" ] && [ "$(cat "$TMP/state/looked-up")" = yum.oracle.com ]; then
     pass "a rerun after a handover checks DNS again and changes nothing"
 else
     fail "rerun after handover: rc=$RC calls='$(calls)': $OUT"
 fi
 
-if [ "$(dropin)" = "$(printf '[main]\ndns=systemd-resolved\nrc-manager=unmanaged')" ]; then
-    pass "the handover tells NetworkManager to send DNS to resolved and leave resolv.conf alone"
-else
-    fail "NetworkManager drop-in: '$(dropin)'"
-fi
-
-# An install made before the drop-in, whose NetworkManager replaced the link.
-host
-mkdir -p "$TMP/root/saved"
-echo "the original" > "$TMP/root/saved/resolv.conf.orig"
-: > "$TMP/state/active/systemd-resolved.service"
-run hand_dns_to_resolved
-if [ "$RC" = 0 ] && linked && [ "$(calls)" = "reload NetworkManager.service" ] \
-    && [ "$(dropin)" = "$(printf '[main]\ndns=systemd-resolved\nrc-manager=unmanaged')" ]; then
-    pass "a rerun after a handover whose link NetworkManager replaced adds the drop-in, reloads NM and relinks the stub"
-else
-    fail "rerun repair: rc=$RC calls='$(calls)' link='$(readlink "$TMP/root/etc/resolv.conf")' dropin='$(dropin)': $OUT"
-fi
-
-host
-printf '[main]\nrc-manager=symlink\n' > "$TMP/state/nm-config-late"
-run hand_dns_to_resolved
-expect_fatal "a NetworkManager file read after the drop-in that sets rc-manager stops the install, naming it" \
-    "rc-manager=symlink" "rm -f $TMP/root/nm/90-spinifex-dns.conf"
-case "$(calls)" in *"restart NetworkManager"*) fail "rc-manager override: restarted NetworkManager first: $(calls)" ;; esac
-
-host
-stub ln ':'
-run hand_dns_to_resolved
-expect_fatal "a resolv.conf that will not stay linked to the stub stops the install" \
-    "is not a link to /run/systemd/resolve/stub-resolv.conf"
-
 host
 mkdir -p "$TMP/root/saved"
 echo "the original" > "$TMP/root/saved/resolv.conf.orig"
@@ -219,8 +155,7 @@ host
 printf 'Global:\nLink 2 (br-wan):\n' > "$TMP/state/resolvectl-dns"
 run hand_dns_to_resolved
 expect_fatal "no upstream servers stops the install, naming the check and printing the restore" \
-    "'resolvectl dns' lists no DNS servers" "systemctl disable --now systemd-resolved" "cp -a --remove-destination $TMP/root/saved/resolv.conf.orig" \
-    "rm -f $TMP/root/nm/90-spinifex-dns.conf"
+    "'resolvectl dns' lists no DNS servers" "systemctl disable --now systemd-resolved" "cp -a --remove-destination $TMP/root/saved/resolv.conf.orig"
 
 host
 : > "$TMP/state/getent-fail"

scripts/setup_sqs_worker_image_test.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
-# setup_sqs_store_image_test.sh — tests for shipping the SQS store and worker
-# microVM images: staging them into a release tree and installing with setup.sh.
+# setup_sqs_worker_image_test.sh — tests for shipping the SQS microVM
+# image: staging it into a release tree and installing with setup.sh.
 #
 # setup.sh is sourced with INSTALL_SPINIFEX_LIB_ONLY=1 so main never runs, and
 # installs into a temporary SPINIFEX_MICROVM_DIR with SUDO cleared.
@@ -16,7 +16,7 @@ fails=0
 pass() { echo "  ok   - $1"; }
 fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
 
-echo "-- SQS store and worker microVM image release and install"
+echo "-- SQS microVM image release and install"
 
 # image <dir> <tag>: a fake built image whose bytes name its tag.
 image() {
@@ -35,11 +35,11 @@ release() {
 }
 
 # install_image <extract-dir> <microvm-dir> [path-prefix]: runs
-# install_sqs_store_image and sets RC and OUT.
+# install_sqs_worker_image and sets RC and OUT.
 install_image() {
     RC=0
     OUT=$(PATH="${3:-}$PATH" EXTRACT_DIR="$1" SPINIFEX_MICROVM_DIR="$2" INSTALL_SPINIFEX_LIB_ONLY=1 \
-        bash -c '. "$1"; SUDO=""; install_sqs_store_image' _ "$SETUP" 2>&1) || RC=$?
+        bash -c '. "$1"; SUDO=""; install_sqs_worker_image' _ "$SETUP" 2>&1) || RC=$?
 }
 
 same() { cmp -s "$1" "$2"; }
@@ -48,13 +48,13 @@ same() { cmp -s "$1" "$2"; }
 image "$TMP/built" v1
 release "$TMP/rel"
 sh "$STAGE" "$TMP/built" "$TMP/rel" > "$TMP/stage.out"
-if same "$TMP/built/vmlinuz" "$TMP/rel/microvm-sqs-store/vmlinuz" \
-    && same "$TMP/built/initramfs.cpio.gz" "$TMP/rel/microvm-sqs-store/initramfs.cpio.gz"; then
+if same "$TMP/built/vmlinuz" "$TMP/rel/microvm-sqs-worker/vmlinuz" \
+    && same "$TMP/built/initramfs.cpio.gz" "$TMP/rel/microvm-sqs-worker/initramfs.cpio.gz"; then
     pass "stages the built image beside the load balancer's"
 else
     fail "staged image differs from the build"
 fi
-if [ ! -e "$TMP/rel/microvm/sqs-store" ] && [ "$(ls "$TMP/rel/microvm" | wc -l)" -eq 3 ]; then
+if [ ! -e "$TMP/rel/microvm/sqs-worker" ] && [ "$(ls "$TMP/rel/microvm" | wc -l)" -eq 3 ]; then
     pass "leaves the load balancer's microvm directory as it was"
 else
     fail "load balancer directory changed: $(ls "$TMP/rel/microvm")"
@@ -63,21 +63,21 @@ fi
 release "$TMP/rel-none"
 mkdir -p "$TMP/half"; printf 'k\n' > "$TMP/half/vmlinuz"
 OUT=$(sh "$STAGE" "$TMP/half" "$TMP/rel-none")
-if [ ! -e "$TMP/rel-none/microvm-sqs-store" ] && echo "$OUT" | grep -q 'WARNING: SQS store microVM image missing'; then
+if [ ! -e "$TMP/rel-none/microvm-sqs-worker" ] && echo "$OUT" | grep -q 'WARNING: SQS worker microVM image missing'; then
     pass "stages nothing from an incomplete build, and says so"
 else
     fail "incomplete build staged or unannounced: $OUT"
 fi
 
-# --- an older installer meets a release that carries the store image ---
+# --- an older installer meets a release that carries the SQS image ---
 # It copies microvm/* without -r; the sibling directory must not reach it.
 mkdir -p "$TMP/old-dest"
-if cp "$TMP/rel/microvm/"* "$TMP/old-dest/" 2> "$TMP/old.err" && [ ! -e "$TMP/old-dest/sqs-store" ]; then
-    pass "an older installer's copy still succeeds and ignores the store image"
+if cp "$TMP/rel/microvm/"* "$TMP/old-dest/" 2> "$TMP/old.err" && [ ! -e "$TMP/old-dest/sqs-worker" ]; then
+    pass "an older installer's copy still succeeds and ignores the SQS image"
 else
     fail "older installer copy failed: $(cat "$TMP/old.err")"
 fi
-mkdir -p "$TMP/nested/microvm/sqs-store" "$TMP/nested-dest"
+mkdir -p "$TMP/nested/microvm/sqs-worker" "$TMP/nested-dest"
 printf 'x\n' > "$TMP/nested/microvm/vmlinuz"
 if cp "$TMP/nested/microvm/"* "$TMP/nested-dest/" 2> /dev/null; then
     fail "control: a nested directory did not break the older copy, so the test proves nothing"
@@ -87,58 +87,58 @@ fi
 
 # --- install ---
 install_image "$TMP/rel" "$TMP/mv"
-if [ "$RC" = 0 ] && same "$TMP/built/vmlinuz" "$TMP/mv/sqs-store/vmlinuz" \
-    && same "$TMP/built/initramfs.cpio.gz" "$TMP/mv/sqs-store/initramfs.cpio.gz"; then
+if [ "$RC" = 0 ] && same "$TMP/built/vmlinuz" "$TMP/mv/sqs-worker/vmlinuz" \
+    && same "$TMP/built/initramfs.cpio.gz" "$TMP/mv/sqs-worker/initramfs.cpio.gz"; then
     pass "installs the staged image where the daemon looks"
 else
     fail "install rc=$RC: $OUT"
 fi
-mode=$(stat -c '%a' "$TMP/mv/sqs-store/initramfs.cpio.gz" 2> /dev/null || echo none)
+mode=$(stat -c '%a' "$TMP/mv/sqs-worker/initramfs.cpio.gz" 2> /dev/null || echo none)
 if [ "$mode" = 644 ]; then pass "installs the image 0644"; else fail "initramfs mode $mode"; fi
 
 image "$TMP/built2" v2
 release "$TMP/rel2"
 sh "$STAGE" "$TMP/built2" "$TMP/rel2" > /dev/null
 install_image "$TMP/rel2" "$TMP/mv"
-if [ "$RC" = 0 ] && same "$TMP/built2/vmlinuz" "$TMP/mv/sqs-store/vmlinuz" \
-    && same "$TMP/built2/initramfs.cpio.gz" "$TMP/mv/sqs-store/initramfs.cpio.gz"; then
+if [ "$RC" = 0 ] && same "$TMP/built2/vmlinuz" "$TMP/mv/sqs-worker/vmlinuz" \
+    && same "$TMP/built2/initramfs.cpio.gz" "$TMP/mv/sqs-worker/initramfs.cpio.gz"; then
     pass "an upgrade replaces both files"
 else
     fail "upgrade rc=$RC: $OUT"
 fi
 
-# A release without the store image: nothing installed, and a clear warning.
+# A release without the SQS image: nothing installed, and a clear warning.
 install_image "$TMP/rel-none" "$TMP/mv-none"
-if [ "$RC" = 0 ] && [ ! -e "$TMP/mv-none/sqs-store" ] && echo "$OUT" | grep -q 'no SQS store microVM image; SQS store launches will be refused'; then
+if [ "$RC" = 0 ] && [ ! -e "$TMP/mv-none/sqs-worker" ] && echo "$OUT" | grep -q 'no SQS worker microVM image; SQS worker launches will be refused'; then
     pass "a release without the image installs nothing and warns"
 else
-    fail "release without image: rc=$RC exists=$([ -e "$TMP/mv-none/sqs-store" ] && echo yes || echo no) $OUT"
+    fail "release without image: rc=$RC exists=$([ -e "$TMP/mv-none/sqs-worker" ] && echo yes || echo no) $OUT"
 fi
 
-# installed <dir>: a store image from an earlier install, plus an unrelated file.
+# installed <dir>: an SQS image from an earlier install, plus an unrelated file.
 installed() {
-    image "$1/sqs-store" old
-    printf 'keep\n' > "$1/sqs-store/operator-note"
+    image "$1/sqs-worker" old
+    printf 'keep\n' > "$1/sqs-worker/operator-note"
 }
-gone() { [ ! -e "$1/sqs-store/vmlinuz" ] && [ ! -e "$1/sqs-store/initramfs.cpio.gz" ]; }
-kept() { [ "$(cat "$1/sqs-store/operator-note" 2> /dev/null)" = keep ]; }
+gone() { [ ! -e "$1/sqs-worker/vmlinuz" ] && [ ! -e "$1/sqs-worker/initramfs.cpio.gz" ]; }
+kept() { [ "$(cat "$1/sqs-worker/operator-note" 2> /dev/null)" = keep ]; }
 
 # Upgrading to a release without the image must not leave the old one usable.
 installed "$TMP/mv-up"
 install_image "$TMP/rel-none" "$TMP/mv-up"
 if [ "$RC" = 0 ] && gone "$TMP/mv-up" && kept "$TMP/mv-up" \
-    && echo "$OUT" | grep -q 'Removed the SQS store microVM image left by an earlier install' \
-    && echo "$OUT" | grep -q 'SQS store launches will be refused'; then
+    && echo "$OUT" | grep -q 'Removed the SQS worker microVM image left by an earlier install' \
+    && echo "$OUT" | grep -q 'SQS worker launches will be refused'; then
     pass "a release without the image removes the earlier one, keeps other files, and warns"
 else
     fail "release without image over an installed one: rc=$RC $OUT"
 fi
 
-# An incomplete store image refuses and removes the earlier pair, because the
+# An incomplete SQS image refuses and removes the earlier pair, because the
 # binaries this release installed before it would otherwise boot the old one.
 installed "$TMP/mv-bad"
 cp -r "$TMP/rel2" "$TMP/rel-bad"
-rm "$TMP/rel-bad/microvm-sqs-store/initramfs.cpio.gz"
+rm "$TMP/rel-bad/microvm-sqs-worker/initramfs.cpio.gz"
 install_image "$TMP/rel-bad" "$TMP/mv-bad"
 if [ "$RC" != 0 ] && echo "$OUT" | grep -q 'incomplete: initramfs.cpio.gz is missing' \
     && gone "$TMP/mv-bad" && kept "$TMP/mv-bad"; then
@@ -154,17 +154,17 @@ for case in missing incomplete; do
     sh "$STAGE" "$TMP/built" "$TMP/rel-$case" > /dev/null
     if [ "$case" = missing ]; then rm -r "$TMP/built-$case"; else rm "$TMP/built-$case/initramfs.cpio.gz"; fi
     OUT=$(sh "$STAGE" "$TMP/built-$case" "$TMP/rel-$case")
-    if [ ! -e "$TMP/rel-$case/microvm-sqs-store" ] && echo "$OUT" | grep -q 'tarball will not include it'; then
+    if [ ! -e "$TMP/rel-$case/microvm-sqs-worker" ] && echo "$OUT" | grep -q 'tarball will not include it'; then
         pass "restaging from a $case build ships no earlier image"
     else
-        fail "restaging from a $case build left: $(ls -R "$TMP/rel-$case/microvm-sqs-store" 2> /dev/null)"
+        fail "restaging from a $case build left: $(ls -R "$TMP/rel-$case/microvm-sqs-worker" 2> /dev/null)"
     fi
 done
-mkdir -p "$TMP/rel-other/microvm-sqs-store"
-image "$TMP/rel-other/microvm-sqs-store" old
-printf 'keep\n' > "$TMP/rel-other/microvm-sqs-store/note"
+mkdir -p "$TMP/rel-other/microvm-sqs-worker"
+image "$TMP/rel-other/microvm-sqs-worker" old
+printf 'keep\n' > "$TMP/rel-other/microvm-sqs-worker/note"
 sh "$STAGE" "$TMP/no-such-build" "$TMP/rel-other" > /dev/null
-if [ ! -e "$TMP/rel-other/microvm-sqs-store/vmlinuz" ] && [ "$(cat "$TMP/rel-other/microvm-sqs-store/note")" = keep ]; then
+if [ ! -e "$TMP/rel-other/microvm-sqs-worker/vmlinuz" ] && [ "$(cat "$TMP/rel-other/microvm-sqs-worker/note")" = keep ]; then
     pass "restaging removes only the image files"
 else
     fail "restaging touched more than the image files"
@@ -180,65 +180,25 @@ exec /usr/bin/install "$@"
 EOF
 chmod +x "$TMP/bin/install"
 install_image "$TMP/rel" "$TMP/mv" "$TMP/bin:"
-if [ "$RC" != 0 ] && same "$TMP/built/vmlinuz" "$TMP/mv/sqs-store/vmlinuz" && [ ! -e "$TMP/mv/sqs-store/initramfs.cpio.gz" ]; then
+if [ "$RC" != 0 ] && same "$TMP/built/vmlinuz" "$TMP/mv/sqs-worker/vmlinuz" && [ ! -e "$TMP/mv/sqs-worker/initramfs.cpio.gz" ]; then
     pass "an interrupted install leaves the initramfs missing, not mismatched"
 else
-    fail "interrupted install: rc=$RC initramfs=$([ -e "$TMP/mv/sqs-store/initramfs.cpio.gz" ] && echo present || echo missing)"
+    fail "interrupted install: rc=$RC initramfs=$([ -e "$TMP/mv/sqs-worker/initramfs.cpio.gz" ] && echo present || echo missing)"
 fi
 
-# --- SQS worker image: same staging and install path, its own directory ---
-install_worker() {
-    RC=0
-    OUT=$(EXTRACT_DIR="$1" SPINIFEX_MICROVM_DIR="$2" INSTALL_SPINIFEX_LIB_ONLY=1 \
-        bash -c '. "$1"; SUDO=""; install_sqs_worker_image' _ "$SETUP" 2>&1) || RC=$?
-}
-image "$TMP/wbuilt" w1
-release "$TMP/wrel"
-sh "$STAGE" "$TMP/built" "$TMP/wrel" > /dev/null
-sh "$STAGE" "$TMP/wbuilt" "$TMP/wrel" sqs-worker > /dev/null
-if same "$TMP/wbuilt/vmlinuz" "$TMP/wrel/microvm-sqs-worker/vmlinuz" \
-    && same "$TMP/wbuilt/initramfs.cpio.gz" "$TMP/wrel/microvm-sqs-worker/initramfs.cpio.gz" \
-    && same "$TMP/built/vmlinuz" "$TMP/wrel/microvm-sqs-store/vmlinuz"; then
-    pass "the worker image stages beside the store image"
+image "$TMP/keep/sqs-store" old-store
+install_image "$TMP/rel" "$TMP/keep"
+if [ "$RC" = 0 ] && [ "$(cat "$TMP/keep/sqs-store/vmlinuz")" = kernel-old-store ] \
+    && [ "$(cat "$TMP/keep/sqs-store/initramfs.cpio.gz")" = initramfs-old-store ]; then
+    pass "an install leaves a separate store image from an earlier install untouched"
 else
-    fail "worker staging: $(ls -R "$TMP/wrel")"
+    fail "earlier store image: rc=$RC $(ls -R "$TMP/keep")"
 fi
-install_worker "$TMP/wrel" "$TMP/wmv"
-if [ "$RC" = 0 ] && same "$TMP/wbuilt/initramfs.cpio.gz" "$TMP/wmv/sqs-worker/initramfs.cpio.gz" \
-    && [ ! -e "$TMP/wmv/sqs-store" ]; then
-    pass "the worker image installs into its own directory only"
+if OUT=$(sh "$STAGE" "$TMP/built" "$TMP/rel" sqs-store 2>&1); then
+    fail "the separate store image was staged"
 else
-    fail "worker install: rc=$RC $OUT"
-fi
-release "$TMP/wrel-none"
-OUT=$(sh "$STAGE" "$TMP/no-such-build" "$TMP/wrel-none" sqs-worker)
-if [ ! -e "$TMP/wrel-none/microvm-sqs-worker" ] && echo "$OUT" | grep -q 'SQS worker microVM image missing'; then
-    pass "a missing worker build stages nothing and says so"
-else
-    fail "missing worker build: $OUT"
-fi
-install_worker "$TMP/wrel-none" "$TMP/wmv"
-if [ "$RC" = 0 ] && [ ! -e "$TMP/wmv/sqs-worker/vmlinuz" ] && [ ! -e "$TMP/wmv/sqs-worker/initramfs.cpio.gz" ] \
-    && echo "$OUT" | grep -q 'no SQS worker microVM image; SQS worker launches will be refused'; then
-    pass "a release without the worker image removes the earlier one and warns"
-else
-    fail "release without worker image: rc=$RC $OUT"
-fi
-cp -r "$TMP/wrel" "$TMP/wrel-bad"
-rm "$TMP/wrel-bad/microvm-sqs-worker/vmlinuz"
-install_worker "$TMP/wrel" "$TMP/wmv"
-install_worker "$TMP/wrel-bad" "$TMP/wmv"
-if [ "$RC" != 0 ] && echo "$OUT" | grep -q 'SQS worker microVM image in this release is incomplete: vmlinuz is missing' \
-    && [ ! -e "$TMP/wmv/sqs-worker/initramfs.cpio.gz" ]; then
-    pass "an incomplete worker image is refused and the earlier pair is removed"
-else
-    fail "incomplete worker image: rc=$RC $OUT"
-fi
-if OUT=$(sh "$STAGE" "$TMP/wbuilt" "$TMP/wrel" lb 2>&1); then
-    fail "an unknown image name was staged"
-else
-    pass "an unknown image name is refused"
+    pass "the separate store image is no longer staged"
 fi
 
 [ "$fails" = 0 ] || { echo "FAIL: $fails"; exit 1; }
-echo "  setup_sqs_store_image_test ok"
+echo "  setup_sqs_worker_image_test ok"

scripts/sync-aws-models.sh

@@ -29,6 +29,7 @@ MODELS=(
     rds/2014-10-31
     route-53/2013-04-01
     s3/2006-03-01
+    sqs/2012-11-05
     sts/2011-06-15
 )
 

spinifex/agents/eks/gatewayfetch/fetch.go

@@ -16,9 +16,6 @@
 //
 // The recovery resource returns a single epoch\taction\tsnapshot line the on-VM
 // k3s-recovery agent applies before k3s starts (etcd cluster-reset / wipe-rejoin).
-//
-// The workers resource returns one instance ID per line: every worker the
-// cluster's nodegroups still own, which the on-VM node-gc agent keeps Nodes for.
 package gatewayfetch
 
 import (
@@ -49,7 +46,7 @@ type Config struct {
 	Region      string
 	AccountID   string
 	ClusterName string
-	// Resource is "addons", "recovery" or "workers".
+	// Resource is "addons" or "recovery".
 	Resource   string
 	InstanceID string
 	// Out receives the rendered TSV.
@@ -64,8 +61,8 @@ func Run(ctx context.Context, cfg Config) error {
 		return errors.New("--account-id is required (or set EKS_ACCOUNT_ID)")
 	case cfg.ClusterName == "":
 		return errors.New("--cluster is required (or set EKS_CLUSTER_NAME)")
-	case cfg.Resource != "addons" && cfg.Resource != "recovery" && cfg.Resource != "workers":
-		return errors.New("--resource must be addons, recovery or workers")
+	case cfg.Resource != "addons" && cfg.Resource != "recovery":
+		return errors.New("--resource must be addons or recovery")
 	case cfg.Resource == "recovery" && cfg.InstanceID == "":
 		return errors.New("--instance-id is required for --resource recovery (or set EKS_INSTANCE_ID)")
 	}
@@ -83,9 +80,6 @@ func Run(ctx context.Context, cfg Config) error {
 	case "recovery":
 		path = "/clusters/" + cfg.ClusterName + "/internal-recovery/" + cfg.AccountID + "/" + cfg.InstanceID
 		emit = emitRecoveryTSV
-	case "workers":
-		path = "/clusters/" + cfg.ClusterName + "/internal-workers/" + cfg.AccountID
-		emit = emitWorkers
 	default:
 		path = "/clusters/" + cfg.ClusterName + "/internal-addons/" + cfg.AccountID
 		emit = emitAddonsTSV
@@ -169,24 +163,3 @@ func emitRecoveryTSV(out io.Writer, body []byte) error {
 	}
 	return nil
 }
-
-type internalWorkersResponse struct {
-	InstanceIDs []string `json:"instanceIds"`
-}
-
-// emitWorkers writes one managed-worker instance ID per line. An empty set
-// writes nothing, which the node-gc agent tells apart from a failed fetch by
-// this command's exit status, never by the output being empty.
-func emitWorkers(out io.Writer, body []byte) error {
-	var resp internalWorkersResponse
-	if err := json.Unmarshal(body, &resp); err != nil {
-		return fmt.Errorf("unmarshal internal-workers response: %w", err)
-	}
-	w := bufio.NewWriter(out)
-	for _, id := range resp.InstanceIDs {
-		if _, err := fmt.Fprintln(w, id); err != nil {
-			return err
-		}
-	}
-	return w.Flush()
-}

spinifex/agents/eks/gatewayfetch/fetch_test.go

@@ -49,7 +49,7 @@ func TestRun_RejectsBeforeContactingGateway(t *testing.T) {
 	}{
 		{"account", func(c *Config) { c.AccountID = "" }, "--account-id is required (or set EKS_ACCOUNT_ID)"},
 		{"cluster", func(c *Config) { c.ClusterName = "" }, "--cluster is required (or set EKS_CLUSTER_NAME)"},
-		{"resource", func(c *Config) { c.Resource = "secrets" }, "--resource must be addons, recovery or workers"},
+		{"resource", func(c *Config) { c.Resource = "secrets" }, "--resource must be addons or recovery"},
 		{"recovery instance", func(c *Config) { c.Resource = "recovery" }, "--instance-id is required for --resource recovery (or set EKS_INSTANCE_ID)"},
 		{"gateway client", func(c *Config) { c.GatewayURL = "" }, "build gateway client: eksgw: baseURL is required"},
 	}
@@ -92,26 +92,6 @@ func TestRun_RendersRecoveryDirective(t *testing.T) {
 	}
 }
 
-// The gateway's restjson marshaler names the field InstanceIDs; both that and
-// the json-tag spelling decode, so the agent reads either.
-func TestRun_RendersManagedWorkers(t *testing.T) {
-	for _, body := range []string{
-		`{"InstanceIDs":["i-0aaa","i-0bbb"]}`,
-		`{"instanceIds":["i-0aaa","i-0bbb"]}`,
-	} {
-		var hits atomic.Int32
-		cfg, out := gateway(t, "/clusters/demo/internal-workers/123456789012", http.StatusOK, body, &hits)
-		cfg.Resource = "workers"
-
-		if err := Run(context.Background(), cfg); err != nil {
-			t.Fatalf("Run: %v", err)
-		}
-		if got, want := out.String(), "i-0aaa\ni-0bbb\n"; got != want {
-			t.Fatalf("stdout = %q, want %q", got, want)
-		}
-	}
-}
-
 func TestRun_RenderFailureIsNotRetried(t *testing.T) {
 	var hits atomic.Int32
 	cfg, _ := gateway(t, "/clusters/demo/internal-addons/123456789012", http.StatusOK, `{not-json`, &hits)

spinifex/agents/sqs/workerboot/claimer.go

@@ -0,0 +1,22 @@
+package workerboot
+
+import (
+	"context"
+
+	"github.com/mulgadc/spinifex/internal/sqsworkergw"
+	"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerkey"
+)
+
+// GatewayClaimer adapts the signed gateway client to the receipt key decision.
+type GatewayClaimer struct{ Client *sqsworkergw.Client }
+
+var _ workerkey.Claimer = GatewayClaimer{}
+
+func (c GatewayClaimer) Claim(ctx context.Context) (workerkey.Decision, error) {
+	k, err := c.Client.ClaimReceiptKey(ctx)
+	return workerkey.Decision{Pending: k.Pending, Fingerprint: k.Fingerprint, HostNodes: k.HostNodes}, err
+}
+
+func (c GatewayClaimer) RecordFingerprint(ctx context.Context, fingerprint string) error {
+	return c.Client.RecordReceiptKey(ctx, fingerprint)
+}

spinifex/agents/sqs/workerboot/gateway_test.go

@@ -3,22 +3,14 @@ package workerboot
 import (
 	"bytes"
 	"context"
-	"crypto/ecdsa"
-	"crypto/elliptic"
-	"crypto/rand"
 	"crypto/sha256"
-	"crypto/x509"
-	"crypto/x509/pkix"
 	"encoding/hex"
 	"encoding/pem"
-	"errors"
 	"fmt"
 	"io"
 	"maps"
-	"math/big"
 	"net/http"
 	"net/http/httptest"
-	"os"
 	"slices"
 	"strings"
 	"sync"
@@ -33,20 +25,16 @@ import (
 	testutil "github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerkey"
 	sqsworkerapi "github.com/mulgadc/spinifex/spinifex/domains/sqs/awsapi/worker"
-	"github.com/mulgadc/spinifex/spinifex/domains/sqs/firststore"
-	"github.com/mulgadc/spinifex/spinifex/domains/sqs/storelaunch"
 	"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerident"
 	"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerrole"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
-	"github.com/nats-io/nkeys"
 )
 
 const (
-	gwInstance  = "i-0abc12345def67890"
-	gwSuccessor = "i-0fedcba9876543210"
-	gwVolume    = "vol-0a1b2c3d4e5f60718"
+	gwInstance = "i-0abc12345def67890"
+	gwVolume   = "vol-0a1b2c3d4e5f60718"
 )
 
 // session is one instance-role credential: its secret and the instance ID
@@ -230,58 +218,55 @@ func (g *localGateway) reply(code string) {
 	}
 }
 
-func (g *localGateway) sentBodies() []string {
-	g.mu.Lock()
-	defer g.mu.Unlock()
-	return slices.Clone(g.bodies)
-}
-
-// gwCluster is the domain's real worker-identity and first-store records.
+// gwCluster is the domain's real worker-identity record behind the gateway.
 type gwCluster struct {
-	ids    workerident.Store
-	fence  firststore.Store
-	stores storelaunch.Records
-	gw     *localGateway
+	ids workerident.Store
+	gw  *localGateway
 }
 
-func newGWCluster(t *testing.T) gwCluster {
+// newGWCluster hands the gateway's work actions to work and has it report
+// hostNodes as its cluster size; nil hostNodes is the declared size.
+func newGWCluster(t *testing.T, work sqsworkerapi.Work, hostNodes func() int) gwCluster {
 	t.Helper()
 	clustersize.DeclareForTest(t, 1)
 	_, _, js := testutil.StartTestJetStream(t)
-	ctx := context.Background()
-	ids, err := workerident.Open(ctx, js)
+	ids, err := workerident.Open(context.Background(), js)
 	if err != nil {
 		t.Fatal(err)
 	}
-	fence, err := firststore.Open(ctx, js)
-	if err != nil {
-		t.Fatal(err)
-	}
-	stores, err := storelaunch.Open(ctx, js)
-	if err != nil {
-		t.Fatal(err)
-	}
-	return gwCluster{ids: ids, fence: fence, stores: stores, gw: newLocalGateway(t, sqsworkerapi.JetStreamDeps(js))}
+	deps := sqsworkerapi.JetStreamDeps(js)
+	deps.Work = work
+	deps.HostNodes = hostNodes
+	return gwCluster{ids: ids, gw: newLocalGateway(t, deps)}
 }
 
-func (c gwCluster) launch(t *testing.T, claim string, supersedes uint64, inst string) workerident.Intent {
+// launch records a worker on a newly created volume, so its receipt key is
+// pending, and binds it to inst.
+func (c gwCluster) launch(t *testing.T, inst string) workerident.Intent {
 	t.Helper()
 	ctx := context.Background()
-	in, err := c.ids.BeginLaunch(ctx, testRegion, workerident.LaunchRequest{InstanceType: "t3.small",
-		ClaimID: claim, Supersedes: supersedes, NodeID: "node-a", DataVolumeID: gwVolume, DataVolumeEncrypted: true,
+	in, err := c.ids.BeginLaunch(ctx, testRegion, workerident.LaunchRequest{
+		InstanceType: "t3.small", ClaimID: "claim-0001", NodeID: "node-a", DataVolumeEncrypted: true, SizeGiB: 8,
 	})
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := c.ids.BindInstance(ctx, in, inst); err != nil {
-		t.Fatal(err)
+	for _, step := range []func() error{
+		func() error { return c.ids.MarkVolumeAttempted(ctx, in) },
+		func() error { return c.ids.RecordVolume(ctx, in, gwVolume) },
+		func() error { return c.ids.MarkDispatched(ctx, in) },
+		func() error { return c.ids.BindInstance(ctx, in, inst) },
+	} {
+		if err := step(); err != nil {
+			t.Fatal(err)
+		}
 	}
 	return in
 }
 
 // guestFor points the fake guest's boot at the local gateway, signing with
 // the instance's own credential through the real client.
-func (c gwCluster) guestFor(t *testing.T, g *guest, accessKey, inst string) {
+func (c gwCluster) guestFor(t *testing.T, g *guest, accessKey, inst string) *gwsign.Signer {
 	t.Helper()
 	signer := c.gw.issue(accessKey, "secret-"+accessKey, inst)
 	g.fw("data-serial", testSerial)
@@ -289,388 +274,14 @@ func (c gwCluster) guestFor(t *testing.T, g *guest, accessKey, inst string) {
 	g.fw("gateway-url", c.gw.srv.URL)
 	g.cfg.Wait = 10 * time.Millisecond
 	g.cfg.NewClaimer = func(region, url string) (workerkey.Claimer, error) {
-		return sqsworkergw.New(url, region, c.gw.caPEM(), signer)
+		cl, err := sqsworkergw.New(url, region, c.gw.caPEM(), signer)
+		return GatewayClaimer{Client: cl}, err
 	}
+	return signer
 }
 
-func (c gwCluster) assertNoSeedSent(t *testing.T, seed string) {
-	t.Helper()
-	seed = strings.TrimSpace(seed)
-	for _, b := range c.gw.sentBodies() {
-		if strings.Contains(b, seed) {
-			t.Fatal("a request carried seed material")
-		}
-	}
-}
-
-// A bound worker boots: the gateway authorizes creation, the guest creates
-// its key on the mounted disk and records only the public key; reboots get
-// that same key back from the domain.
-func TestGateway_FirstBootCreatesKeyAndRebootsReuseIt(t *testing.T) {
-	c := newGWCluster(t)
-	c.launch(t, "claim-0001", 0, gwInstance)
-	g := newGuest(t)
-	c.guestFor(t, g, "ASIAWORKER1", gwInstance)
-
-	pub, err := g.boot()
-	if err != nil {
-		t.Fatal(err)
-	}
-	seed := g.seed()
-	for boot := 2; boot <= 3; boot++ {
-		g.resetMounts()
-		got, err := g.boot()
-		if err != nil || got != pub || g.seed() != seed {
-			t.Fatalf("reboot %d = %q, %v", boot, got, err)
-		}
-	}
-	records := 0
-	for _, b := range c.gw.sentBodies() {
-		if strings.Contains(b, "Action=RecordWorkerPublicKey") {
-			records++
-			if !strings.Contains(b, "PublicKey="+pub) {
-				t.Fatal("a record carried a different key")
-			}
-		}
-	}
-	if records != 1 {
-		t.Fatalf("records = %d, want 1", records)
-	}
-	c.assertNoSeedSent(t, seed)
-	g.assertNoSeedLeak(nil, seed)
-}
-
-// The record's reply is lost after it landed: the boot retries, the domain
-// answers with the recorded key, and the same key is used.
-func TestGateway_LostRecordReplyKeepsTheKey(t *testing.T) {
-	c := newGWCluster(t)
-	c.launch(t, "claim-0001", 0, gwInstance)
-	g := newGuest(t)
-	c.guestFor(t, g, "ASIAWORKER1", gwInstance)
-	var dropped bool
-	c.gw.dropReply = func(action string) bool {
-		if action == "RecordWorkerPublicKey" && !dropped {
-			dropped = true
-			return true
-		}
-		return false
-	}
-	pub, err := g.boot()
-	if err != nil || !dropped {
-		t.Fatalf("boot = %q, %v (dropped=%v)", pub, err, dropped)
-	}
-	g.resetMounts()
-	if again, err := g.boot(); err != nil || again != pub {
-		t.Fatalf("reboot = %q, %v", again, err)
-	}
-	c.assertNoSeedSent(t, g.seed())
-}
-
-// After the key is recorded, a lost guest copy fails closed: the domain
-// answers with the recorded key and no replacement is created or sent.
-func TestGateway_KeyLostAfterRecordingFailsClosed(t *testing.T) {
-	c := newGWCluster(t)
-	c.launch(t, "claim-0001", 0, gwInstance)
-	g := newGuest(t)
-	c.guestFor(t, g, "ASIAWORKER1", gwInstance)
-	if _, err := g.boot(); err != nil {
-		t.Fatal(err)
-	}
-	if err := os.Remove(g.keyFile()); err != nil {
-		t.Fatal(err)
-	}
-	before := len(c.gw.sentBodies())
-	g.resetMounts()
-	if _, err := g.boot(); !errors.Is(err, workerkey.ErrKeyMissing) {
-		t.Fatalf("err = %v, want ErrKeyMissing", err)
-	}
-	if _, err := os.Lstat(g.keyFile()); !errors.Is(err, os.ErrNotExist) {
-		t.Fatal("a replacement key was created")
-	}
-	for _, b := range c.gw.sentBodies()[before:] {
-		if strings.Contains(b, "RecordWorkerPublicKey") {
-			t.Fatal("a replacement key was offered")
-		}
-	}
-}
-
-// An instance the domain never bound, or one already replaced, is refused
-// at the first call and creates no key; AccessDenied is not retried.
-func TestGateway_UnboundOrReplacedInstanceIsRefused(t *testing.T) {
-	t.Run("unbound", func(t *testing.T) {
-		c := newGWCluster(t)
-		c.launch(t, "claim-0001", 0, gwInstance)
-		g := newGuest(t)
-		c.guestFor(t, g, "ASIAOTHER", gwSuccessor)
-		_, err := g.boot()
-		api, ok := errors.AsType[*sqsworkergw.APIError](err)
-		if !ok || api.Code != "AccessDenied" {
-			t.Fatalf("err = %v, want AccessDenied", err)
-		}
-		if n := len(c.gw.sentBodies()); n != g.cfg.Attempts {
-			t.Fatalf("calls = %d, want the %d attempts and no more", n, g.cfg.Attempts)
-		}
-		g.assertNoKey()
-	})
-	t.Run("replaced", func(t *testing.T) {
-		c := newGWCluster(t)
-		first := c.launch(t, "claim-0001", 0, gwInstance)
-		ctx := context.Background()
-		if err := c.ids.RecordStopped(ctx, first, gwInstance); err != nil {
-			t.Fatal(err)
-		}
-		if err := c.fence.Fence(ctx, testRegion, first.Generation+1); err != nil {
-			t.Fatal(err)
-		}
-		c.launch(t, "claim-0002", first.Generation, gwSuccessor)
-		g := newGuest(t)
-		c.guestFor(t, g, "ASIAOLD", gwInstance)
-		if _, err := g.boot(); err == nil {
-			t.Fatal("the replaced instance booted")
-		}
-		g.assertNoKey()
-	})
-}
-
-// A bad signature is refused before any decision, and creates no key.
-func TestGateway_BadSignatureCreatesNoKey(t *testing.T) {
-	c := newGWCluster(t)
-	c.launch(t, "claim-0001", 0, gwInstance)
-	g := newGuest(t)
-	c.guestFor(t, g, "ASIAWORKER1", gwInstance)
-	forged := gwsign.NewStatic("ASIAWORKER1", "not-the-secret")
-	g.cfg.NewClaimer = func(region, url string) (workerkey.Claimer, error) {
-		return sqsworkergw.New(url, region, c.gw.caPEM(), forged)
-	}
-	if _, err := g.boot(); err == nil {
-		t.Fatal("a forged signature booted")
-	}
-	g.assertNoKey()
-}
-
-// The client trusts only the cluster CA it was given: a gateway whose
-// certificate another CA signed is never sent a request.
-func TestGateway_UntrustedServerIsNotContacted(t *testing.T) {
-	c := newGWCluster(t)
-	c.launch(t, "claim-0001", 0, gwInstance)
-	g := newGuest(t)
-	c.guestFor(t, g, "ASIAWORKER1", gwInstance)
-	g.cfg.Attempts = 1
-	signer := c.gw.issue("ASIAWORKER1", "secret-ASIAWORKER1", gwInstance)
-	g.cfg.NewClaimer = func(region, url string) (workerkey.Claimer, error) {
-		return sqsworkergw.New(url, region, otherCA(t), signer)
-	}
-	if _, err := g.boot(); err == nil || !strings.Contains(err.Error(), "certificate") {
-		t.Fatalf("err = %v, want a TLS refusal", err)
-	}
-	if len(c.gw.sentBodies()) != 0 {
-		t.Fatal("a request reached a server outside the trusted CA")
-	}
-	g.assertNoKey()
-}
-
-// otherCA is a fresh self-signed CA unrelated to the test server's.
-func otherCA(t *testing.T) []byte {
-	t.Helper()
-	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
-	if err != nil {
-		t.Fatal(err)
-	}
-	tmpl := &x509.Certificate{
-		SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "other"},
-		NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour),
-		IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign,
-	}
-	der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
-	if err != nil {
-		t.Fatal(err)
-	}
-	return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
-}
-
-// The worker VM starts before its launch binds the instance, so its first
-// calls are denied. It creates nothing while denied, and boots once bound.
-func TestGateway_BootBeforeBindSucceedsAfterBinding(t *testing.T) {
-	c := newGWCluster(t)
-	ctx := context.Background()
-	intent, err := c.ids.BeginLaunch(ctx, testRegion, workerident.LaunchRequest{InstanceType: "t3.small",
-		ClaimID: "claim-0001", NodeID: "node-a", DataVolumeID: gwVolume, DataVolumeEncrypted: true,
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-	g := newGuest(t)
-	c.guestFor(t, g, "ASIAWORKER1", gwInstance)
-	g.cfg.Attempts = 1000
-	c.gw.replied = make(chan string, 2000)
-
-	type result struct {
-		pub string
-		err error
-	}
-	done := make(chan result, 1)
-	go func() {
-		pub, err := g.boot()
-		done <- result{pub, err}
-	}()
-	if code := <-c.gw.replied; code != "AccessDenied" {
-		t.Fatalf("first reply = %q, want AccessDenied before binding", code)
-	}
-	g.assertNoKey()
-	if err := c.ids.BindInstance(ctx, intent, gwInstance); err != nil {
-		t.Fatal(err)
-	}
-	r := <-done
-	if r.err != nil || r.pub == "" {
-		t.Fatalf("boot after binding = %q, %v", r.pub, r.err)
-	}
-	if g.seed() == "" {
-		t.Fatal("booted without a key on the worker disk")
-	}
-}
-
-// A replaced worker is never bound again: it keeps being denied, creates
-// nothing and gives up within its attempts.
-func TestGateway_StaleWorkerGivesUpWithinTheBound(t *testing.T) {
-	c := newGWCluster(t)
-	ctx := context.Background()
-	first := c.launch(t, "claim-0001", 0, gwInstance)
-	if err := c.ids.RecordStopped(ctx, first, gwInstance); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.fence.Fence(ctx, testRegion, first.Generation+1); err != nil {
-		t.Fatal(err)
-	}
-	c.launch(t, "claim-0002", first.Generation, gwSuccessor)
-	g := newGuest(t)
-	c.guestFor(t, g, "ASIAOLD", gwInstance)
-	g.cfg.Attempts = 5
-	_, err := g.boot()
-	if api, ok := errors.AsType[*sqsworkergw.APIError](err); !ok || !api.Pending() {
-		t.Fatalf("err = %v, want the denial", err)
-	}
-	if n := len(c.gw.sentBodies()); n != 5 {
-		t.Fatalf("calls = %d, want exactly the 5 attempts", n)
-	}
-	g.assertNoKey()
-}
-
-// bindStore runs the store launch record through to bound for pub at address,
-// as the operator's EnsureSQSStore would.
-func (c gwCluster) bindStore(t *testing.T, pub, address string) {
-	t.Helper()
-	ctx := context.Background()
-	const claim = "store-claim-0001"
-	steps := []func() error{
-		func() error { return c.stores.Begin(ctx, testRegion, claim, "node-a", pub, "t3.medium", 8) },
-		func() error { return c.stores.MarkVolumeAttempted(ctx, testRegion, claim) },
-		func() error { return c.stores.RecordVolume(ctx, testRegion, claim, "vol-0b1c2d3e4f5a60718") },
-		func() error { return c.stores.MarkNetworkAttempted(ctx, testRegion, claim) },
-		func() error { return c.stores.MarkDispatched(ctx, testRegion, claim) },
-		func() error { return c.stores.Bind(ctx, testRegion, claim, "i-0123456789abcdef0", address) },
-	}
-	for i, step := range steps {
-		if err := step(); err != nil {
-			t.Fatalf("store launch step %d: %v", i, err)
-		}
-	}
-}
-
-// readyFor points the guest's readiness at the local gateway and a store on port.
-func (c gwCluster) readyFor(g *guest, accessKey, inst string, port int) ReadyConfig {
-	signer := c.gw.issue(accessKey, "secret-"+accessKey, inst)
-	return ReadyConfig{
-		Config: g.cfg,
-		NewDescriber: func(region, url string) (StoreDescriber, error) {
-			cl, err := sqsworkergw.New(url, region, c.gw.caPEM(), signer)
-			if err != nil {
-				return nil, err
-			}
-			return gwDescriber{cl}, nil
-		},
-		Checker: NATSChecker{Port: port, Dial: time.Second},
-	}
-}
-
-type gwDescriber struct{ c *sqsworkergw.Client }
-
-func (d gwDescriber) DescribeStore(ctx context.Context) (Store, error) {
-	st, err := d.c.DescribeStore(ctx)
-	return Store(st), err
-}
-
-// The whole path with real components: the worker boots and records its key,
-// waits while the operator has not launched the store, then reaches the real
-// store's JetStream with its own key once the store is bound; after a reboot
-// it is ready again with the same key. No request carries the seed.
-func TestGateway_ReadyAfterADelayedStoreLaunch(t *testing.T) {
-	c := newGWCluster(t)
-	c.launch(t, "claim-0001", 0, gwInstance)
-	g := newGuest(t)
-	c.guestFor(t, g, "ASIAWORKER1", gwInstance)
-	pub, err := g.boot()
-	if err != nil {
-		t.Fatal(err)
-	}
-	port := storeServer(t, pub)
-	err = g.watch(c.readyFor(g, "ASIAWORKER1", gwInstance, port), func(call int, ready bool) bool {
-		if ready {
-			if call < 4 {
-				t.Fatalf("ready at call %d, before the store launch", call)
-			}
-			return true
-		}
-		if call == 3 {
-			c.bindStore(t, pub, "127.0.0.1")
-		}
-		return false
-	})
-	if !errors.Is(err, context.Canceled) {
-		t.Fatalf("watch = %v", err)
-	}
-	for boot := 1; boot <= 2; boot++ {
-		g.resetMounts()
-		if again, err := g.boot(); err != nil || again != pub {
-			t.Fatalf("reboot %d = %q %v", boot, again, err)
-		}
-		if err := g.watch(c.readyFor(g, "ASIAWORKER1", gwInstance, port), func(_ int, ready bool) bool { return ready }); !errors.Is(err, context.Canceled) {
-			t.Fatalf("ready after reboot %d: %v", boot, err)
-		}
-	}
-	c.assertNoSeedSent(t, g.seed())
-	g.assertNoSeedLeak(nil, g.seed())
-}
-
-// A store bound to a key other than this worker's is never checked, and a
-// replaced worker cannot read the store at all; neither is ever ready.
-func TestGateway_ReadyRefusesWrongKeyAndReplacedWorker(t *testing.T) {
-	c := newGWCluster(t)
-	first := c.launch(t, "claim-0001", 0, gwInstance)
-	g := newGuest(t)
-	c.guestFor(t, g, "ASIAWORKER1", gwInstance)
-	if _, err := g.boot(); err != nil {
-		t.Fatal(err)
-	}
-	other, _ := nkeys.CreateUser()
-	otherPub, _ := other.PublicKey()
-	c.bindStore(t, otherPub, "127.0.0.1")
-	port := storeServer(t, otherPub)
-	never := func(call int, ready bool) bool {
-		if ready {
-			t.Fatal("ready against a store for another key")
-		}
-		return call == 3
-	}
-	_ = g.watch(c.readyFor(g, "ASIAWORKER1", gwInstance, port), never)
-	if !slices.ContainsFunc(g.logs, func(l string) bool { return strings.Contains(l, ErrStoreKeyMismatch.Error()) }) {
-		t.Fatalf("no mismatch reported: %v", g.logs)
-	}
-
-	if err := c.ids.RecordStopped(context.Background(), first, gwInstance); err != nil {
-		t.Fatal(err)
-	}
-	_ = g.watch(c.readyFor(g, "ASIAWORKER1", gwInstance, port), never)
-	if !strings.Contains(g.logs[len(g.logs)-1], "AccessDenied") {
-		t.Fatalf("stopped worker: %v", g.logs[len(g.logs)-1])
-	}
+func (g *localGateway) sentBodies() []string {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	return slices.Clone(g.bodies)
 }

spinifex/agents/sqs/workerboot/natsready.go

@@ -10,15 +10,14 @@ import (
 
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
-	"github.com/nats-io/nkeys"
 )
 
-// StorePort is the port the store guest's NATS listens on.
+// StorePort is the port the guest's own SQS NATS server listens on, bound
+// to loopback only; nothing outside the guest can reach it.
 const StorePort = 4222
 
-// NATSChecker connects with the worker key and asks JetStream for its account
-// info. It authenticates the client only: there is no TLS, so a pass says
-// something answering at the address accepted the key.
+// NATSChecker asks the guest's loopback JetStream for its account info. A
+// pass says the local store is up and serving JetStream.
 type NATSChecker struct {
 	// Port overrides StorePort, for tests against a local server.
 	Port int
@@ -35,36 +34,22 @@ func (e *StoreUnreachableError) Error() string   { return "store not answering:
 func (e *StoreUnreachableError) Unwrap() error   { return e.Err }
 func (e *StoreUnreachableError) Retryable() bool { return true }
 
-// ErrStoreRefusedKey is a store that answered but refused the worker key.
-var ErrStoreRefusedKey = errors.New("the store refused the worker key")
-
-// Check passes once JetStream answers for the worker key.
-func (c NATSChecker) Check(ctx context.Context, address string, kp nkeys.KeyPair) error {
-	if ip := net.ParseIP(address); ip == nil || ip.To4() == nil {
-		return errors.New("store address is not an IPv4 address")
-	}
-	pub, err := kp.PublicKey()
-	if err != nil {
-		return fmt.Errorf("derive worker public key: %w", err)
-	}
-	port := c.Port
+func localURL(port int) string {
 	if port == 0 {
 		port = StorePort
 	}
+	return "nats://" + net.JoinHostPort("127.0.0.1", strconv.Itoa(port))
+}
+
+// Check passes once the local JetStream answers.
+func (c NATSChecker) Check(ctx context.Context) error {
 	dial := c.Dial
 	if dial == 0 {
 		dial = 5 * time.Second
 	}
-	nc, err := nats.Connect("nats://"+net.JoinHostPort(address, strconv.Itoa(port)),
-		nats.Nkey(pub, kp.Sign),
-		nats.Timeout(dial),
-		nats.NoReconnect(),
-		nats.Name("spinifex-sqs-worker-readiness"),
-	)
+	nc, err := nats.Connect(localURL(c.Port), nats.Timeout(dial), nats.NoReconnect(),
+		nats.Name("spinifex-sqs-worker-readiness"))
 	if err != nil {
-		if errors.Is(err, nats.ErrAuthorization) || errors.Is(err, nats.ErrAuthExpired) {
-			return fmt.Errorf("%w: %w", ErrStoreRefusedKey, err)
-		}
 		return &StoreUnreachableError{Err: err}
 	}
 	defer nc.Close()
@@ -79,3 +64,5 @@ func (c NATSChecker) Check(ctx context.Context, address string, kp nkeys.KeyPair
 	}
 	return nil
 }
+
+var errNoChecker = errors.New("no store checker is configured")

spinifex/agents/sqs/workerboot/ready.go

@@ -9,37 +9,19 @@ import (
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerkey"
-	"github.com/nats-io/nkeys"
 )
 
-// Store is the Region's store launch as the gateway reports it. Bound means a
-// VM was launched for the store, not that it serves.
-type Store struct {
-	Bound     bool
-	Address   string
-	PublicKey string
-}
-
-// StoreDescriber asks the gateway for this worker's store.
-type StoreDescriber interface {
-	DescribeStore(ctx context.Context) (Store, error)
-}
-
-// NewDescriber builds the gateway client the store's address comes from.
-type NewDescriber func(region, gatewayURL string) (StoreDescriber, error)
-
-// Checker proves the store's JetStream answers this worker's key at address.
-// Only an error with Retryable() true is asked again.
+// Checker proves the guest's local JetStream answers. Only an error with
+// Retryable() true is asked again.
 type Checker interface {
-	Check(ctx context.Context, address string, kp nkeys.KeyPair) error
+	Check(ctx context.Context) error
 }
 
 // ReadyConfig is Config plus the readiness capabilities and cadence.
 type ReadyConfig struct {
 	Config
 
-	NewDescriber NewDescriber
-	Checker      Checker
+	Checker Checker
 	// CallTimeout bounds one describe-and-check pass; zero is 30 seconds.
 	CallTimeout time.Duration
 	// Refresh is the wait after a pass before checking again; zero is 30s.
@@ -47,18 +29,15 @@ type ReadyConfig struct {
 	// Refused is the wait after a failure that cannot clear by itself, such
 	// as a refused or mismatched key; zero is five minutes.
 	Refused time.Duration
+	// Serve, when set, also serves SQS owner calls while Watch runs.
+	Serve *ServeConfig
 }
 
-// ErrStoreKeyMismatch is a bound store launched for a key other than the one
-// on this worker's disk; it is asked again only at the Refused cadence.
-var ErrStoreKeyMismatch = errors.New("the store was launched for a different worker key")
-
 const readyFile = "store-ready"
 
 // Watch keeps the readiness marker true until ctx ends: it re-proves the disk,
-// then checks the store with the worker's existing key every pass, writing the
-// marker on a pass and removing it on any failure and on return. It never
-// creates a key. NKey authenticates this client only, not the store server.
+// then checks the guest's local store every pass, writing the marker on a
+// pass and removing it on any failure and on return. It never creates a key.
 func Watch(ctx context.Context, cfg ReadyConfig) error {
 	marker := filepath.Join(cfg.RunDir, readyFile)
 	defer func() { _ = os.Remove(marker) }()
@@ -72,14 +51,19 @@ func Watch(ctx context.Context, cfg ReadyConfig) error {
 	if err := mountWorkerDisk(cfg.Config, c.serial); err != nil {
 		return err
 	}
-	if cfg.NewDescriber == nil || cfg.Checker == nil {
-		return errors.New("no gateway client or store checker is configured")
-	}
-	d, err := cfg.NewDescriber(c.region, c.gatewayURL)
-	if err != nil {
-		return fmt.Errorf("build gateway client: %w", err)
+	if cfg.Checker == nil {
+		return errNoChecker
 	}
 	keys := workerkey.Store{Dir: filepath.Join(cfg.DataDir, keyDirName)}
+	if cfg.Serve != nil {
+		done := make(chan struct{})
+		go func() {
+			defer close(done)
+			serve(ctx, cfg, c, keys)
+		}()
+		// Calls in flight finish before the marker is removed and Watch returns.
+		defer func() { <-done }()
+	}
 	callTimeout := orDefault(cfg.CallTimeout, 30*time.Second)
 	refresh := orDefault(cfg.Refresh, 30*time.Second)
 	refused := orDefault(cfg.Refused, 5*time.Minute)
@@ -87,21 +71,21 @@ func Watch(ctx context.Context, cfg ReadyConfig) error {
 	last := ""
 	for {
 		callCtx, cancel := context.WithTimeout(ctx, callTimeout)
-		st, err := readyOnce(callCtx, cfg, d, keys)
+		err := cfg.Checker.Check(callCtx)
 		cancel()
 		delay, state := refresh, ""
 		if err == nil {
-			err = writeReady(cfg.RunDir, st)
+			err = writeReady(cfg.RunDir)
 		}
 		switch {
 		case ctx.Err() != nil:
 			return ctx.Err()
 		case err == nil:
-			state = fmt.Sprintf("store at %s answered the worker key (client-authenticated only; the store server is not authenticated)", st.Address)
+			state = "local SQS store is serving JetStream"
 		default:
 			_ = os.Remove(marker)
 			delay = cfg.Wait
-			if !pending(err) && !retryable(err) && !errors.Is(err, errStorePending) {
+			if !pending(err) && !retryable(err) {
 				delay = refused
 			}
 			state = fmt.Sprintf("store not ready: %v", err)
@@ -126,37 +110,12 @@ func orDefault(d, def time.Duration) time.Duration {
 	return d
 }
 
-var errStorePending = errors.New("the store is not yet bound")
-
-func readyOnce(ctx context.Context, cfg ReadyConfig, d StoreDescriber, keys workerkey.Store) (Store, error) {
-	st, err := d.DescribeStore(ctx)
-	if err != nil {
-		return Store{}, err
-	}
-	if !st.Bound {
-		return Store{}, errStorePending
-	}
-	kp, err := keys.ForExistingStore(st.PublicKey)
-	if err != nil {
-		if errors.Is(err, workerkey.ErrKeyMismatch) {
-			return Store{}, fmt.Errorf("%w: %w", ErrStoreKeyMismatch, err)
-		}
-		return Store{}, err
-	}
-	defer kp.Wipe()
-	if err := cfg.Checker.Check(ctx, st.Address, kp); err != nil {
-		return Store{}, err
-	}
-	return st, nil
-}
-
-// writeReady publishes only the address and public key, never the seed.
-func writeReady(runDir string, st Store) error {
+// writeReady publishes that the local store answered; it holds no secret.
+func writeReady(runDir string) error {
 	if err := os.MkdirAll(runDir, 0o750); err != nil {
 		return fmt.Errorf("create %s: %w", runDir, err)
 	}
-	body := "address=" + st.Address + "\npublic=" + st.PublicKey + "\n"
-	if err := os.WriteFile(filepath.Join(runDir, readyFile), []byte(body), 0o600); err != nil {
+	if err := os.WriteFile(filepath.Join(runDir, readyFile), []byte("ready\n"), 0o600); err != nil {
 		return fmt.Errorf("publish store readiness: %w", err)
 	}
 	return nil

spinifex/agents/sqs/workerboot/ready_test.go

@@ -1,438 +0,0 @@
-package workerboot
-
-import (
-	"context"
-	"errors"
-	"net"
-	"os"
-	"path/filepath"
-	"slices"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/nats-io/nats-server/v2/server"
-	"github.com/nats-io/nkeys"
-)
-
-// describer is the gateway's DescribeStore, answering from a script.
-type describer struct {
-	answers []describeAnswer
-	calls   int
-}
-
-type describeAnswer struct {
-	st  Store
-	err error
-}
-
-func (d *describer) DescribeStore(context.Context) (Store, error) {
-	d.calls++
-	a := d.answers[min(d.calls, len(d.answers))-1]
-	return a.st, a.err
-}
-
-type checker struct {
-	errs  []error
-	calls int
-	seen  []string
-}
-
-func (c *checker) Check(_ context.Context, address string, kp nkeys.KeyPair) error {
-	c.calls++
-	pub, _ := kp.PublicKey()
-	c.seen = append(c.seen, address+"|"+pub)
-	if len(c.errs) > 0 {
-		err := c.errs[0]
-		c.errs = c.errs[1:]
-		return err
-	}
-	return nil
-}
-
-// bootedGuest has run the boot step, so its disk holds a recorded key.
-func bootedGuest(t *testing.T) (*guest, string) {
-	t.Helper()
-	g := newGuest(t)
-	g.contract()
-	pub, err := g.boot()
-	if err != nil {
-		t.Fatal(err)
-	}
-	return g, pub
-}
-
-func (g *guest) readyCfg(d StoreDescriber, c Checker) ReadyConfig {
-	return ReadyConfig{
-		Config:       g.cfg,
-		NewDescriber: func(string, string) (StoreDescriber, error) { return d, nil },
-		Checker:      c,
-	}
-}
-
-// hooked asks stop before each describe, after the previous pass has written
-// or removed the marker, and ends the watch when stop says so.
-type hooked struct {
-	d      StoreDescriber
-	marker string
-	stop   func(call int, ready bool) bool
-	cancel context.CancelFunc
-	calls  int
-}
-
-func (h *hooked) DescribeStore(ctx context.Context) (Store, error) {
-	h.calls++
-	_, err := os.Stat(h.marker)
-	if h.stop(h.calls, err == nil) {
-		h.cancel()
-		return Store{}, context.Canceled
-	}
-	return h.d.DescribeStore(ctx)
-}
-
-// watch runs Watch with millisecond cadences until stop ends it, then checks
-// the marker was removed on the way out.
-func (g *guest) watch(cfg ReadyConfig, stop func(call int, ready bool) bool) error {
-	g.t.Helper()
-	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
-	defer cancel()
-	inner := cfg.NewDescriber
-	cfg.NewDescriber = func(region, url string) (StoreDescriber, error) {
-		d, err := inner(region, url)
-		if err != nil {
-			return nil, err
-		}
-		return &hooked{d: d, marker: filepath.Join(cfg.RunDir, readyFile), stop: stop, cancel: cancel}, nil
-	}
-	for _, p := range []*time.Duration{&cfg.Wait, &cfg.Refresh, &cfg.Refused} {
-		if *p == 0 {
-			*p = time.Millisecond
-		}
-	}
-	err := Watch(ctx, cfg)
-	if errors.Is(ctx.Err(), context.DeadlineExceeded) {
-		g.t.Fatal("the watch never reached its stop condition")
-	}
-	g.assertNotReady()
-	return err
-}
-
-func bound(pub string) describeAnswer {
-	return describeAnswer{st: Store{Bound: true, Address: "10.240.0.9", PublicKey: pub}}
-}
-
-func (g *guest) marker() string {
-	g.t.Helper()
-	b, _ := os.ReadFile(filepath.Join(g.cfg.RunDir, readyFile))
-	return string(b)
-}
-
-// A store launched long after the worker booted, far past the old 15-minute
-// bound of 180 attempts, still becomes ready; nothing is published before.
-func TestWatch_StoreLaunchedLateBecomesReady(t *testing.T) {
-	g, pub := bootedGuest(t)
-	answers := make([]describeAnswer, 201)
-	answers[200] = bound(pub)
-	d := &describer{answers: answers}
-	c := &checker{errs: []error{&StoreUnreachableError{Err: errors.New("connection refused")}}}
-	published := ""
-	err := g.watch(g.readyCfg(d, c), func(call int, ready bool) bool {
-		if ready {
-			published = g.marker()
-			return true
-		}
-		if call > 202 {
-			t.Fatalf("not ready by call %d", call)
-		}
-		return false
-	})
-	if !errors.Is(err, context.Canceled) {
-		t.Fatalf("watch = %v", err)
-	}
-	if d.calls != 202 || c.calls != 2 || c.seen[0] != "10.240.0.9|"+pub {
-		t.Fatalf("describe %d, check %d, seen %v", d.calls, c.calls, c.seen)
-	}
-	if published != "address=10.240.0.9\npublic="+pub+"\n" {
-		t.Fatalf("published %q", published)
-	}
-	notReady := 0
-	for _, l := range g.logs {
-		if strings.HasPrefix(l, "store not ready") {
-			notReady++
-		}
-	}
-	if notReady > 2 {
-		t.Fatalf("%d not-ready lines for two distinct states", notReady)
-	}
-}
-
-// After a pass, the marker goes as soon as a later check fails, and returns
-// once the store answers again.
-func TestWatch_LossAfterReadyRemovesTheMarker(t *testing.T) {
-	cases := map[string]func(pub string) (*describer, *checker){
-		"store stops answering": func(pub string) (*describer, *checker) {
-			return &describer{answers: []describeAnswer{bound(pub)}},
-				&checker{errs: []error{nil, &StoreUnreachableError{Err: errors.New("connection refused")}, nil}}
-		},
-		"gateway reply lost": func(pub string) (*describer, *checker) {
-			return &describer{answers: []describeAnswer{bound(pub), {err: transient{}}, bound(pub)}}, &checker{}
-		},
-		"store no longer bound": func(pub string) (*describer, *checker) {
-			return &describer{answers: []describeAnswer{bound(pub), {}, bound(pub)}}, &checker{}
-		},
-		"store now refuses the key": func(pub string) (*describer, *checker) {
-			return &describer{answers: []describeAnswer{bound(pub)}}, &checker{errs: []error{nil, ErrStoreRefusedKey, nil}}
-		},
-	}
-	for name, mk := range cases {
-		t.Run(name, func(t *testing.T) {
-			g, pub := bootedGuest(t)
-			d, c := mk(pub)
-			var seen []bool
-			if err := g.watch(g.readyCfg(d, c), func(call int, ready bool) bool {
-				seen = append(seen, ready)
-				return call == 4
-			}); !errors.Is(err, context.Canceled) {
-				t.Fatalf("watch = %v", err)
-			}
-			if want := []bool{false, true, false, true}; !slices.Equal(seen, want) {
-				t.Fatalf("marker before each pass = %v, want %v", seen, want)
-			}
-		})
-	}
-}
-
-// A marker left by an earlier run is cleared before the first pass.
-func TestWatch_ClearsAStaleMarker(t *testing.T) {
-	g, pub := bootedGuest(t)
-	mustWrite(t, filepath.Join(g.cfg.RunDir, readyFile), "address=10.0.0.1\n")
-	d := &describer{answers: []describeAnswer{{}}}
-	_ = g.watch(g.readyCfg(d, &checker{}), func(call int, ready bool) bool {
-		if call == 1 && ready {
-			t.Fatal("a stale marker survived into the first pass")
-		}
-		return call == 2
-	})
-	_ = pub
-}
-
-// Failures that cannot clear by themselves wait the Refused cadence: in the
-// window, each is tried once and never published.
-func TestWatch_FinalFailuresDoNotSpin(t *testing.T) {
-	other, _ := nkeys.CreateUser()
-	otherPub, _ := other.PublicKey()
-	cases := map[string]struct {
-		d      func(pub string) *describer
-		c      *checker
-		checks int
-	}{
-		"store refuses the key": {func(pub string) *describer { return &describer{answers: []describeAnswer{bound(pub)}} },
-			&checker{errs: []error{ErrStoreRefusedKey}}, 1},
-		"store launched for another key": {func(string) *describer { return &describer{answers: []describeAnswer{bound(otherPub)}} },
-			&checker{}, 0},
-		"unclassified gateway refusal": {func(string) *describer {
-			return &describer{answers: []describeAnswer{{err: errors.New("InvalidParameterValue")}}}
-		}, &checker{}, 0},
-	}
-	for name, tc := range cases {
-		t.Run(name, func(t *testing.T) {
-			g, pub := bootedGuest(t)
-			g.cfg.Wait = time.Millisecond
-			d := tc.d(pub)
-			cfg := g.readyCfg(d, tc.c)
-			cfg.Refresh, cfg.Refused = time.Millisecond, time.Hour
-			ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
-			defer cancel()
-			if err := Watch(ctx, cfg); !errors.Is(err, context.DeadlineExceeded) {
-				t.Fatalf("watch = %v", err)
-			}
-			if d.calls != 1 || tc.c.calls != tc.checks {
-				t.Fatalf("describe %d, check %d in the window", d.calls, tc.c.calls)
-			}
-			g.assertNotReady()
-		})
-	}
-}
-
-// A mismatched store is asked again at the Refused cadence, so a store the
-// operator relaunches for this worker's key is picked up.
-func TestWatch_RecoversWhenTheStoreIsRelaunchedForThisKey(t *testing.T) {
-	g, pub := bootedGuest(t)
-	other, _ := nkeys.CreateUser()
-	otherPub, _ := other.PublicKey()
-	d := &describer{answers: []describeAnswer{bound(otherPub), bound(pub)}}
-	c := &checker{}
-	if err := g.watch(g.readyCfg(d, c), func(_ int, ready bool) bool { return ready }); !errors.Is(err, context.Canceled) {
-		t.Fatalf("watch = %v", err)
-	}
-	if d.calls != 2 || c.calls != 1 {
-		t.Fatalf("describe %d, check %d", d.calls, c.calls)
-	}
-}
-
-// A key missing from the disk is never replaced, however often it is asked.
-func TestWatch_LostKeyIsNeverReplaced(t *testing.T) {
-	g, pub := bootedGuest(t)
-	dir := filepath.Join(g.cfg.DataDir, keyDirName)
-	entries, _ := os.ReadDir(dir)
-	for _, e := range entries {
-		if err := os.Remove(filepath.Join(dir, e.Name())); err != nil {
-			t.Fatal(err)
-		}
-	}
-	d := &describer{answers: []describeAnswer{bound(pub)}}
-	c := &checker{}
-	_ = g.watch(g.readyCfg(d, c), func(call int, ready bool) bool {
-		if ready {
-			t.Fatal("ready without a key")
-		}
-		return call == 5
-	})
-	if c.calls != 0 {
-		t.Fatalf("checked %d times without a key", c.calls)
-	}
-	if after, _ := os.ReadDir(dir); len(after) != 0 {
-		t.Fatal("a replacement key was written")
-	}
-}
-
-// Readiness never runs on a disk that is not proven mounted.
-func TestWatch_RefusesWithoutTheMountedWorkerDisk(t *testing.T) {
-	g, pub := bootedGuest(t)
-	g.resetMounts()
-	g.disk.noopMount = true
-	d := &describer{answers: []describeAnswer{bound(pub)}}
-	if err := Watch(context.Background(), g.readyCfg(d, &checker{})); err == nil || d.calls != 0 {
-		t.Fatalf("err = %v, describe %d", err, d.calls)
-	}
-}
-
-// After a restart the same key reaches readiness again with no new key.
-func TestWatch_SurvivesARestart(t *testing.T) {
-	g, pub := bootedGuest(t)
-	for range 2 {
-		d := &describer{answers: []describeAnswer{bound(pub)}}
-		if err := g.watch(g.readyCfg(d, &checker{}), func(_ int, ready bool) bool { return ready }); !errors.Is(err, context.Canceled) {
-			t.Fatal(err)
-		}
-		g.resetMounts()
-		again, err := g.boot()
-		if err != nil || again != pub {
-			t.Fatalf("reboot = %q %v", again, err)
-		}
-	}
-}
-
-func (g *guest) assertNotReady() {
-	g.t.Helper()
-	if _, err := os.Stat(filepath.Join(g.cfg.RunDir, readyFile)); !os.IsNotExist(err) {
-		g.t.Fatalf("readiness was published: %v", err)
-	}
-}
-
-func startStore(t *testing.T, pub string, port int) *server.Server {
-	t.Helper()
-	opts := &server.Options{Host: "127.0.0.1", Port: port, JetStream: true, StoreDir: t.TempDir(), NoLog: true,
-		NoSigs: true, Nkeys: []*server.NkeyUser{{Nkey: pub}}}
-	ns, err := server.NewServer(opts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	go ns.Start()
-	if !ns.ReadyForConnections(10 * time.Second) {
-		t.Fatal("store server did not start")
-	}
-	t.Cleanup(ns.Shutdown)
-	return ns
-}
-
-// End to end on the guest with a real store: not ready before it serves,
-// ready once it does, not ready again once it stops, and the marker never
-// holds the seed.
-func TestWatch_RealStoreStartsLateThenStops(t *testing.T) {
-	g, pub := bootedGuest(t)
-	seedFile, _ := os.ReadDir(filepath.Join(g.cfg.DataDir, keyDirName))
-	key, _ := os.ReadFile(filepath.Join(g.cfg.DataDir, keyDirName, seedFile[0].Name()))
-	l, _ := net.Listen("tcp", "127.0.0.1:0")
-	port := l.Addr().(*net.TCPAddr).Port
-	_ = l.Close()
-	d := &describer{answers: []describeAnswer{{st: Store{Bound: true, Address: "127.0.0.1", PublicKey: pub}}}}
-	cfg := g.readyCfg(d, NATSChecker{Port: port, Dial: time.Second})
-	var ns *server.Server
-	phase := 0
-	err := g.watch(cfg, func(call int, ready bool) bool {
-		switch {
-		case phase == 0 && call == 3:
-			if ready {
-				t.Fatal("ready before the store served")
-			}
-			ns, phase = startStore(t, pub, port), 1
-		case phase == 1 && ready:
-			if strings.Contains(g.marker(), strings.TrimSpace(string(key))) {
-				t.Fatal("the seed was published")
-			}
-			ns.Shutdown()
-			ns.WaitForShutdown()
-			phase = 2
-		case phase == 2:
-			if ready {
-				t.Fatal("still ready after the store stopped")
-			}
-			return true
-		}
-		return false
-	})
-	if !errors.Is(err, context.Canceled) || phase != 2 {
-		t.Fatalf("watch = %v, phase %d", err, phase)
-	}
-}
-
-// storeServer is a real NATS server with JetStream that admits only pub.
-func storeServer(t *testing.T, pub string) int {
-	t.Helper()
-	opts := &server.Options{
-		Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true,
-		Nkeys: []*server.NkeyUser{{Nkey: pub}},
-	}
-	ns, err := server.NewServer(opts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	go ns.Start()
-	if !ns.ReadyForConnections(10 * time.Second) {
-		t.Fatal("store server did not start")
-	}
-	t.Cleanup(ns.Shutdown)
-	return ns.Addr().(*net.TCPAddr).Port
-}
-
-// The real checker passes only with the admitted key; a wrong key is refused
-// as final, and nothing listening is retryable.
-func TestNATSChecker_AgainstARealStore(t *testing.T) {
-	kp, _ := nkeys.CreateUser()
-	pub, _ := kp.PublicKey()
-	port := storeServer(t, pub)
-	c := NATSChecker{Port: port, Dial: 2 * time.Second}
-	if err := c.Check(context.Background(), "127.0.0.1", kp); err != nil {
-		t.Fatalf("admitted key: %v", err)
-	}
-	other, _ := nkeys.CreateUser()
-	err := c.Check(context.Background(), "127.0.0.1", other)
-	if !errors.Is(err, ErrStoreRefusedKey) || retryable(err) {
-		t.Fatalf("wrong key: %v", err)
-	}
-	seed, _ := kp.Seed()
-	if strings.Contains(err.Error(), string(seed)) {
-		t.Fatal("the error echoes a seed")
-	}
-	l, _ := net.Listen("tcp", "127.0.0.1:0")
-	closed := l.Addr().(*net.TCPAddr).Port
-	_ = l.Close()
-	err = NATSChecker{Port: closed, Dial: time.Second}.Check(context.Background(), "127.0.0.1", kp)
-	if !retryable(err) {
-		t.Fatalf("nothing listening: %v is not retryable", err)
-	}
-	if err := c.Check(context.Background(), "store.example", kp); err == nil || retryable(err) {
-		t.Fatalf("hostname accepted: %v", err)
-	}
-}

spinifex/agents/sqs/workerboot/serve.go

@@ -0,0 +1,114 @@
+package workerboot
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/nats-io/nats.go"
+	"github.com/nats-io/nats.go/jetstream"
+
+	"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerkey"
+)
+
+// ServeConfig makes the ready step also serve SQS owner calls.
+type ServeConfig struct {
+	Connector Connector
+	// Run serves until ctx ends against the store's JetStream, with the
+	// receipt key from the worker disk; it must not retain the key.
+	Run func(ctx context.Context, region, gatewayURL string, js jetstream.JetStream, receiptKey []byte) error
+}
+
+// Connector opens the guest's local JetStream.
+type Connector interface {
+	Connect(ctx context.Context) (jetstream.JetStream, func(), error)
+}
+
+// NATSConnector connects to the guest's loopback SQS NATS server.
+type NATSConnector struct {
+	// Port overrides StorePort, for tests against a local server.
+	Port int
+}
+
+var _ Connector = NATSConnector{}
+
+// ErrUnsupportedTopology refuses serving, and so creating any store state,
+// unless the gateway reports a single-node host cluster.
+var ErrUnsupportedTopology = errors.New("SQS serving supports only a single-node host cluster")
+
+// ErrReceiptPending refuses serving while the platform has not recorded the
+// receipt key; the boot step records it before the store first starts.
+var ErrReceiptPending = errors.New("the receipt key is not yet recorded")
+
+func (c NATSConnector) Connect(_ context.Context) (jetstream.JetStream, func(), error) {
+	nc, err := nats.Connect(localURL(c.Port), nats.Timeout(5*time.Second), nats.MaxReconnects(-1),
+		nats.Name("spinifex-sqs-worker"))
+	if err != nil {
+		return nil, nil, &StoreUnreachableError{Err: err}
+	}
+	js, err := jetstream.New(nc)
+	if err != nil {
+		nc.Close()
+		return nil, nil, &StoreUnreachableError{Err: err}
+	}
+	return js, nc.Close, nil
+}
+
+// serve runs the owners until ctx ends, starting again after any failure to
+// open them. It never creates a key: the receipt key must be on the disk.
+func serve(ctx context.Context, cfg ReadyConfig, c contract, keys workerkey.Store) {
+	last := ""
+	for ctx.Err() == nil {
+		err := serveOnce(ctx, cfg, c, keys)
+		if ctx.Err() != nil {
+			return
+		}
+		if state := fmt.Sprintf("SQS serving stopped: %v", err); state != last {
+			cfg.Log(state)
+			last = state
+		}
+		delay := cfg.Wait
+		if !pending(err) && !retryable(err) && !errors.Is(err, ErrReceiptPending) {
+			delay = orDefault(cfg.Refused, 5*time.Minute)
+		}
+		select {
+		case <-ctx.Done():
+		case <-time.After(max(delay, time.Millisecond)):
+		}
+	}
+}
+
+func serveOnce(ctx context.Context, cfg ReadyConfig, c contract, keys workerkey.Store) error {
+	callCtx, cancel := context.WithTimeout(ctx, orDefault(cfg.CallTimeout, 30*time.Second))
+	defer cancel()
+	if cfg.NewClaimer == nil {
+		return errors.New("no gateway client is configured")
+	}
+	claimer, err := cfg.NewClaimer(c.region, c.gatewayURL)
+	if err != nil {
+		return fmt.Errorf("build gateway client: %w", err)
+	}
+	d, err := claimer.Claim(callCtx)
+	if err != nil {
+		return err
+	}
+	if d.HostNodes != 1 {
+		return fmt.Errorf("%w: the gateway reports %d host nodes", ErrUnsupportedTopology, d.HostNodes)
+	}
+	if d.Pending {
+		return ErrReceiptPending
+	}
+	receipt, err := keys.ForRecorded(d.Fingerprint)
+	if err != nil {
+		return err
+	}
+	defer clear(receipt)
+	js, closeConn, err := cfg.Serve.Connector.Connect(callCtx)
+	if err != nil {
+		return err
+	}
+	defer closeConn()
+	cfg.Log("serving SQS owner calls from the local store")
+	return cfg.Serve.Run(ctx, c.region, c.gatewayURL, js, receipt)
+}

spinifex/agents/sqs/workerboot/serve_test.go

@@ -0,0 +1,434 @@
+package workerboot
+
+import (
+	"context"
+	"errors"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/sqs"
+	sqstypes "github.com/aws/aws-sdk-go-v2/service/sqs/types"
+	"github.com/mulgadc/bluebottle/pkg/iampolicy"
+	"github.com/nats-io/nats-server/v2/server"
+	"github.com/nats-io/nats.go/jetstream"
+
+	"github.com/mulgadc/spinifex/internal/gwsign"
+	"github.com/mulgadc/spinifex/internal/sqsworkergw"
+	"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerkey"
+	"github.com/mulgadc/spinifex/spinifex/domains/sqs/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/domains/sqs/relay"
+	"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerident"
+	"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerserve"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
+	"github.com/mulgadc/spinifex/spinifex/ingress/aws/envelope"
+)
+
+const tenantAccount = "123456789012"
+
+// publicSQS serves the public SQS registration over the relay, as the
+// ingress would after authenticating the tenant; tenant SigV4 and IAM are
+// not under test here.
+func publicSQS(t *testing.T, r *relay.Relay) *sqs.Client {
+	t.Helper()
+	var reg dispatch.Registration
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		err := reg.Dispatch(w, dispatch.Invocation{
+			Request: req, AccountID: tenantAccount, Region: testRegion, Service: awsapi.ServiceName,
+			Authorize: func(string, string, []string, iampolicy.ConditionKeys) error { return nil },
+			Actor: func() (dispatch.Actor, error) {
+				return dispatch.Actor{UserID: "AIDAPOLLER", AccountID: tenantAccount}, nil
+			},
+		})
+		if err != nil {
+			code, msg, _ := awserrors.ResolveErrorDetail(err)
+			if code == "" {
+				code = err.Error()
+			}
+			meta := awserrors.LookupErrorMessage(awsapi.ServiceName, code)
+			if msg == "" {
+				msg = meta.Message
+			}
+			qc := code
+			if q, ok := reg.QueryErrorCodes[code]; ok {
+				qc = q
+			}
+			envelope.WriteJSON(w, envelope.ProtocolJSON10Query, envelope.JSONError{
+				Code: code, Verbatim: true, Message: msg, Status: meta.HTTPCode, Namespace: reg.ErrorNamespace, QueryCode: qc,
+			})
+		}
+	}))
+	t.Cleanup(srv.Close)
+	reg = awsapi.NewRegistration(awsapi.Deps{
+		Queues: r, Messages: r, Receives: r, Deletes: r, Visibility: r, Endpoint: srv.URL,
+	})
+	return sqs.New(sqs.Options{
+		BaseEndpoint: aws.String(srv.URL), Region: testRegion, RetryMaxAttempts: 1,
+		Credentials: aws.AnonymousCredentials{},
+	})
+}
+
+// localStore is the guest's own SQS NATS server: loopback only, no auth,
+// JetStream kept under the worker disk so a restart finds its data.
+func localStore(t *testing.T, g *guest, port int) *server.Server {
+	t.Helper()
+	if port == 0 {
+		port = -1
+	}
+	ns, err := server.NewServer(&server.Options{
+		Host: "127.0.0.1", Port: port, JetStream: true, StoreDir: g.cfg.DataDir, NoLog: true, NoSigs: true,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	go ns.Start()
+	if !ns.ReadyForConnections(10 * time.Second) {
+		t.Fatal("store server did not start")
+	}
+	t.Cleanup(func() { ns.Shutdown(); ns.WaitForShutdown() })
+	return ns
+}
+
+func storePort(ns *server.Server) int { return ns.Addr().(*net.TCPAddr).Port }
+
+func freePort(t *testing.T) int {
+	t.Helper()
+	l, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer l.Close()
+	return l.Addr().(*net.TCPAddr).Port
+}
+
+// servingWorker is the guest's ready step serving through the local
+// gateway's signed work actions against the local store on port.
+type servingWorker struct {
+	cancel context.CancelFunc
+	done   chan struct{}
+	once   sync.Once
+	mu     sync.Mutex
+	logs   []string
+	g      *guest
+}
+
+func (w *servingWorker) log(msg string) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	w.logs = append(w.logs, msg)
+}
+
+func (w *servingWorker) logged(substr string) bool {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	return slices.ContainsFunc(w.logs, func(l string) bool { return strings.Contains(l, substr) })
+}
+
+func (w *servingWorker) ready() bool {
+	_, err := os.Stat(filepath.Join(w.g.cfg.RunDir, readyFile))
+	return err == nil
+}
+
+func (c gwCluster) startServing(t *testing.T, g *guest, signer *gwsign.Signer, port int) *servingWorker {
+	t.Helper()
+	cfg := ReadyConfig{Config: g.cfg, Checker: NATSChecker{Port: port, Dial: time.Second}}
+	cfg.Serve = &ServeConfig{
+		Connector: NATSConnector{Port: port},
+		Run: func(ctx context.Context, region, url string, js jetstream.JetStream, receiptKey []byte) error {
+			gw, err := sqsworkergw.New(url, region, c.gw.caPEM(), signer)
+			if err != nil {
+				return err
+			}
+			return workerserve.Run(ctx, gw, js, receiptKey, cfg.Log)
+		},
+	}
+	cfg.Wait, cfg.Refresh, cfg.Refused, cfg.CallTimeout = 20*time.Millisecond, 50*time.Millisecond, 50*time.Millisecond, 5*time.Second
+	ctx, cancel := context.WithCancel(context.Background())
+	w := &servingWorker{cancel: cancel, done: make(chan struct{}), g: g}
+	cfg.Log = w.log
+	go func() {
+		defer close(w.done)
+		_ = Watch(ctx, cfg)
+	}()
+	t.Cleanup(w.stop)
+	return w
+}
+
+func (w *servingWorker) stop() {
+	w.once.Do(w.cancel)
+	<-w.done
+}
+
+// eventually repeats call until it succeeds.
+func eventually(t *testing.T, call func() error) {
+	t.Helper()
+	deadline := time.Now().Add(15 * time.Second)
+	for {
+		err := call()
+		if err == nil {
+			return
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("never succeeded: %v", err)
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+}
+
+// bootedWorker launches a worker on a new volume and boots its guest, which
+// creates and records the receipt key; it returns the guest and its signer.
+func (c gwCluster) bootedWorker(t *testing.T) (*guest, *gwsign.Signer, workerident.Intent) {
+	t.Helper()
+	in := c.launch(t, gwInstance)
+	g := newGuest(t)
+	signer := c.guestFor(t, g, "ASIAWORKER1", gwInstance)
+	if _, err := g.boot(); err != nil {
+		t.Fatal(err)
+	}
+	return g, signer, in
+}
+
+// The SDK path through one SQS VM: the worker serves create, send, receive
+// and delete from its local store; after the store and worker restart on the
+// same disk the queue is still there and an earlier handle still works.
+func TestServe_SDKPathThroughTheWorkerSurvivesRestart(t *testing.T) {
+	r := &relay.Relay{}
+	c := newGWCluster(t, r, nil)
+	g, signer, _ := c.bootedWorker(t)
+	ns := localStore(t, g, 0)
+	port := storePort(ns)
+	w := c.startServing(t, g, signer, port)
+	cl := publicSQS(t, r)
+	ctx := context.Background()
+
+	var url string
+	eventually(t, func() error {
+		out, err := cl.CreateQueue(ctx, &sqs.CreateQueueInput{QueueName: aws.String("orders")})
+		if err == nil {
+			url = aws.ToString(out.QueueUrl)
+		}
+		return err
+	})
+	body := strings.Repeat(`"quoted" \ body `, 4096)
+	sent, err := cl.SendMessage(ctx, &sqs.SendMessageInput{
+		QueueUrl: aws.String(url), MessageBody: aws.String(body),
+		MessageAttributes: map[string]sqstypes.MessageAttributeValue{
+			"kind": {DataType: aws.String("String"), StringValue: aws.String("new\norder")},
+			"blob": {DataType: aws.String("Binary"), BinaryValue: []byte{0, 1, 2, 255}},
+		},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := cl.ReceiveMessage(ctx, &sqs.ReceiveMessageInput{
+		QueueUrl: aws.String(url), MaxNumberOfMessages: 10, WaitTimeSeconds: 2,
+		MessageAttributeNames: []string{"All"}, VisibilityTimeout: 60,
+	})
+	if err != nil || len(got.Messages) != 1 {
+		t.Fatalf("receive = %v, %v", got, err)
+	}
+	m := got.Messages[0]
+	if aws.ToString(m.MessageId) != aws.ToString(sent.MessageId) || aws.ToString(m.Body) != body ||
+		aws.ToString(m.MD5OfMessageAttributes) != aws.ToString(sent.MD5OfMessageAttributes) ||
+		aws.ToString(m.MessageAttributes["kind"].StringValue) != "new\norder" {
+		t.Fatalf("received a different message: %+v", m)
+	}
+	if _, err := cl.DeleteMessage(ctx, &sqs.DeleteMessageInput{QueueUrl: aws.String(url), ReceiptHandle: m.ReceiptHandle}); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := cl.SendMessage(ctx, &sqs.SendMessageInput{QueueUrl: aws.String(url), MessageBody: aws.String("second")}); err != nil {
+		t.Fatal(err)
+	}
+	held, err := cl.ReceiveMessage(ctx, &sqs.ReceiveMessageInput{QueueUrl: aws.String(url), WaitTimeSeconds: 2, VisibilityTimeout: 60})
+	if err != nil || len(held.Messages) != 1 {
+		t.Fatalf("receive before restart = %v, %v", held, err)
+	}
+
+	w.stop()
+	ns.Shutdown()
+	ns.WaitForShutdown()
+	g.resetMounts()
+	if _, err := g.boot(); err != nil {
+		t.Fatalf("reboot: %v", err)
+	}
+	localStore(t, g, port)
+	c.startServing(t, g, signer, port)
+	eventually(t, func() error {
+		_, err := cl.GetQueueUrl(ctx, &sqs.GetQueueUrlInput{QueueName: aws.String("orders")})
+		return err
+	})
+	if _, err := cl.DeleteMessage(ctx, &sqs.DeleteMessageInput{QueueUrl: aws.String(url), ReceiptHandle: held.Messages[0].ReceiptHandle}); err != nil {
+		t.Fatalf("handle from before the restart: %v", err)
+	}
+	empty, err := cl.ReceiveMessage(ctx, &sqs.ReceiveMessageInput{QueueUrl: aws.String(url)})
+	if err != nil || len(empty.Messages) != 0 {
+		t.Fatalf("queue after delete = %v, %v", empty, err)
+	}
+
+	key, err := workerkey.Store{Dir: g.keyDir()}.ReceiptKey()
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, b := range c.gw.sentBodies() {
+		if strings.Contains(b, string(key)) {
+			t.Fatal("a request carried key material")
+		}
+	}
+}
+
+// The store comes up before the worker serves: while the local store is not
+// answering the guest is not ready and no call is served; once it starts,
+// readiness and serving follow.
+func TestServe_UnavailableStoreIsNeverReady(t *testing.T) {
+	r := &relay.Relay{CallLimit: time.Second, PollerStale: 200 * time.Millisecond}
+	c := newGWCluster(t, r, nil)
+	g, signer, _ := c.bootedWorker(t)
+	port := freePort(t)
+	w := c.startServing(t, g, signer, port)
+	cl := publicSQS(t, r)
+	ctx := context.Background()
+	eventually(t, func() error {
+		if !w.logged("store not ready") {
+			return errors.New("no failed check yet")
+		}
+		return nil
+	})
+	if w.ready() {
+		t.Fatal("ready without a store")
+	}
+	if _, err := cl.ListQueues(ctx, &sqs.ListQueuesInput{}); err == nil {
+		t.Fatal("a call was served without a store")
+	}
+	localStore(t, g, port)
+	eventually(t, func() error {
+		_, err := cl.ListQueues(ctx, &sqs.ListQueuesInput{})
+		return err
+	})
+	eventually(t, func() error {
+		if !w.ready() {
+			return errors.New("not ready")
+		}
+		return nil
+	})
+}
+
+// A worker whose recorded receipt key is gone, or replaced, never serves and
+// never makes a new key, so handles issued earlier are not invalidated.
+func TestServe_ReceiptKeyLossOrChangeNeverServes(t *testing.T) {
+	for _, tc := range []struct {
+		name   string
+		mutate func(t *testing.T, path string)
+		want   error
+	}{
+		{"missing", func(t *testing.T, path string) {
+			if err := os.Remove(path); err != nil {
+				t.Fatal(err)
+			}
+		}, workerkey.ErrReceiptKeyMissing},
+		{"changed", func(t *testing.T, path string) {
+			if err := os.WriteFile(path, []byte(strings.Repeat("k", workerkey.ReceiptKeyLen)), 0o600); err != nil {
+				t.Fatal(err)
+			}
+		}, workerkey.ErrKeyMismatch},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			r := &relay.Relay{CallLimit: time.Second, PollerStale: 200 * time.Millisecond}
+			c := newGWCluster(t, r, nil)
+			g, signer, _ := c.bootedWorker(t)
+			port := storePort(localStore(t, g, 0))
+			w := c.startServing(t, g, signer, port)
+			cl := publicSQS(t, r)
+			eventually(t, func() error {
+				_, err := cl.ListQueues(context.Background(), &sqs.ListQueuesInput{})
+				return err
+			})
+			w.stop()
+
+			tc.mutate(t, g.keyFile())
+			again := c.startServing(t, g, signer, port)
+			eventually(t, func() error {
+				if again.logged(tc.want.Error()) {
+					return nil
+				}
+				return errors.New("refusal not logged yet")
+			})
+			if _, err := cl.ListQueues(context.Background(), &sqs.ListQueuesInput{}); err == nil {
+				t.Fatal("a call was served without the recorded receipt key")
+			}
+			g.resetMounts()
+			if _, err := g.boot(); !errors.Is(err, tc.want) {
+				t.Fatalf("reboot = %v, want %v", err, tc.want)
+			}
+			if tc.name == "missing" {
+				if _, err := os.Lstat(g.keyFile()); !errors.Is(err, os.ErrNotExist) {
+					t.Fatal("a replacement receipt key was created")
+				}
+			}
+		})
+	}
+}
+
+// A worker the domain has recorded as stopped cannot take work.
+func TestServe_StoppedWorkerTakesNoWork(t *testing.T) {
+	r := &relay.Relay{CallLimit: time.Second, PollerStale: 200 * time.Millisecond}
+	c := newGWCluster(t, r, nil)
+	g, signer, in := c.bootedWorker(t)
+	w := c.startServing(t, g, signer, storePort(localStore(t, g, 0)))
+	cl := publicSQS(t, r)
+	ctx := context.Background()
+	eventually(t, func() error {
+		_, err := cl.CreateQueue(ctx, &sqs.CreateQueueInput{QueueName: aws.String("orders")})
+		return err
+	})
+	if err := c.ids.RecordStopped(ctx, in, gwInstance); err != nil {
+		t.Fatal(err)
+	}
+	eventually(t, func() error {
+		if _, err := cl.ListQueues(ctx, &sqs.ListQueuesInput{}); err == nil {
+			return errors.New("still served")
+		}
+		return nil
+	})
+	if !w.logged("AccessDenied") {
+		t.Fatal("the worker's polls were not refused")
+	}
+}
+
+// A gateway reporting a multi-node host cluster is refused before the worker
+// opens its store, so no queue or marker state is created.
+func TestServe_MultiNodeHostCreatesNoStoreState(t *testing.T) {
+	c := newGWCluster(t, &relay.Relay{}, func() int { return 3 })
+	g, signer, _ := c.bootedWorker(t)
+	port := storePort(localStore(t, g, 0))
+	w := c.startServing(t, g, signer, port)
+	eventually(t, func() error {
+		if !w.logged(ErrUnsupportedTopology.Error()) {
+			return errors.New("not yet refused")
+		}
+		return nil
+	})
+	w.stop()
+
+	js, closeConn, err := NATSConnector{Port: port}.Connect(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer closeConn()
+	names := js.StreamNames(context.Background())
+	var got []string
+	for n := range names.Name() {
+		got = append(got, n)
+	}
+	if names.Err() != nil || len(got) != 0 {
+		t.Fatalf("store streams = %v, %v; want none", got, names.Err())
+	}
+}

spinifex/agents/sqs/workerboot/workerboot.go

@@ -1,7 +1,7 @@
-// Package workerboot prepares the shared SQS worker guest: it selects exactly
-// the expected data disk by serial, checks and mounts it, proves that disk is
-// what is mounted, and only then loads or creates the worker's store key on it
-// under the current first-store decision the gateway returns.
+// Package workerboot prepares the SQS guest: it selects exactly the expected
+// data disk by serial, checks and mounts it, proves that disk is what is
+// mounted, and only then settles the receipt key under the gateway's decision,
+// before the guest's own SQS store first starts on that disk.
 package workerboot
 
 import (
@@ -18,11 +18,10 @@ import (
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerkey"
-	"github.com/nats-io/nkeys"
 )
 
 // Guest contract, read from fw_cfg under opt/spinifex/sqs-worker/. None of it
-// is a decision: whether a store key may be created is asked of the gateway.
+// is a decision: whether a receipt key may be created is asked of the gateway.
 //
 //	data-serial   virtio-blk serial of the worker's own ext4 data disk
 //	region        the Region the worker signs its gateway calls for
@@ -30,11 +29,15 @@ import (
 const (
 	keyDirName = "sqs-worker-key"
 	maxFwCfg   = 128
+	// StoreDirName is the store's JetStream directory on the data disk. Its
+	// presence is SQS data, so a pending decision never creates a key there.
+	StoreDirName = "jetstream"
+	receiptReady = "receipt-ready"
 )
 
 var regionPattern = regexp.MustCompile(`^[a-z]{2}(-[a-z]+)+-[0-9]+$`)
 
-// NewClaimer builds the gateway client the first-store decision comes from.
+// NewClaimer builds the gateway client the receipt key decision comes from.
 type NewClaimer func(region, gatewayURL string) (workerkey.Claimer, error)
 
 // Disk checks and mounts the worker data disk. The guest never formats.
@@ -67,9 +70,9 @@ type contract struct {
 	serial, region, gatewayURL string
 }
 
-// Boot runs the boot step and returns the worker's public key. The key is
-// loaded or created only under the gateway's current decision; any missing,
-// malformed or ambiguous input or answer refuses.
+// Boot runs the boot step and returns the receipt key's fingerprint. The key
+// is loaded, or created on a new volume, only under the gateway's current
+// decision; any missing, malformed or ambiguous input or answer refuses.
 func Boot(ctx context.Context, cfg Config) (string, error) {
 	c, err := readContract(cfg.FwCfgDir)
 	if err != nil {
@@ -86,20 +89,26 @@ func Boot(ctx context.Context, cfg Config) (string, error) {
 	if err != nil {
 		return "", fmt.Errorf("build gateway client: %w", err)
 	}
-	kp, err := acquire(ctx, cfg, workerkey.Store{Dir: filepath.Join(cfg.DataDir, keyDirName)}, claimer)
+	key, err := acquire(ctx, cfg, workerkey.Store{Dir: filepath.Join(cfg.DataDir, keyDirName)}, claimer)
 	if err != nil {
 		return "", err
 	}
-	defer kp.Wipe()
-	pub, err := kp.PublicKey()
-	if err != nil {
-		return "", fmt.Errorf("derive worker public key: %w", err)
-	}
-	if err := writePublicKey(cfg.RunDir, pub); err != nil {
+	fp := workerkey.Fingerprint(key)
+	clear(key)
+	if err := writeMarker(cfg.RunDir, receiptReady, fp); err != nil {
 		return "", err
 	}
-	cfg.Log("store key ready: public=" + pub)
-	return pub, nil
+	cfg.Log("receipt key ready: fingerprint=" + fp)
+	return fp, nil
+}
+
+// hasStoreData reports whether the disk already holds the store's directory.
+func hasStoreData(dataDir string) (bool, error) {
+	_, err := os.Lstat(filepath.Join(dataDir, StoreDirName))
+	if errors.Is(err, fs.ErrNotExist) {
+		return false, nil
+	}
+	return err == nil, err
 }
 
 // mountWorkerDisk selects the disk with serial, checks and mounts it at
@@ -176,20 +185,21 @@ func readContract(dir string) (contract, error) {
 // acquire runs the decision and key step, repeating it only on an error that
 // says it is safe to repeat, including a denial a not-yet-bound worker gets.
 // A denial creates nothing, and each repeat re-asks the gateway.
-func acquire(ctx context.Context, cfg Config, store workerkey.Store, c workerkey.Claimer) (kp nkeys.KeyPair, err error) {
+func acquire(ctx context.Context, cfg Config, store workerkey.Store, c workerkey.Claimer) (key []byte, err error) {
 	attempts := max(cfg.Attempts, 1)
+	hasData := func() (bool, error) { return hasStoreData(cfg.DataDir) }
 	for i := range attempts {
-		kp, err = store.Acquire(ctx, c)
+		key, err = store.Acquire(ctx, c, hasData)
 		if err == nil || i == attempts-1 {
-			return kp, err
+			return key, err
 		}
 		switch {
 		case pending(err):
 			cfg.Log(fmt.Sprintf("worker not yet authorized, asking again: %v", err))
 		case retryable(err):
-			cfg.Log(fmt.Sprintf("first-store decision not settled, retrying: %v", err))
+			cfg.Log(fmt.Sprintf("receipt key decision not settled, retrying: %v", err))
 		default:
-			return kp, err
+			return key, err
 		}
 		select {
 		case <-ctx.Done():
@@ -200,6 +210,13 @@ func acquire(ctx context.Context, cfg Config, store workerkey.Store, c workerkey
 	return nil, err
 }
 
+// Transient reports whether a failed Boot may be run again: the decision was
+// still pending or retryable when its bound ran out. Every other refusal,
+// including a missing or different key, is final.
+func Transient(err error) bool {
+	return pending(err) || retryable(err) || errors.Is(err, context.DeadlineExceeded)
+}
+
 func pending(err error) bool {
 	p, ok := errors.AsType[interface {
 		error
@@ -333,29 +350,25 @@ func readTrimmed(path string) (string, error) {
 	return strings.TrimSpace(string(b)), nil
 }
 
-// writePublicKey publishes the public key for later reporting; it is not secret.
-func writePublicKey(runDir, pub string) error {
+// writeMarker publishes a non-secret value under runDir atomically.
+func writeMarker(runDir, name, value string) error {
 	if err := os.MkdirAll(runDir, 0o750); err != nil {
 		return fmt.Errorf("create %s: %w", runDir, err)
 	}
-	tmp, err := os.CreateTemp(runDir, ".public-key-*")
+	tmp, err := os.CreateTemp(runDir, "."+name+"-*")
 	if err != nil {
-		return fmt.Errorf("write public key: %w", err)
+		return fmt.Errorf("write %s: %w", name, err)
 	}
 	defer os.Remove(tmp.Name())
-	if _, err := tmp.WriteString(pub + "\n"); err != nil {
-		tmp.Close()
-		return fmt.Errorf("write public key: %w", err)
+	_, werr := tmp.WriteString(value + "\n")
+	if cerr := tmp.Close(); werr == nil {
+		werr = cerr
 	}
-	if err := tmp.Chmod(0o644); err != nil {
-		tmp.Close()
-		return fmt.Errorf("write public key: %w", err)
+	if werr != nil {
+		return fmt.Errorf("write %s: %w", name, werr)
 	}
-	if err := tmp.Close(); err != nil {
-		return fmt.Errorf("write public key: %w", err)
-	}
-	if err := os.Rename(tmp.Name(), filepath.Join(runDir, "public-key")); err != nil {
-		return fmt.Errorf("write public key: %w", err)
+	if err := os.Rename(tmp.Name(), filepath.Join(runDir, name)); err != nil {
+		return fmt.Errorf("write %s: %w", name, err)
 	}
 	return nil
 }

spinifex/agents/sqs/workerboot/workerboot_test.go

@@ -2,6 +2,9 @@ package workerboot
 
 import (
 	"context"
+	"crypto/rand"
+	"encoding/base64"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"os"
@@ -11,7 +14,6 @@ import (
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerkey"
-	"github.com/nats-io/nkeys"
 )
 
 const (
@@ -32,12 +34,12 @@ type guest struct {
 	logs []string
 }
 
-// authority stands for the domain's first-store record behind the gateway:
-// it authorizes creation until a key is recorded, then only that key.
+// authority stands for the worker record behind the gateway: pending until a
+// receipt key fingerprint is recorded, then exactly that fingerprint.
 type authority struct {
 	recorded  string
 	claimErrs []error
-	// recordErrs fail RecordPublicKey; landed ones record before failing,
+	// recordErrs fail RecordFingerprint; landed ones record before failing,
 	// as a reply lost after the write.
 	recordErrs []recordErr
 	claims     int
@@ -63,30 +65,30 @@ type notBoundError struct{}
 func (notBoundError) Error() string { return "AccessDenied" }
 func (notBoundError) Pending() bool { return true }
 
-func (a *authority) Claim(context.Context) (bool, string, error) {
+func (a *authority) Claim(context.Context) (workerkey.Decision, error) {
 	a.claims++
 	if len(a.claimErrs) > 0 {
 		err := a.claimErrs[0]
 		a.claimErrs = a.claimErrs[1:]
-		return false, "", err
+		return workerkey.Decision{}, err
 	}
-	return a.recorded == "", a.recorded, nil
+	return workerkey.Decision{Pending: a.recorded == "", Fingerprint: a.recorded, HostNodes: 1}, nil
 }
 
-func (a *authority) RecordPublicKey(_ context.Context, pub string) error {
-	a.sent = append(a.sent, pub)
-	if a.recorded != "" && a.recorded != pub {
+func (a *authority) RecordFingerprint(_ context.Context, fp string) error {
+	a.sent = append(a.sent, fp)
+	if a.recorded != "" && a.recorded != fp {
 		return errors.New("InvalidParameterValue")
 	}
 	if len(a.recordErrs) > 0 {
 		e := a.recordErrs[0]
 		a.recordErrs = a.recordErrs[1:]
 		if e.landed {
-			a.recorded = pub
+			a.recorded = fp
 		}
 		return e.err
 	}
-	a.recorded = pub
+	a.recorded = fp
 	return nil
 }
 
@@ -188,34 +190,26 @@ func (g *guest) boot() (string, error) {
 	return Boot(context.Background(), g.cfg)
 }
 
-// assertSentOnlyPublicKeys checks nothing but valid public keys left the guest.
-func (g *guest) assertSentOnlyPublicKeys(seed string) {
+// recordKey stands for a receipt key created on this volume earlier and
+// recorded by the platform; it returns that key.
+func (g *guest) recordKey() []byte {
 	g.t.Helper()
-	for _, v := range g.auth.sent {
-		if !nkeys.IsValidPublicUserKey(v) || (seed != "" && strings.Contains(v, seed)) {
-			g.t.Fatal("the guest sent something other than a public key")
-		}
-	}
-}
-
-// seedKey stands for a worker key created when the store was first made and
-// recorded by the domain; it returns that key's public half.
-func (g *guest) seedKey() string {
-	g.t.Helper()
-	kp, err := workerkey.Store{Dir: g.keyDir()}.ForNewStore()
-	if err != nil {
-		g.t.Fatalf("seed worker key: %v", err)
-	}
-	defer kp.Wipe()
-	pub, err := kp.PublicKey()
-	if err != nil {
+	key := make([]byte, workerkey.ReceiptKeyLen)
+	if _, err := rand.Read(key); err != nil {
 		g.t.Fatal(err)
 	}
-	g.auth.recorded = pub
-	return pub
+	mustMkdir(g.t, g.keyDir())
+	if err := os.Chmod(g.keyDir(), 0o700); err != nil {
+		g.t.Fatal(err)
+	}
+	if err := os.WriteFile(g.keyFile(), key, 0o600); err != nil {
+		g.t.Fatal(err)
+	}
+	g.auth.recorded = workerkey.Fingerprint(key)
+	return key
 }
 
-func (g *guest) keyFile() string { return filepath.Join(g.keyDir(), "store-client.nk") }
+func (g *guest) keyFile() string { return filepath.Join(g.keyDir(), "receipt.key") }
 
 // resetMounts is a reboot: only the initramfs root is mounted.
 func (g *guest) resetMounts() {
@@ -235,12 +229,12 @@ func (g *guest) appendMount(majMin, root, dir, fstype, source string) {
 
 func (g *guest) keyDir() string { return filepath.Join(g.cfg.DataDir, keyDirName) }
 
-func (g *guest) seed() string {
-	b, err := os.ReadFile(filepath.Join(g.keyDir(), "store-client.nk"))
+func (g *guest) key() []byte {
+	b, err := os.ReadFile(g.keyFile())
 	if err != nil {
 		g.t.Fatalf("read key: %v", err)
 	}
-	return string(b)
+	return b
 }
 
 func (g *guest) assertNoKey() {
@@ -248,145 +242,130 @@ func (g *guest) assertNoKey() {
 	if _, err := os.Lstat(g.keyDir()); !errors.Is(err, os.ErrNotExist) {
 		g.t.Fatalf("a key directory exists although boot refused: %v", err)
 	}
-	if _, err := os.Lstat(filepath.Join(g.cfg.RunDir, "public-key")); !errors.Is(err, os.ErrNotExist) {
-		g.t.Fatal("a public key was published although boot refused")
+	if _, err := os.Lstat(filepath.Join(g.cfg.RunDir, receiptReady)); !errors.Is(err, os.ErrNotExist) {
+		g.t.Fatal("receipt readiness was published although boot refused")
 	}
 }
 
-func (g *guest) assertNoSeedLeak(err error, seed string) {
+// assertNoKeyLeak checks no error or log line carries the key's bytes.
+func (g *guest) assertNoKeyLeak(err error) {
 	g.t.Helper()
-	if seed == "" {
+	b, rerr := os.ReadFile(g.keyFile())
+	if rerr != nil || len(b) == 0 {
 		return
 	}
-	if err != nil && strings.Contains(err.Error(), seed) {
-		g.t.Fatal("an error exposes the seed")
-	}
-	for _, l := range g.logs {
-		if strings.Contains(l, seed) {
-			g.t.Fatal("a log line exposes the seed")
+	for _, enc := range []string{string(b), hex.EncodeToString(b), base64.StdEncoding.EncodeToString(b)} {
+		if err != nil && strings.Contains(err.Error(), enc) {
+			g.t.Fatal("an error exposes the receipt key")
+		}
+		for _, l := range g.logs {
+			if strings.Contains(l, enc) {
+				g.t.Fatal("a log line exposes the receipt key")
+			}
 		}
 	}
 }
 
-// otherPub is a valid store key the worker does not hold.
-func otherPub(t *testing.T) string {
-	t.Helper()
-	pub, _ := userKey(t)
-	return pub
-}
-
-func userKey(t *testing.T) (pub, seed string) {
-	t.Helper()
-	kp, err := nkeys.CreateUser()
-	if err != nil {
-		t.Fatal(err)
-	}
-	p, err := kp.PublicKey()
-	if err != nil {
-		t.Fatal(err)
-	}
-	s, err := kp.Seed()
-	if err != nil {
-		t.Fatal(err)
-	}
-	return p, string(s)
-}
-
-func TestBoot_FirstStoreCreatesKeyThenReusesItAcrossReboots(t *testing.T) {
+func TestBoot_NewVolumeCreatesKeyThenReusesItAcrossReboots(t *testing.T) {
 	g := newGuest(t)
 	g.contract()
-	pub, err := g.boot()
+	fp, err := g.boot()
 	if err != nil {
 		t.Fatal(err)
 	}
-	seed := g.seed()
-	if g.auth.recorded != pub || len(g.auth.sent) != 1 {
-		t.Fatalf("recorded %q after %d sends, want %s once", g.auth.recorded, len(g.auth.sent), pub)
+	key := g.key()
+	if fp != workerkey.Fingerprint(key) || g.auth.recorded != fp || len(g.auth.sent) != 1 {
+		t.Fatalf("recorded %q after %d sends, want %s once", g.auth.recorded, len(g.auth.sent), fp)
 	}
+	// The store has since written data; a recorded key still reloads.
+	mustMkdir(t, filepath.Join(g.cfg.DataDir, StoreDirName))
 	for boot := 2; boot <= 3; boot++ {
 		g.resetMounts()
 		got, err := g.boot()
-		if err != nil || got != pub || g.seed() != seed {
-			t.Fatalf("boot %d = %q, %v; the worker key must not change", boot, got, err)
+		if err != nil || got != fp || string(g.key()) != string(key) {
+			t.Fatalf("boot %d = %q, %v; the receipt key must not change", boot, got, err)
 		}
 	}
 	if len(g.auth.sent) != 1 {
-		t.Fatalf("a recorded key was sent again: %d sends", len(g.auth.sent))
+		t.Fatalf("a recorded fingerprint was sent again: %d sends", len(g.auth.sent))
 	}
 	if g.disk.mounts != 3 {
 		t.Fatalf("mounts = %d, want one per boot", g.disk.mounts)
 	}
-	published, err := os.ReadFile(filepath.Join(g.cfg.RunDir, "public-key"))
-	if err != nil || strings.TrimSpace(string(published)) != pub {
-		t.Fatalf("published key = %q, %v", published, err)
+	published, err := os.ReadFile(filepath.Join(g.cfg.RunDir, receiptReady))
+	if err != nil || strings.TrimSpace(string(published)) != fp {
+		t.Fatalf("published fingerprint = %q, %v", published, err)
 	}
-	g.assertSentOnlyPublicKeys(seed)
-	g.assertNoSeedLeak(nil, seed)
+	g.assertNoKeyLeak(nil)
 }
 
-// Once the domain holds a key, losing the guest's copy never yields a new
-// one, whatever a replayed first-start fw_cfg says.
+// Once the platform holds a fingerprint, losing the disk's key never yields a
+// new one; the boot refuses without asking again.
 func TestBoot_KeyLostAfterRecordingIsNeverReplaced(t *testing.T) {
 	g := newGuest(t)
 	g.contract()
-	g.seedKey()
-	if err := os.Remove(g.keyFile()); err != nil {
-		t.Fatal(err)
-	}
-	g.fw("store-status", "new\n")
-	g.resetMounts()
+	g.recordKey()
+	mustRemove(t, g.keyFile())
 	_, err := g.boot()
-	if !errors.Is(err, workerkey.ErrKeyMissing) {
-		t.Fatalf("err = %v, want ErrKeyMissing", err)
+	if !errors.Is(err, workerkey.ErrReceiptKeyMissing) {
+		t.Fatalf("err = %v, want ErrReceiptKeyMissing", err)
 	}
 	if _, err := os.Lstat(g.keyFile()); !errors.Is(err, os.ErrNotExist) {
-		t.Fatal("a replacement key was created for an existing store")
+		t.Fatal("a replacement key was created for a recorded volume")
 	}
 	if len(g.auth.sent) != 0 || g.auth.claims != 1 {
 		t.Fatalf("sends=%d claims=%d; a final refusal must not be retried or answered", len(g.auth.sent), g.auth.claims)
 	}
-	if _, err := os.Lstat(filepath.Join(g.cfg.RunDir, "public-key")); !errors.Is(err, os.ErrNotExist) {
-		t.Fatal("a public key was published")
-	}
 }
 
 func TestBoot_RecordedKeyMismatchRefuses(t *testing.T) {
 	g := newGuest(t)
 	g.contract()
-	g.seedKey()
-	seed := g.seed()
-	g.auth.recorded = otherPub(t)
+	key := g.recordKey()
+	g.auth.recorded = workerkey.Fingerprint(append([]byte("x"), key[1:]...))
 	_, err := g.boot()
 	if !errors.Is(err, workerkey.ErrKeyMismatch) {
 		t.Fatalf("err = %v, want ErrKeyMismatch", err)
 	}
-	if g.seed() != seed || len(g.auth.sent) != 0 {
-		t.Fatal("a mismatch changed or sent the worker key")
+	if string(g.key()) != string(key) || len(g.auth.sent) != 0 {
+		t.Fatal("a mismatch changed or sent the receipt key")
 	}
-	g.assertNoSeedLeak(err, seed)
+	g.assertNoKeyLeak(err)
 }
 
-// A reply lost after the key was recorded is retried, and the retry is
-// answered with that same key: one key, recorded once.
+// A pending decision on a disk that already holds the store's directory
+// creates no key: the platform and the disk disagree about a new volume.
+func TestBoot_PendingWithExistingStoreDataCreatesNoKey(t *testing.T) {
+	g := newGuest(t)
+	g.contract()
+	g.appendMount(testMajMin, "/", g.cfg.DataDir, "ext4", "/dev/vda")
+	mustMkdir(t, filepath.Join(g.cfg.DataDir, StoreDirName))
+	if _, err := g.boot(); !errors.Is(err, workerkey.ErrDiskHasData) {
+		t.Fatalf("err = %v, want ErrDiskHasData", err)
+	}
+	g.assertNoKey()
+}
+
+// A reply lost after recording is retried with the same key: one key,
+// recorded once.
 func TestBoot_LostRecordReplyKeepsTheSameKey(t *testing.T) {
 	for _, landed := range []bool{true, false} {
 		t.Run(fmt.Sprintf("landed=%v", landed), func(t *testing.T) {
 			g := newGuest(t)
 			g.contract()
 			g.auth.recordErrs = []recordErr{{err: transient{}, landed: landed}}
-			pub, err := g.boot()
+			fp, err := g.boot()
 			if err != nil {
 				t.Fatal(err)
 			}
-			if g.auth.recorded != pub || g.auth.claims != 2 {
+			if g.auth.recorded != fp || g.auth.claims != 2 {
 				t.Fatalf("recorded=%q claims=%d", g.auth.recorded, g.auth.claims)
 			}
 			for _, sent := range g.auth.sent {
-				if sent != pub {
-					t.Fatal("a second key was sent")
+				if sent != fp {
+					t.Fatal("a second key was recorded")
 				}
 			}
-			g.assertSentOnlyPublicKeys(g.seed())
 		})
 	}
 }
@@ -512,7 +491,7 @@ func TestBoot_DiskRefusalsNeverAskTheGateway(t *testing.T) {
 func TestBoot_AlreadyMountedWorkerDiskIsReused(t *testing.T) {
 	g := newGuest(t)
 	g.contract()
-	g.seedKey()
+	g.recordKey()
 	g.appendMount(testMajMin, "/", g.cfg.DataDir, "ext4", "/dev/vda")
 	if _, err := g.boot(); err != nil {
 		t.Fatal(err)

spinifex/agents/sqs/workerkey/acquire.go

@@ -3,43 +3,73 @@ package workerkey
 import (
 	"context"
 	"errors"
-
-	"github.com/nats-io/nkeys"
+	"fmt"
 )
 
-// Claimer is the SQS domain's first-store decision for this worker identity.
-type Claimer interface {
-	// Claim returns createKey, or the one public key the store authorizes.
-	Claim(ctx context.Context) (createKey bool, publicKey string, err error)
-	RecordPublicKey(ctx context.Context, publicKey string) error
+// Decision is the platform's receipt-key initialization state for this worker
+// volume: Pending authorizes creating or resuming the key, otherwise exactly
+// the recorded fingerprint is accepted.
+type Decision struct {
+	Pending     bool
+	Fingerprint string
+	// HostNodes is the gateway's declared cluster node count, zero if unknown.
+	HostNodes int
 }
 
-// Acquire returns the worker's store key under a current domain decision. A
-// new key is usable only once its public key is recorded; a recorded key is
-// never replaced, so a missing or different key on disk fails closed.
-func (s Store) Acquire(ctx context.Context, c Claimer) (nkeys.KeyPair, error) {
-	create, authorized, err := c.Claim(ctx)
-	if err != nil {
-		return nil, err
-	}
-	if !create {
-		return s.ForExistingStore(authorized)
-	}
-	if authorized != "" {
-		return nil, errors.New("first-store decision is contradictory")
-	}
-	// After a crash before recording, this loads the key already on disk.
-	kp, err := s.ForNewStore()
-	if err != nil {
-		return nil, err
-	}
-	pub, err := kp.PublicKey()
-	if err == nil {
-		err = c.RecordPublicKey(ctx, pub)
-	}
-	if err != nil {
-		kp.Wipe()
-		return nil, err
-	}
-	return kp, nil
+// Claimer reads and completes the platform's receipt-key initialization.
+type Claimer interface {
+	Claim(ctx context.Context) (Decision, error)
+	RecordFingerprint(ctx context.Context, fingerprint string) error
+}
+
+// ErrDiskHasData refuses a pending initialization on a disk that already holds
+// SQS data: the platform and the disk disagree about whether it is new.
+var ErrDiskHasData = errors.New("receipt key initialization is pending but the disk already holds SQS data")
+
+// Acquire returns the receipt key under the current platform decision. A
+// pending decision creates or resumes the key only while hasData reports no
+// SQS data, then records its fingerprint; the caller must clear the key.
+func (s Store) Acquire(ctx context.Context, c Claimer, hasData func() (bool, error)) ([]byte, error) {
+	d, err := c.Claim(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if !d.Pending {
+		return s.ForRecorded(d.Fingerprint)
+	}
+	if d.Fingerprint != "" {
+		return nil, errors.New("receipt key decision is contradictory")
+	}
+	data, err := hasData()
+	if err != nil {
+		return nil, fmt.Errorf("check the disk for SQS data: %w", err)
+	}
+	if data {
+		return nil, ErrDiskHasData
+	}
+	key, err := s.ensureReceipt()
+	if err != nil {
+		return nil, err
+	}
+	if err := c.RecordFingerprint(ctx, Fingerprint(key)); err != nil {
+		clear(key)
+		return nil, err
+	}
+	return key, nil
+}
+
+// ForRecorded returns the disk's receipt key only if it matches fingerprint.
+func (s Store) ForRecorded(fingerprint string) ([]byte, error) {
+	if len(fingerprint) != 64 {
+		return nil, errors.New("recorded receipt key fingerprint is malformed")
+	}
+	key, err := s.ReceiptKey()
+	if err != nil {
+		return nil, err
+	}
+	if Fingerprint(key) != fingerprint {
+		clear(key)
+		return nil, ErrKeyMismatch
+	}
+	return key, nil
 }

spinifex/agents/sqs/workerkey/acquire_test.go

@@ -1,4 +1,4 @@
-package workerkey
+package workerkey_test
 
 import (
 	"context"
@@ -7,260 +7,126 @@ import (
 	"path/filepath"
 	"testing"
 
-	testutil "github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/spinifex/domains/sqs/firststore"
-	"github.com/nats-io/nats.go/jetstream"
-	"github.com/nats-io/nkeys"
+	"github.com/mulgadc/spinifex/spinifex/agents/sqs/workerkey"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
-const region = "ap-southeast-2"
-
-// domainClaimer drives the real first-store state machine for one worker
-// identity. A failing record either drops the write or applies it and loses
-// the reply.
-type domainClaimer struct {
-	s          firststore.Store
-	id         firststore.Identity
-	failRecord bool
-	applied    bool
+// platform stands for the worker record behind the gateway: pending until a
+// fingerprint is recorded, then exactly that fingerprint.
+type platform struct {
+	recorded  string
+	recordErr error
+	landed    bool
+	records   int
 }
 
-func (c *domainClaimer) Claim(ctx context.Context) (bool, string, error) {
-	d, err := c.s.Claim(ctx, region, c.id)
-	return d.CreateKey, d.PublicKey, err
+func (p *platform) Claim(context.Context) (workerkey.Decision, error) {
+	return workerkey.Decision{Pending: p.recorded == "", Fingerprint: p.recorded, HostNodes: 1}, nil
 }
 
-func (c *domainClaimer) RecordPublicKey(ctx context.Context, pub string) error {
-	if c.failRecord {
-		c.failRecord = false
-		if c.applied {
-			c.s.RecordPublicKey(ctx, region, c.id, pub)
+func (p *platform) RecordFingerprint(_ context.Context, fp string) error {
+	p.records++
+	if p.recordErr != nil {
+		if p.landed {
+			p.recorded = fp
 		}
-		return errors.New("nats: timeout")
+		return p.recordErr
 	}
-	return c.s.RecordPublicKey(ctx, region, c.id, pub)
+	p.recorded = fp
+	return nil
 }
 
-func domain(t *testing.T) (firststore.Store, jetstream.KeyValue) {
+func newStore(t *testing.T) workerkey.Store {
 	t.Helper()
-	_, _, js := testutil.StartTestJetStream(t)
-	kv, err := js.CreateKeyValue(context.Background(), firststore.BucketConfig(1))
-	if err != nil {
-		t.Fatal(err)
-	}
-	return firststore.Store{JS: js, KV: kv}, kv
+	return workerkey.Store{Dir: filepath.Join(t.TempDir(), "key")}
 }
 
-func worker(gen uint64, n byte) firststore.Identity {
-	return firststore.Identity{InstanceID: "i-0000000000000000" + string('0'+n), ClaimID: "claim-" + string('a'+n) + "-0001", Generation: gen}
+func noData() (bool, error) { return false, nil }
+
+func TestAcquire_NewVolumeCreatesAndRecordsThenReloads(t *testing.T) {
+	s, p := newStore(t), &platform{}
+	key, err := s.Acquire(t.Context(), p, noData)
+	require.NoError(t, err)
+	assert.Len(t, key, workerkey.ReceiptKeyLen)
+	assert.Equal(t, workerkey.Fingerprint(key), p.recorded)
+
+	again, err := s.Acquire(t.Context(), p, func() (bool, error) { return true, nil })
+	require.NoError(t, err)
+	assert.Equal(t, key, again, "a recorded key is reloaded, store data or not")
+	assert.Equal(t, 1, p.records)
 }
 
-func workerDisk(t *testing.T) Store {
-	return Store{Dir: filepath.Join(t.TempDir(), "sqs-worker-key")}
-}
+// A reply lost after the write, or a crash before recording, resumes with the
+// key already on disk rather than a new one.
+func TestAcquire_LostRecordReplyResumesTheSameKey(t *testing.T) {
+	for _, landed := range []bool{false, true} {
+		s := newStore(t)
+		p := &platform{recordErr: errors.New("connection reset"), landed: landed}
+		_, err := s.Acquire(t.Context(), p, noData)
+		require.Error(t, err)
+		first, err := s.ReceiptKey()
+		require.NoError(t, err)
 
-func keyPath(s Store) string { return filepath.Join(s.Dir, keyFile) }
-
-func pubOf(t *testing.T, kp nkeys.KeyPair) string {
-	t.Helper()
-	p, err := kp.PublicKey()
-	if err != nil {
-		t.Fatal(err)
-	}
-	return p
-}
-
-func recorded(t *testing.T, s firststore.Store, id firststore.Identity) string {
-	t.Helper()
-	d, err := s.Claim(context.Background(), region, id)
-	if err != nil || d.CreateKey {
-		t.Fatalf("domain decision = %+v, %v; want a recorded key", d, err)
-	}
-	return d.PublicKey
-}
-
-func TestAcquire_CrashBeforeRecordingReusesTheKeyOnDisk(t *testing.T) {
-	ctx := context.Background()
-	fs, _ := domain(t)
-	disk := workerDisk(t)
-	c := &domainClaimer{s: fs, id: worker(1, 1), failRecord: true}
-	if _, err := disk.Acquire(ctx, c); err == nil {
-		t.Fatal("a key was returned before its public key was recorded")
-	}
-	before, err := os.ReadFile(keyPath(disk))
-	if err != nil {
-		t.Fatalf("key not synced to disk before recording: %v", err)
-	}
-	kp, err := disk.Acquire(ctx, c)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer kp.Wipe()
-	after, _ := os.ReadFile(keyPath(disk))
-	if string(before) != string(after) {
-		t.Fatal("the retry replaced the key")
-	}
-	if recorded(t, fs, c.id) != pubOf(t, kp) {
-		t.Fatal("the recorded key is not the key on disk")
+		p.recordErr = nil
+		key, err := s.Acquire(t.Context(), p, noData)
+		require.NoError(t, err)
+		assert.Equal(t, first, key)
+		assert.Equal(t, workerkey.Fingerprint(first), p.recorded)
 	}
 }
 
-func TestAcquire_LostRecordReplyIsRecoveredOnReboot(t *testing.T) {
-	ctx := context.Background()
-	fs, _ := domain(t)
-	disk := workerDisk(t)
-	c := &domainClaimer{s: fs, id: worker(1, 1), failRecord: true, applied: true}
-	if _, err := disk.Acquire(ctx, c); err == nil {
-		t.Fatal("expected the lost reply to surface")
-	}
-	kp, err := disk.Acquire(ctx, c)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer kp.Wipe()
-	if recorded(t, fs, c.id) != pubOf(t, kp) {
-		t.Fatal("reboot did not use the recorded key")
-	}
+// A pending decision on a disk that already holds store data creates nothing:
+// the platform and the disk disagree about whether the volume is new.
+func TestAcquire_PendingWithStoreDataCreatesNothing(t *testing.T) {
+	s, p := newStore(t), &platform{}
+	_, err := s.Acquire(t.Context(), p, func() (bool, error) { return true, nil })
+	require.ErrorIs(t, err, workerkey.ErrDiskHasData)
+	_, err = os.Stat(s.Dir)
+	assert.ErrorIs(t, err, os.ErrNotExist)
+	assert.Zero(t, p.records)
 }
 
-func TestAcquire_AfterRecordingKeyLossOrMismatchFailsClosed(t *testing.T) {
-	ctx := context.Background()
-	setup := func(t *testing.T, launch bool) (firststore.Store, Store, *domainClaimer, string) {
-		fs, _ := domain(t)
-		disk := workerDisk(t)
-		c := &domainClaimer{s: fs, id: worker(1, 1)}
-		kp, err := disk.Acquire(ctx, c)
-		if err != nil {
-			t.Fatal(err)
-		}
-		pub := pubOf(t, kp)
-		kp.Wipe()
-		if launch {
-			if err := fs.RecordLaunchIntent(ctx, region, c.id, pub); err != nil {
-				t.Fatal(err)
-			}
-		}
-		return fs, disk, c, pub
-	}
-	for _, launch := range []bool{false, true} {
-		name := map[bool]string{false: "key recorded", true: "launch intent"}[launch]
-		t.Run(name+"/key lost then reboot", func(t *testing.T) {
-			fs, disk, c, pub := setup(t, launch)
-			os.Remove(keyPath(disk))
-			// A relaunched worker generation gets the same answer.
-			for _, id := range []firststore.Identity{c.id, worker(2, 2)} {
-				c.id = id
-				if _, err := disk.Acquire(ctx, c); !errors.Is(err, ErrKeyMissing) {
-					t.Fatalf("acquire = %v, want ErrKeyMissing", err)
-				}
-				if _, err := os.Lstat(keyPath(disk)); !errors.Is(err, os.ErrNotExist) {
-					t.Fatal("a replacement key was created")
-				}
-			}
-			if recorded(t, fs, c.id) != pub {
-				t.Fatal("the recorded key changed")
-			}
-		})
-		t.Run(name+"/different key on disk", func(t *testing.T) {
-			_, _, c, _ := setup(t, launch)
-			other := workerDisk(t)
-			kp, err := other.ForNewStore()
-			if err != nil {
-				t.Fatal(err)
-			}
-			kp.Wipe()
-			before, _ := os.ReadFile(keyPath(other))
-			if _, err := other.Acquire(ctx, c); !errors.Is(err, ErrKeyMismatch) {
-				t.Fatalf("acquire = %v, want ErrKeyMismatch", err)
-			}
-			after, _ := os.ReadFile(keyPath(other))
-			if string(before) != string(after) {
-				t.Fatal("the mismatched key was replaced")
-			}
-		})
-	}
+// Once recorded, a missing, changed or damaged key refuses and is never
+// regenerated: a new key would invalidate every handle already issued.
+func TestAcquire_RecordedKeyIsNeverRegenerated(t *testing.T) {
+	s, p := newStore(t), &platform{}
+	key, err := s.Acquire(t.Context(), p, noData)
+	require.NoError(t, err)
+	path := filepath.Join(s.Dir, "receipt.key")
+
+	require.NoError(t, os.Remove(path))
+	_, err = s.Acquire(t.Context(), p, noData)
+	require.ErrorIs(t, err, workerkey.ErrReceiptKeyMissing)
+	_, err = os.Stat(path)
+	assert.ErrorIs(t, err, os.ErrNotExist, "no replacement key is written")
+
+	other := append([]byte(nil), key...)
+	other[0] ^= 0xff
+	require.NoError(t, os.WriteFile(path, other, 0o600))
+	_, err = s.Acquire(t.Context(), p, noData)
+	require.ErrorIs(t, err, workerkey.ErrKeyMismatch)
+
+	require.NoError(t, os.WriteFile(path, key[:8], 0o600))
+	_, err = s.Acquire(t.Context(), p, noData)
+	require.Error(t, err)
+	got, rerr := os.ReadFile(path)
+	require.NoError(t, rerr)
+	assert.Equal(t, key[:8], got, "a damaged key is refused, not replaced")
+	assert.Equal(t, 1, p.records)
 }
 
-func TestAcquire_RefusedClaimCreatesNoKey(t *testing.T) {
-	ctx := context.Background()
-	cases := map[string]func(t *testing.T, fs firststore.Store, kv jetstream.KeyValue) firststore.Identity{
-		"rival holds the claim": func(t *testing.T, fs firststore.Store, _ jetstream.KeyValue) firststore.Identity {
-			fs.Claim(ctx, region, worker(1, 1))
-			return worker(1, 2)
-		},
-		"stale replayed generation": func(t *testing.T, fs firststore.Store, _ jetstream.KeyValue) firststore.Identity {
-			fs.Claim(ctx, region, worker(1, 1))
-			fs.Claim(ctx, region, worker(2, 2))
-			return worker(1, 1)
-		},
-		"unreadable record": func(t *testing.T, _ firststore.Store, kv jetstream.KeyValue) firststore.Identity {
-			kv.Put(ctx, "region."+region, []byte("{"))
-			return worker(1, 1)
-		},
-	}
-	for name, setup := range cases {
-		t.Run(name, func(t *testing.T) {
-			fs, kv := domain(t)
-			id := setup(t, fs, kv)
-			disk := workerDisk(t)
-			if _, err := disk.Acquire(ctx, &domainClaimer{s: fs, id: id}); err == nil {
-				t.Fatal("a refused claim returned a key")
-			}
-			if _, err := os.Lstat(keyPath(disk)); !errors.Is(err, os.ErrNotExist) {
-				t.Fatal("a refused claim created a key")
-			}
-		})
-	}
+func TestAcquire_ContradictoryDecisionRefuses(t *testing.T) {
+	s := newStore(t)
+	_, err := s.Acquire(t.Context(), contradictory{}, noData)
+	require.Error(t, err)
+	_, err = os.Stat(s.Dir)
+	assert.ErrorIs(t, err, os.ErrNotExist)
 }
 
-type contradictoryClaimer struct{ pub string }
+type contradictory struct{}
 
-func (c contradictoryClaimer) Claim(context.Context) (bool, string, error) { return true, c.pub, nil }
-func (contradictoryClaimer) RecordPublicKey(context.Context, string) error {
-	return errors.New("unexpected")
-}
-
-func TestAcquire_ContradictoryDecisionCreatesNoKey(t *testing.T) {
-	kp, _ := nkeys.CreateUser()
-	disk := workerDisk(t)
-	if _, err := disk.Acquire(context.Background(), contradictoryClaimer{pub: pubOf(t, kp)}); err == nil {
-		t.Fatal("a contradictory decision returned a key")
-	}
-	if _, err := os.Lstat(keyPath(disk)); !errors.Is(err, os.ErrNotExist) {
-		t.Fatal("a contradictory decision created a key")
-	}
-}
-
-// A replica that has applied the claim but not the key record must not lead a
-// worker whose key is lost to create a replacement.
-func TestAcquire_StaleClaimedReadAfterRecordingCreatesNoKey(t *testing.T) {
-	ctx := context.Background()
-	fs, kv := domain(t)
-	disk := workerDisk(t)
-	id := worker(1, 1)
-	fs.Claim(ctx, region, id)
-	claimed, err := kv.Get(ctx, "region."+region)
-	if err != nil {
-		t.Fatal(err)
-	}
-	kp, err := disk.Acquire(ctx, &domainClaimer{s: fs, id: id})
-	if err != nil {
-		t.Fatal(err)
-	}
-	kp.Wipe()
-	os.Remove(keyPath(disk))
-
-	_, _, replicaJS := testutil.StartTestJetStream(t)
-	replica, err := replicaJS.CreateKeyValue(ctx, firststore.BucketConfig(1))
-	if err != nil {
-		t.Fatal(err)
-	}
-	replica.Put(ctx, "region."+region, claimed.Value())
-	stale := firststore.Store{JS: replicaJS, KV: fs.KV}
-	if _, err := disk.Acquire(ctx, &domainClaimer{s: stale, id: id}); err == nil {
-		t.Fatal("a stale claimed read returned a key")
-	}
-	if _, err := os.Lstat(keyPath(disk)); !errors.Is(err, os.ErrNotExist) {
-		t.Fatal("a stale claimed read created a replacement key")
-	}
+func (contradictory) Claim(context.Context) (workerkey.Decision, error) {
+	return workerkey.Decision{Pending: true, Fingerprint: "ab"}, nil
 }
+func (contradictory) RecordFingerprint(context.Context, string) error { return nil }

spinifex/agents/sqs/workerkey/receipt.go

@@ -0,0 +1,121 @@
+package workerkey
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
+const (
+	receiptFile = "receipt.key"
+	// ReceiptKeyLen is the receipt-signing key's length in bytes.
+	ReceiptKeyLen = 32
+)
+
+// ErrReceiptKeyMissing is a worker disk without the receipt-signing key once
+// its fingerprint is recorded. It is never regenerated: a new key would
+// invalidate every handle already issued.
+var ErrReceiptKeyMissing = errors.New("worker receipt-signing key is missing")
+
+func (s Store) receiptPath() string { return filepath.Join(s.Dir, receiptFile) }
+
+// ReceiptKey returns the receipt-signing key already on disk; it never
+// creates one. The caller must not log or retain it beyond use.
+func (s Store) ReceiptKey() ([]byte, error) {
+	unlock, err := s.prepareDir()
+	if err != nil {
+		return nil, err
+	}
+	defer unlock()
+	key, err := s.loadReceipt()
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil, ErrReceiptKeyMissing
+	}
+	return key, err
+}
+
+// ensureReceipt loads the receipt-signing key, creating it only if absent.
+// Only a pending initialization calls it, so a crash before the fingerprint
+// is recorded resumes with the key already written.
+func (s Store) ensureReceipt() ([]byte, error) {
+	unlock, err := s.prepareDir()
+	if err != nil {
+		return nil, err
+	}
+	defer unlock()
+	key, err := s.loadReceipt()
+	if err == nil {
+		if err := s.syncNames(); err != nil {
+			clear(key)
+			return nil, err
+		}
+		return key, nil
+	}
+	if !errors.Is(err, fs.ErrNotExist) {
+		return nil, err
+	}
+	key = make([]byte, ReceiptKeyLen)
+	if _, err := rand.Read(key); err != nil {
+		clear(key)
+		return nil, fmt.Errorf("generate worker receipt key: %w", err)
+	}
+	if err := s.install(key); err != nil {
+		clear(key)
+		return nil, err
+	}
+	return key, nil
+}
+
+// install links a synced temporary file into place, so the key appears whole
+// or not at all, then makes both directory entries durable.
+func (s Store) install(key []byte) error {
+	tmp, err := os.CreateTemp(s.Dir, tempPrefix+"*"+tempSuffix)
+	if err != nil {
+		return fmt.Errorf("create worker receipt key file: %w", err)
+	}
+	tmpName := tmp.Name()
+	defer os.Remove(tmpName)
+	if err := writeSynced(tmp, key); err != nil {
+		return err
+	}
+	if err := os.Link(tmpName, s.receiptPath()); err != nil {
+		return fmt.Errorf("install worker receipt key file: %w", err)
+	}
+	return s.syncNames()
+}
+
+// Fingerprint is the SHA-256 of a receipt key, hex encoded. It identifies the
+// key without revealing it.
+func Fingerprint(key []byte) string {
+	sum := sha256.Sum256(key)
+	return hex.EncodeToString(sum[:])
+}
+
+// loadReceipt refuses a file that is present but unusable rather than
+// replacing it.
+func (s Store) loadReceipt() ([]byte, error) {
+	fi, err := os.Lstat(s.receiptPath())
+	if err != nil {
+		return nil, err
+	}
+	if !fi.Mode().IsRegular() || fi.Mode().Perm()&0o077 != 0 {
+		return nil, ErrUnsafe
+	}
+	if fi.Size() != ReceiptKeyLen {
+		return nil, errors.New("worker receipt key file has the wrong length")
+	}
+	key, err := os.ReadFile(s.receiptPath())
+	if err != nil {
+		return nil, fmt.Errorf("read worker receipt key: %w", err)
+	}
+	if len(key) != ReceiptKeyLen {
+		clear(key)
+		return nil, errors.New("worker receipt key file has the wrong length")
+	}
+	return key, nil
+}

spinifex/agents/sqs/workerkey/workerkey.go

@@ -1,34 +1,26 @@
-// Package workerkey keeps the shared SQS worker's store-client NKey on the
-// worker's own persistent disk. The seed is generated and held only inside the
-// worker guest; the platform learns the public key alone.
+// Package workerkey keeps the SQS guest's receipt-signing key on its own
+// persistent disk. The key is generated and held only inside the guest; the
+// platform learns its fingerprint alone.
 package workerkey
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
 	"io/fs"
 	"os"
 	"path/filepath"
 	"syscall"
-
-	"github.com/nats-io/nkeys"
 )
 
 const (
-	keyFile    = "store-client.nk"
-	tempPrefix = ".store-client-"
+	tempPrefix = ".receipt-"
 	tempSuffix = ".tmp"
-	// A user seed is 58 bytes; anything much larger is not one.
-	maxKeyFileSize = 256
 )
 
-// Each refusal is fail-closed: the worker never replaces a key it cannot use.
+// Each refusal is fail-closed: the guest never replaces a key it cannot use.
 var (
-	ErrKeyMissing  = errors.New("worker store key is missing but the store already exists")
-	ErrKeyMismatch = errors.New("worker store key does not match the store's authorized key")
-	ErrUnsafe      = errors.New("worker store key path is not owner-only")
-	ErrCorrupt     = errors.New("worker store key file does not hold a user NKey seed")
+	ErrKeyMismatch = errors.New("worker receipt key does not match the recorded fingerprint")
+	ErrUnsafe      = errors.New("worker receipt key path is not owner-only")
 )
 
 // Store is the key directory on the worker disk. Workerkey cannot tell a
@@ -38,66 +30,12 @@ type Store struct {
 	Dir string
 }
 
-// ForNewStore returns the worker's store-client key for a store that the
-// caller has positively established does not exist yet, generating it on first
-// use. An unreadable or absent store record is not that proof.
-func (s Store) ForNewStore() (nkeys.KeyPair, error) {
-	return s.ensure("")
-}
-
-// ForExistingStore returns the worker's key only if it is already on disk and
-// is the one the store authorizes; it never generates a key.
-func (s Store) ForExistingStore(authorized string) (nkeys.KeyPair, error) {
-	if !nkeys.IsValidPublicUserKey(authorized) {
-		return nil, errors.New("authorized store key is not a user public NKey")
-	}
-	return s.ensure(authorized)
-}
-
-func (s Store) ensure(expected string) (nkeys.KeyPair, error) {
-	unlock, err := s.prepareDir()
-	if err != nil {
-		return nil, err
-	}
-	defer unlock()
-	kp, err := s.load()
-	if errors.Is(err, fs.ErrNotExist) {
-		if expected != "" {
-			return nil, ErrKeyMissing
-		}
-		kp, err = s.create()
-	}
-	if err != nil {
-		return nil, err
-	}
-	if expected != "" {
-		pub, err := kp.PublicKey()
-		if err != nil {
-			kp.Wipe()
-			return nil, ErrCorrupt
-		}
-		if pub != expected {
-			kp.Wipe()
-			return nil, fmt.Errorf("%w: have %s, store authorizes %s", ErrKeyMismatch, pub, expected)
-		}
-	}
-	// Re-synced on every return, so a key whose earlier sync failed is never
-	// handed out before its name and directory are durable.
-	if err := s.syncNames(); err != nil {
-		kp.Wipe()
-		return nil, err
-	}
-	return kp, nil
-}
-
-func (s Store) path() string { return filepath.Join(s.Dir, keyFile) }
-
 // prepareDir creates only the key directory, inside a parent that must already
 // exist, refuses one that is not owner-only, locks it and removes temporary
 // files a crash left behind.
 func (s Store) prepareDir() (unlock func(), err error) {
 	if s.Dir == "" {
-		return nil, errors.New("worker store key directory is not set")
+		return nil, errors.New("worker receipt key directory is not set")
 	}
 	parent, err := os.Stat(filepath.Dir(s.Dir))
 	if err != nil || !parent.IsDir() {
@@ -136,69 +74,6 @@ func (s Store) prepareDir() (unlock func(), err error) {
 	return unlock, nil
 }
 
-// load reads an existing key. Any file that is present but unusable refuses
-// rather than being replaced.
-func (s Store) load() (nkeys.KeyPair, error) {
-	fi, err := os.Lstat(s.path())
-	if err != nil {
-		return nil, err
-	}
-	if !fi.Mode().IsRegular() || fi.Mode().Perm()&0o077 != 0 {
-		return nil, ErrUnsafe
-	}
-	if fi.Size() > maxKeyFileSize {
-		return nil, ErrCorrupt
-	}
-	seed, err := os.ReadFile(s.path())
-	if err != nil {
-		return nil, fmt.Errorf("read worker store key: %w", err)
-	}
-	kp, err := nkeys.FromSeed(bytes.TrimSpace(seed))
-	if err != nil {
-		return nil, ErrCorrupt
-	}
-	pub, err := kp.PublicKey()
-	if err != nil || !nkeys.IsValidPublicUserKey(pub) {
-		kp.Wipe()
-		return nil, ErrCorrupt
-	}
-	return kp, nil
-}
-
-// create writes a new seed to a synced temporary file and links it into place
-// without replacing anything; the caller syncs the directory names.
-func (s Store) create() (nkeys.KeyPair, error) {
-	kp, err := nkeys.CreateUser()
-	if err != nil {
-		return nil, fmt.Errorf("generate worker store key: %w", err)
-	}
-	seed, err := kp.Seed()
-	if err != nil {
-		kp.Wipe()
-		return nil, fmt.Errorf("encode worker store key: %w", err)
-	}
-
-	tmp, err := os.CreateTemp(s.Dir, tempPrefix+"*"+tempSuffix)
-	if err != nil {
-		kp.Wipe()
-		return nil, fmt.Errorf("create worker store key file: %w", err)
-	}
-	tmpName := tmp.Name()
-	defer os.Remove(tmpName)
-	if err := writeSynced(tmp, seed); err != nil {
-		kp.Wipe()
-		return nil, err
-	}
-	if err := os.Link(tmpName, s.path()); err != nil {
-		kp.Wipe()
-		if errors.Is(err, fs.ErrExist) {
-			return s.load()
-		}
-		return nil, fmt.Errorf("install worker store key file: %w", err)
-	}
-	return kp, nil
-}
-
 // syncNames makes the key's directory entry and the key directory's own entry
 // in its parent durable.
 func (s Store) syncNames() error {
@@ -211,18 +86,18 @@ func (s Store) syncNames() error {
 func writeSynced(f *os.File, data []byte) error {
 	if err := f.Chmod(0o600); err != nil {
 		f.Close()
-		return fmt.Errorf("restrict worker store key file: %w", err)
+		return fmt.Errorf("restrict worker receipt key file: %w", err)
 	}
 	if _, err := f.Write(data); err != nil {
 		f.Close()
-		return fmt.Errorf("write worker store key file: %w", err)
+		return fmt.Errorf("write worker receipt key file: %w", err)
 	}
 	if err := f.Sync(); err != nil {
 		f.Close()
-		return fmt.Errorf("sync worker store key file: %w", err)
+		return fmt.Errorf("sync worker receipt key file: %w", err)
 	}
 	if err := f.Close(); err != nil {
-		return fmt.Errorf("close worker store key file: %w", err)
+		return fmt.Errorf("close worker receipt key file: %w", err)
 	}
 	return nil
 }

spinifex/agents/sqs/workerkey/workerkey_test.go

@@ -1,423 +0,0 @@
-package workerkey
-
-import (
-	"errors"
-	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-	"sync"
-	"syscall"
-	"testing"
-	"time"
-
-	"github.com/nats-io/nkeys"
-)
-
-func newStore(t *testing.T) Store {
-	t.Helper()
-	return Store{Dir: filepath.Join(t.TempDir(), "keys")}
-}
-
-func publicKey(t *testing.T, kp nkeys.KeyPair) string {
-	t.Helper()
-	pub, err := kp.PublicKey()
-	if err != nil {
-		t.Fatalf("public key: %v", err)
-	}
-	return pub
-}
-
-func otherUserKey(t *testing.T) string {
-	t.Helper()
-	kp, err := nkeys.CreateUser()
-	if err != nil {
-		t.Fatalf("create user: %v", err)
-	}
-	return publicKey(t, kp)
-}
-
-func readFile(t *testing.T, path string) []byte {
-	t.Helper()
-	b, err := os.ReadFile(path)
-	if err != nil {
-		t.Fatalf("read %s: %v", path, err)
-	}
-	return b
-}
-
-func assertNoSeed(t *testing.T, err error, seed []byte) {
-	t.Helper()
-	if err != nil && strings.Contains(err.Error(), string(seed)) {
-		t.Fatalf("error exposes the seed: %v", err)
-	}
-}
-
-func TestEnsure_FirstBootPersistsAndRestartReusesKey(t *testing.T) {
-	s := newStore(t)
-	kp, err := s.ForNewStore()
-	if err != nil {
-		t.Fatalf("first boot: %v", err)
-	}
-	pub := publicKey(t, kp)
-
-	dirInfo, err := os.Stat(s.Dir)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if got := dirInfo.Mode().Perm(); got != 0o700 {
-		t.Fatalf("directory mode = %o, want 700", got)
-	}
-	fileInfo, err := os.Stat(s.path())
-	if err != nil {
-		t.Fatal(err)
-	}
-	if got := fileInfo.Mode().Perm(); got != 0o600 {
-		t.Fatalf("key file mode = %o, want 600", got)
-	}
-	onDisk, err := nkeys.FromSeed(readFile(t, s.path()))
-	if err != nil {
-		t.Fatalf("persisted file is not a seed: %v", err)
-	}
-	if publicKey(t, onDisk) != pub {
-		t.Fatal("persisted seed does not match the returned key")
-	}
-
-	// A restart is a fresh Store over the same disk; neither path may
-	// generate a replacement.
-	again, err := Store{Dir: s.Dir}.ForExistingStore(pub)
-	if err != nil {
-		t.Fatalf("restart for the existing store: %v", err)
-	}
-	if publicKey(t, again) != pub {
-		t.Fatal("restart for the existing store changed the key")
-	}
-	again, err = Store{Dir: s.Dir}.ForNewStore()
-	if err != nil {
-		t.Fatalf("repeated first start: %v", err)
-	}
-	if publicKey(t, again) != pub {
-		t.Fatal("repeated first start replaced the key")
-	}
-	entries, err := os.ReadDir(s.Dir)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(entries) != 1 {
-		t.Fatalf("key directory holds %d entries, want only the key", len(entries))
-	}
-}
-
-func TestEnsure_ExistingStoreWithMissingKeyFailsClosed(t *testing.T) {
-	s := newStore(t)
-	_, err := s.ForExistingStore(otherUserKey(t))
-	if !errors.Is(err, ErrKeyMissing) {
-		t.Fatalf("err = %v, want ErrKeyMissing", err)
-	}
-	if _, err := os.Lstat(s.path()); !errors.Is(err, os.ErrNotExist) {
-		t.Fatalf("a key was generated for an existing store: %v", err)
-	}
-}
-
-func TestEnsure_MismatchedKeyFailsClosedAndKeepsFile(t *testing.T) {
-	s := newStore(t)
-	if _, err := s.ForNewStore(); err != nil {
-		t.Fatal(err)
-	}
-	before := readFile(t, s.path())
-
-	_, err := Store{Dir: s.Dir}.ForExistingStore(otherUserKey(t))
-	if !errors.Is(err, ErrKeyMismatch) {
-		t.Fatalf("err = %v, want ErrKeyMismatch", err)
-	}
-	assertNoSeed(t, err, before)
-	if string(readFile(t, s.path())) != string(before) {
-		t.Fatal("mismatch replaced the key file")
-	}
-}
-
-func TestEnsure_UnusableKeyFileIsNeverReplaced(t *testing.T) {
-	seedKP, err := nkeys.CreateUser()
-	if err != nil {
-		t.Fatal(err)
-	}
-	userSeed, err := seedKP.Seed()
-	if err != nil {
-		t.Fatal(err)
-	}
-	accountKP, err := nkeys.CreateAccount()
-	if err != nil {
-		t.Fatal(err)
-	}
-	accountSeed, err := accountKP.Seed()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	cases := []struct {
-		name    string
-		content []byte
-		mode    os.FileMode
-		want    error
-	}{
-		{"empty", nil, 0o600, ErrCorrupt},
-		{"garbage", []byte("not a seed"), 0o600, ErrCorrupt},
-		{"account seed", accountSeed, 0o600, ErrCorrupt},
-		{"oversized", make([]byte, maxKeyFileSize+1), 0o600, ErrCorrupt},
-		{"group readable", userSeed, 0o640, ErrUnsafe},
-		{"world readable", userSeed, 0o604, ErrUnsafe},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			s := newStore(t)
-			if err := os.MkdirAll(s.Dir, 0o700); err != nil {
-				t.Fatal(err)
-			}
-			if err := os.WriteFile(s.path(), tc.content, 0o600); err != nil {
-				t.Fatal(err)
-			}
-			if err := os.Chmod(s.path(), tc.mode); err != nil {
-				t.Fatal(err)
-			}
-			_, err := s.ForNewStore()
-			if !errors.Is(err, tc.want) {
-				t.Fatalf("err = %v, want %v", err, tc.want)
-			}
-			assertNoSeed(t, err, userSeed)
-			assertNoSeed(t, err, accountSeed)
-			if string(readFile(t, s.path())) != string(tc.content) {
-				t.Fatal("an unusable key file was replaced")
-			}
-		})
-	}
-}
-
-func TestEnsure_RefusesUnsafeDirectoryAndSymlink(t *testing.T) {
-	t.Run("open directory", func(t *testing.T) {
-		s := newStore(t)
-		if err := os.MkdirAll(s.Dir, 0o755); err != nil {
-			t.Fatal(err)
-		}
-		if err := os.Chmod(s.Dir, 0o755); err != nil {
-			t.Fatal(err)
-		}
-		if _, err := s.ForNewStore(); !errors.Is(err, ErrUnsafe) {
-			t.Fatalf("err = %v, want ErrUnsafe", err)
-		}
-		if _, err := os.Lstat(s.path()); !errors.Is(err, os.ErrNotExist) {
-			t.Fatal("a key was written into an open directory")
-		}
-	})
-	t.Run("symlinked key", func(t *testing.T) {
-		s := newStore(t)
-		target := newStore(t)
-		if _, err := target.ForNewStore(); err != nil {
-			t.Fatal(err)
-		}
-		if err := os.MkdirAll(s.Dir, 0o700); err != nil {
-			t.Fatal(err)
-		}
-		if err := os.Symlink(target.path(), s.path()); err != nil {
-			t.Fatal(err)
-		}
-		if _, err := s.ForNewStore(); !errors.Is(err, ErrUnsafe) {
-			t.Fatalf("err = %v, want ErrUnsafe", err)
-		}
-	})
-}
-
-func TestEnsure_CrashLeftoverTempIsNotAKey(t *testing.T) {
-	s := newStore(t)
-	if err := os.MkdirAll(s.Dir, 0o700); err != nil {
-		t.Fatal(err)
-	}
-	stale := filepath.Join(s.Dir, tempPrefix+"123"+tempSuffix)
-	if err := os.WriteFile(stale, []byte("partial"), 0o600); err != nil {
-		t.Fatal(err)
-	}
-
-	// A crash before the link means no store was authorized for it.
-	if _, err := s.ForExistingStore(otherUserKey(t)); !errors.Is(err, ErrKeyMissing) {
-		t.Fatalf("err = %v, want ErrKeyMissing", err)
-	}
-	if _, err := os.Lstat(stale); !errors.Is(err, os.ErrNotExist) {
-		t.Fatal("stale temporary key file was kept")
-	}
-}
-
-func TestEnsure_ConcurrentFirstBootAgreesOnOneKey(t *testing.T) {
-	const rounds, n = 20, 16
-	for range rounds {
-		s := newStore(t)
-		pubs := make([]string, n)
-		errs := make([]error, n)
-		start := make(chan struct{})
-		var wg sync.WaitGroup
-		for i := range n {
-			wg.Go(func() {
-				<-start
-				kp, err := Store{Dir: s.Dir}.ForNewStore()
-				if err != nil {
-					errs[i] = err
-					return
-				}
-				pubs[i], errs[i] = kp.PublicKey()
-			})
-		}
-		close(start)
-		wg.Wait()
-		onDisk, err := nkeys.FromSeed(readFile(t, s.path()))
-		if err != nil {
-			t.Fatal(err)
-		}
-		want := publicKey(t, onDisk)
-		for i := range n {
-			if errs[i] != nil {
-				t.Fatalf("call %d: %v", i, errs[i])
-			}
-			if pubs[i] != want {
-				t.Fatalf("call %d returned a key that is not the persisted one", i)
-			}
-		}
-	}
-}
-
-func TestForExistingStore_RejectsInvalidAuthorizedKey(t *testing.T) {
-	kp, err := nkeys.CreateUser()
-	if err != nil {
-		t.Fatal(err)
-	}
-	seed, err := kp.Seed()
-	if err != nil {
-		t.Fatal(err)
-	}
-	// An empty key is what an unreadable store record would yield; it must not
-	// fall through to first-start generation.
-	for _, authorized := range []string{"", string(seed)} {
-		s := newStore(t)
-		_, err := s.ForExistingStore(authorized)
-		if err == nil {
-			t.Fatalf("authorized key %q was accepted", authorized[:min(len(authorized), 2)])
-		}
-		assertNoSeed(t, err, seed)
-		if _, err := os.Lstat(s.Dir); !errors.Is(err, os.ErrNotExist) {
-			t.Fatal("an invalid authorized key still touched the disk")
-		}
-	}
-}
-
-func TestForNewStore_RefusesMissingParent(t *testing.T) {
-	s := Store{Dir: filepath.Join(t.TempDir(), "missing-parent", "keys")}
-	if _, err := s.ForNewStore(); err == nil {
-		t.Fatal("a key directory was created without its parent")
-	}
-	if _, err := os.Lstat(filepath.Dir(s.Dir)); !errors.Is(err, os.ErrNotExist) {
-		t.Fatal("the missing parent was created")
-	}
-}
-
-// hookSyncs records every directory sync and fails call number failAt
-// (counted from one); zero fails none.
-func hookSyncs(t *testing.T, failAt int) *[]string {
-	t.Helper()
-	var synced []string
-	orig := syncDirFn
-	t.Cleanup(func() { syncDirFn = orig })
-	syncDirFn = func(dir string) error {
-		synced = append(synced, dir)
-		if len(synced) == failAt {
-			return errors.New("injected directory sync failure")
-		}
-		return orig(dir)
-	}
-	return &synced
-}
-
-func TestEnsure_RetryAfterFailedDirectorySyncKeepsKeyAndSyncs(t *testing.T) {
-	retries := map[string]func(s Store, pub string) (nkeys.KeyPair, error){
-		"first start": func(s Store, _ string) (nkeys.KeyPair, error) { return s.ForNewStore() },
-		"existing store": func(s Store, pub string) (nkeys.KeyPair, error) {
-			return s.ForExistingStore(pub)
-		},
-	}
-	for name, retry := range retries {
-		for failAt := 1; failAt <= 2; failAt++ {
-			t.Run(fmt.Sprintf("%s after sync %d failed", name, failAt), func(t *testing.T) {
-				s := newStore(t)
-				hookSyncs(t, failAt)
-				if _, err := s.ForNewStore(); err == nil {
-					t.Fatal("a key was returned although a directory sync failed")
-				}
-				first := readFile(t, s.path())
-				onDisk, err := nkeys.FromSeed(first)
-				if err != nil {
-					t.Fatal(err)
-				}
-				pub := publicKey(t, onDisk)
-
-				synced := hookSyncs(t, 0)
-				kp, err := retry(Store{Dir: s.Dir}, pub)
-				if err != nil {
-					t.Fatalf("retry: %v", err)
-				}
-				if string(readFile(t, s.path())) != string(first) {
-					t.Fatal("retry replaced the key")
-				}
-				if publicKey(t, kp) != pub {
-					t.Fatal("retry returned a key other than the persisted one")
-				}
-				want := []string{s.Dir, filepath.Dir(s.Dir)}
-				if len(*synced) != 2 || (*synced)[0] != want[0] || (*synced)[1] != want[1] {
-					t.Fatalf("retry synced %v, want %v", *synced, want)
-				}
-			})
-		}
-	}
-}
-
-// Another holder of the directory lock, such as a second worker process on
-// the same disk, must finish before this one reads, cleans or creates a key.
-func TestEnsure_WaitsForDirectoryLock(t *testing.T) {
-	s := newStore(t)
-	if err := os.MkdirAll(s.Dir, 0o700); err != nil {
-		t.Fatal(err)
-	}
-	holder, err := os.Open(s.Dir)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer holder.Close()
-	if err := syscall.Flock(int(holder.Fd()), syscall.LOCK_EX); err != nil {
-		t.Fatal(err)
-	}
-
-	done := make(chan error, 1)
-	go func() {
-		_, err := Store{Dir: s.Dir}.ForNewStore()
-		done <- err
-	}()
-	deadline := time.Now().Add(300 * time.Millisecond)
-	for time.Now().Before(deadline) {
-		select {
-		case err := <-done:
-			t.Fatalf("Ensure returned while another holder had the lock: %v", err)
-		default:
-		}
-		if _, err := os.Lstat(s.path()); err == nil {
-			t.Fatal("a key was created while another holder had the lock")
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-
-	if err := syscall.Flock(int(holder.Fd()), syscall.LOCK_UN); err != nil {
-		t.Fatal(err)
-	}
-	select {
-	case err := <-done:
-		if err != nil {
-			t.Fatalf("Ensure after unlock: %v", err)
-		}
-	case <-time.After(5 * time.Second):
-		t.Fatal("Ensure did not proceed after the lock was released")
-	}
-}

spinifex/domains/ec2/systeminstance/kind.go

@@ -0,0 +1,27 @@
+package systeminstance
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+)
+
+// Kind selects the guest a direct-boot system instance runs. The empty kind
+// is the load balancer; every other kind is launched only on its own subject.
+type Kind string
+
+// ErrNonLoadBalancerKind refuses a non-load-balancer guest on the legacy path.
+var ErrNonLoadBalancerKind = errors.New("sysinstance: guest kind must use its own launch subject")
+
+// volumeIDPattern is the platform's volume ID shape. The guest finds its disk
+// by the ID without its dash, which must fit the 20-byte virtio-blk serial.
+var volumeIDPattern = regexp.MustCompile(`^vol-[0-9a-f]{17}$`)
+
+// RefuseNonLoadBalancerKind keeps every non-load-balancer guest off the
+// legacy launch path, which would boot the load balancer image instead.
+func (in *SystemInstanceInput) RefuseNonLoadBalancerKind() error {
+	if in.Kind != "" || in.SQSWorker != nil {
+		return fmt.Errorf("%w: kind %q", ErrNonLoadBalancerKind, in.Kind)
+	}
+	return nil
+}

spinifex/domains/ec2/systeminstance/launcher.go

@@ -88,10 +88,8 @@ type SystemInstanceInput struct {
 	IamInstanceProfileArn string `json:"iam_instance_profile_arn,omitempty"`
 
 	// Kind selects a non-load-balancer direct-boot guest; empty is the load
-	// balancer. SQSStore carries the KindSQSStore guest's volume and key, and
-	// SQSWorker the KindSQSWorker guest's volume and store decision.
+	// balancer. SQSWorker carries the KindSQSWorker guest's volume and Region.
 	Kind      Kind            `json:"kind,omitempty"`
-	SQSStore  *SQSStoreInput  `json:"sqs_store,omitempty"`
 	SQSWorker *SQSWorkerInput `json:"sqs_worker,omitempty"`
 }
 

spinifex/domains/ec2/systeminstance/sqs_store.go

@@ -1,78 +0,0 @@
-package systeminstance
-
-import (
-	"errors"
-	"fmt"
-	"regexp"
-
-	"github.com/nats-io/nkeys"
-)
-
-// Kind selects the guest a direct-boot system instance runs. The empty kind
-// is the load balancer; every other kind is launched only on its own subject.
-type Kind string
-
-// KindSQSStore is the dedicated SQS store guest on a persistent data volume.
-const KindSQSStore Kind = "sqs-store"
-
-// SQSStoreLaunchSubjectRoot is the subject root for SQS store launches. It is
-// versioned and separate from system.LaunchInstance so a daemon that predates
-// it has no subscriber and the launch fails instead of booting another guest.
-const SQSStoreLaunchSubjectRoot = "system.LaunchSQSStore.v1"
-
-// SQSStoreLaunchSubject is the queue subject for an instance type; with a node
-// ID it is that node's targeted subject.
-func SQSStoreLaunchSubject(instanceType, nodeID string) string {
-	if nodeID == "" {
-		return fmt.Sprintf("%s.%s", SQSStoreLaunchSubjectRoot, instanceType)
-	}
-	return fmt.Sprintf("%s.%s.%s", SQSStoreLaunchSubjectRoot, instanceType, nodeID)
-}
-
-// SQSStoreInput is what an SQS store guest needs beyond its network: an
-// existing, already-formatted system-account volume and the one NATS user
-// public key the store accepts. It never carries a private key or seed.
-type SQSStoreInput struct {
-	DataVolumeID string `json:"data_volume_id"`
-	ClientNKey   string `json:"client_nkey"`
-}
-
-// ErrInvalidSQSStoreInput is returned for a malformed SQS store launch.
-var ErrInvalidSQSStoreInput = errors.New("sysinstance: invalid SQS store launch")
-
-// volumeIDPattern is the platform's volume ID shape. The guest finds its disk
-// by the ID without its dash, which must fit the 20-byte virtio-blk serial.
-var volumeIDPattern = regexp.MustCompile(`^vol-[0-9a-f]{17}$`)
-
-// ValidateSQSStore checks an SQS store launch's shape. It reads nothing, so a
-// valid result says only that the volume and key are well formed.
-func (in *SystemInstanceInput) ValidateSQSStore() error {
-	switch {
-	case in.Kind != KindSQSStore:
-		return fmt.Errorf("%w: kind %q is not %q", ErrInvalidSQSStoreInput, in.Kind, KindSQSStore)
-	case in.BootMode != "" && in.BootMode != BootDirect:
-		return fmt.Errorf("%w: boot mode %q, the store boots direct only", ErrInvalidSQSStoreInput, in.BootMode)
-	case in.SQSStore == nil:
-		return fmt.Errorf("%w: no data volume or client key", ErrInvalidSQSStoreInput)
-	case in.SQSWorker != nil:
-		return fmt.Errorf("%w: SQS worker settings on an SQS store launch", ErrInvalidSQSStoreInput)
-	case !volumeIDPattern.MatchString(in.SQSStore.DataVolumeID):
-		return fmt.Errorf("%w: data volume ID %q is malformed", ErrInvalidSQSStoreInput, in.SQSStore.DataVolumeID)
-	case !nkeys.IsValidPublicUserKey(in.SQSStore.ClientNKey):
-		// The value is not echoed: a caller that sent a seed here must not
-		// have it written to a log.
-		return fmt.Errorf("%w: client key is not a NATS user public key", ErrInvalidSQSStoreInput)
-	case in.LBAgentEnv != "" || in.CACert != "" || in.Scheme != "":
-		return fmt.Errorf("%w: load balancer settings on an SQS store launch", ErrInvalidSQSStoreInput)
-	}
-	return nil
-}
-
-// RefuseNonLoadBalancerKind keeps every non-load-balancer guest off the
-// legacy launch path, which would boot the load balancer image instead.
-func (in *SystemInstanceInput) RefuseNonLoadBalancerKind() error {
-	if in.Kind != "" || in.SQSStore != nil || in.SQSWorker != nil {
-		return fmt.Errorf("%w: kind %q must use its own launch subject", ErrInvalidSQSStoreInput, in.Kind)
-	}
-	return nil
-}

spinifex/domains/ec2/systeminstance/sqs_store_test.go

@@ -1,98 +0,0 @@
-package systeminstance
-
-import (
-	"strings"
-	"testing"
-
-	identifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
-	"github.com/nats-io/nkeys"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func validSQSStoreInput(t *testing.T) (*SystemInstanceInput, []byte) {
-	t.Helper()
-	kp, err := nkeys.CreateUser()
-	require.NoError(t, err)
-	pub, err := kp.PublicKey()
-	require.NoError(t, err)
-	seed, err := kp.Seed()
-	require.NoError(t, err)
-	return &SystemInstanceInput{
-		InstanceType: "sys.micro",
-		Kind:         KindSQSStore,
-		SQSStore:     &SQSStoreInput{DataVolumeID: "vol-0123456789abcdef0", ClientNKey: pub},
-	}, seed
-}
-
-func TestValidateSQSStore(t *testing.T) {
-	in, _ := validSQSStoreInput(t)
-	require.NoError(t, in.ValidateSQSStore())
-
-	_, seed := validSQSStoreInput(t)
-	account, err := nkeys.CreateAccount()
-	require.NoError(t, err)
-	accountPub, err := account.PublicKey()
-	require.NoError(t, err)
-
-	cases := map[string]func(*SystemInstanceInput){
-		"no kind":          func(in *SystemInstanceInput) { in.Kind = "" },
-		"other kind":       func(in *SystemInstanceInput) { in.Kind = "lambda" },
-		"ami boot":         func(in *SystemInstanceInput) { in.BootMode = BootAMI },
-		"no store input":   func(in *SystemInstanceInput) { in.SQSStore = nil },
-		"no volume":        func(in *SystemInstanceInput) { in.SQSStore.DataVolumeID = "" },
-		"malformed volume": func(in *SystemInstanceInput) { in.SQSStore.DataVolumeID = "vol-XYZ" },
-		"no key":           func(in *SystemInstanceInput) { in.SQSStore.ClientNKey = "" },
-		"seed as key":      func(in *SystemInstanceInput) { in.SQSStore.ClientNKey = string(seed) },
-		"account key":      func(in *SystemInstanceInput) { in.SQSStore.ClientNKey = accountPub },
-		"lb agent env":     func(in *SystemInstanceInput) { in.LBAgentEnv = "LB_ID=x" },
-		"lb ca cert":       func(in *SystemInstanceInput) { in.CACert = "cert" },
-		"lb scheme":        func(in *SystemInstanceInput) { in.Scheme = "internal" },
-	}
-	for name, mutate := range cases {
-		t.Run(name, func(t *testing.T) {
-			in, _ := validSQSStoreInput(t)
-			mutate(in)
-			err := in.ValidateSQSStore()
-			require.ErrorIs(t, err, ErrInvalidSQSStoreInput)
-			assert.NotContains(t, err.Error(), string(seed), "a seed must never be echoed")
-		})
-	}
-}
-
-func TestValidateSQSStore_VolumeIDMatchesTheGuestSerial(t *testing.T) {
-	accepted := identifiers.GenerateResourceID("vol")
-	in, _ := validSQSStoreInput(t)
-	in.SQSStore.DataVolumeID = accepted
-	require.NoError(t, in.ValidateSQSStore(), "a platform-generated volume ID is accepted")
-	// The guest accepts only vol + 17 lowercase hex, the full 20-byte serial.
-	assert.Len(t, strings.ReplaceAll(accepted, "-", ""), 20)
-
-	for _, id := range []string{
-		// 16 hex
-		"vol-0123456789abcdef",
-		// 18 hex: serial over 20 bytes
-		"vol-0123456789abcdef01",
-		// 8 hex
-		"vol-01234567",
-		// upper case
-		"vol-0123456789ABCDEF0",
-		// no dash
-		"vol0123456789abcdef0",
-		// not a volume
-		"snap-0123456789abcdef0",
-		// trailing newline
-		"vol-0123456789abcdef0\n",
-	} {
-		in, _ := validSQSStoreInput(t)
-		in.SQSStore.DataVolumeID = id
-		require.ErrorIs(t, in.ValidateSQSStore(), ErrInvalidSQSStoreInput, id)
-	}
-}
-
-func TestRefuseNonLoadBalancerKind(t *testing.T) {
-	require.NoError(t, (&SystemInstanceInput{InstanceType: "sys.micro"}).RefuseNonLoadBalancerKind())
-	in, _ := validSQSStoreInput(t)
-	require.ErrorIs(t, in.RefuseNonLoadBalancerKind(), ErrInvalidSQSStoreInput)
-	require.Error(t, (&SystemInstanceInput{SQSStore: &SQSStoreInput{}}).RefuseNonLoadBalancerKind())
-}

spinifex/domains/ec2/systeminstance/sqs_worker.go

@@ -34,8 +34,6 @@ func (in *SystemInstanceInput) ValidateSQSWorker() error {
 		return fmt.Errorf("%w: boot mode %q, the worker boots direct only", ErrInvalidSQSWorkerInput, in.BootMode)
 	case in.SQSWorker == nil:
 		return fmt.Errorf("%w: no data volume or Region", ErrInvalidSQSWorkerInput)
-	case in.SQSStore != nil:
-		return fmt.Errorf("%w: SQS store settings on an SQS worker launch", ErrInvalidSQSWorkerInput)
 	case in.IamInstanceProfileArn == "":
 		return fmt.Errorf("%w: no instance profile", ErrInvalidSQSWorkerInput)
 	case in.LBAgentEnv != "" || in.CACert != "" || in.Scheme != "":

spinifex/domains/eks/awsapi/authz.go

@@ -50,11 +50,10 @@ var eksScopes = map[string][]resourceSource{
 	"ListClusters":    {sourceAny},
 
 	// Internal control-plane routes. The cluster's owning account is a path
-	// segment on these three, and the ARN names that account rather than the
+	// segment on these two, and the ARN names that account rather than the
 	// system caller's, because that is the account the handler reads.
 	"ListInternalAddons":   {sourceInternalCluster},
 	"GetRecoveryDirective": {sourceInternalCluster},
-	"ListInternalWorkers":  {sourceInternalCluster},
 	// The owning account arrives in the body on these two, and the ARN names
 	// that account for the same reason: it is the account the handler acts in.
 	"PublishInternal":    {sourceInternalBodyCluster},

spinifex/domains/eks/awsapi/authz_test.go

@@ -179,10 +179,6 @@ func TestResourceARNsInternalRoutesUsePathAccount(t *testing.T) {
 	assert.Equal(t,
 		[]string{"arn:aws:eks:ap-southeast-2:999999999999:cluster/prod"},
 		resolveARNs(t, "GetRecoveryDirective", []string{"prod", "999999999999", "i-abc"}, ""))
-
-	assert.Equal(t,
-		[]string{"arn:aws:eks:ap-southeast-2:999999999999:cluster/prod"},
-		resolveARNs(t, "ListInternalWorkers", []string{"prod", "999999999999"}, ""))
 }
 
 // An identifier the gate cannot read authorizes account-wide, leaving the

spinifex/domains/eks/awsapi/gateway_wire_test.go

@@ -93,46 +93,6 @@ func TestEKSRequest_InternalAddonsWireBody(t *testing.T) {
 		rec.Body.String())
 }
 
-// The node-gc agent reads the same REST-JSON shape: the Go field name, sorted
-// IDs drawn from every nodegroup record of the cluster.
-func TestEKSRequest_InternalWorkersWireBody(t *testing.T) {
-	const cpInstanceID = "i-cp0000000000001"
-	_, nc, js := testutil.StartTestJetStream(t)
-	kv, err := ekscluster.GetOrCreateAccountBucket(t.Context(), js, authzAccountID)
-	require.NoError(t, err)
-	require.NoError(t, ekscluster.PutClusterMeta(t.Context(), kv, &ekscluster.ClusterMeta{
-		Name:              "prod",
-		Status:            ekscluster.ClusterStatusActive,
-		ControlPlaneNodes: []ekscluster.ControlPlaneNode{{InstanceID: cpInstanceID}},
-	}))
-	for _, rec := range []*ekscluster.NodegroupRecord{
-		{ClusterName: "prod", Name: "ng-b", InstanceIDs: []string{"i-0ccc", "i-0aaa"}},
-		{ClusterName: "prod", Name: "ng-a", InstanceIDs: []string{"i-0bbb"}},
-	} {
-		require.NoError(t, ekscluster.PutNodegroupRecord(t.Context(), kv, rec))
-	}
-
-	svc, err := ekscluster.NewEKSServiceImpl(ekscluster.EKSServiceDeps{NATSConn: nc, Region: authzRegion})
-	require.NoError(t, err)
-	sub, err := nc.Subscribe("eks.ListManagedWorkers", func(m *nats.Msg) {
-		acct := natsmsg.AccountIDFromMsg(m)
-		natsmsg.ServeNATSRequestCtx(m, func(ctx context.Context, in *ekscluster.ListManagedWorkersInput) (*ekscluster.ListManagedWorkersOutput, error) {
-			return svc.ListManagedWorkers(ctx, in, acct)
-		})
-	})
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = sub.Unsubscribe() })
-
-	gw := scopedEKS(gwtest.Statement("Allow", "eks:*", "*"))
-	gw.nc = nc
-	rec := httptest.NewRecorder()
-	req := httptest.NewRequest(http.MethodGet, "/clusters/prod/internal-workers/"+authzAccountID, nil)
-	require.NoError(t, gw.as(cpAgentContext(cpInstanceID)).serve(rec, req))
-
-	assert.Equal(t, http.StatusOK, rec.Code)
-	assert.Equal(t, `{"InstanceIDs":["i-0aaa","i-0bbb","i-0ccc"]}`, rec.Body.String())
-}
-
 // CreateAddon and UpdateAddon hand their service-account role to iam:PassRole,
 // same as CreateCluster's roleArn and CreateNodegroup's nodeRole; an absent
 // role passes nothing.

spinifex/domains/eks/awsapi/internal_authz_test.go

@@ -97,8 +97,6 @@ func TestAuthorizeInternal_RejectsNonCPPrincipals(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			assertDenied(t, eksawsapi.AuthorizeInternal(t.Context(), nil, "ListInternalAddons",
 				tc.caller, []string{"alpha", tenantAccount}, nil))
-			assertDenied(t, eksawsapi.AuthorizeInternal(t.Context(), nil, "ListInternalWorkers",
-				tc.caller, []string{"alpha", tenantAccount}, nil))
 			assertDenied(t, eksawsapi.AuthorizeInternal(t.Context(), nil, "GetRecoveryDirective",
 				tc.caller, []string{"alpha", tenantAccount, cpInstanceID}, nil))
 			assertDenied(t, eksawsapi.AuthorizeInternal(t.Context(), nil, "PublishInternal",
@@ -126,8 +124,6 @@ func TestAuthorizeInternal_AllowsClusterMember(t *testing.T) {
 
 	require.NoError(t, eksawsapi.AuthorizeInternal(t.Context(), nc, "ListInternalAddons",
 		cpAgent(cpInstanceID), []string{"alpha", tenantAccount}, nil))
-	require.NoError(t, eksawsapi.AuthorizeInternal(t.Context(), nc, "ListInternalWorkers",
-		cpAgent(cpInstanceID), []string{"alpha", tenantAccount}, nil))
 	require.NoError(t, eksawsapi.AuthorizeInternal(t.Context(), nc, "GetRecoveryDirective",
 		cpAgent(cpInstanceID), []string{"alpha", tenantAccount, cpInstanceID}, nil))
 	require.NoError(t, eksawsapi.AuthorizeInternal(t.Context(), nc, "PublishInternal",

spinifex/domains/eks/awsapi/internal_workers.go

@@ -1,35 +0,0 @@
-package awsapi
-
-import (
-	"context"
-	"errors"
-
-	ekscluster "github.com/mulgadc/spinifex/spinifex/domains/eks/cluster"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/nats-io/nats.go"
-)
-
-// internalWorkersOutput is the body returned to the on-VM node-gc agent: the
-// instance ID of every worker the cluster's nodegroups still own.
-type internalWorkersOutput struct {
-	InstanceIDs []string `json:"instanceIds"`
-}
-
-// ListInternalWorkers — GET /clusters/{name}/internal-workers/{acct}.
-// Internal control-plane VM route (not an AWS-SDK action): same carve-out as
-// ListInternalAddons, gated by AuthorizeInternal, which admits only a CP agent
-// serving that account's cluster.
-func ListInternalWorkers(ctx context.Context, natsConn *nats.Conn, clusterName, accountID string) (*internalWorkersOutput, error) {
-	if natsConn == nil {
-		return nil, errors.New(awserrors.ErrorServerInternal)
-	}
-	if clusterName == "" || accountID == "" {
-		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
-	}
-	out, err := ekscluster.NewNATSEKSService(natsConn).ListManagedWorkers(ctx,
-		&ekscluster.ListManagedWorkersInput{ClusterName: clusterName}, accountID)
-	if err != nil {
-		return nil, err
-	}
-	return &internalWorkersOutput{InstanceIDs: out.InstanceIDs}, nil
-}

spinifex/domains/eks/awsapi/routes.go

@@ -60,14 +60,6 @@ var routes = []route{
 			return GetRecoveryDirective(ctx, nc, p[0], p[1], p[2])
 		}},
 
-	// Internal control-plane route: the on-VM node-gc agent pulls the cluster's
-	// live nodegroup workers to delete the Nodes of terminated ones. Same
-	// system-cred carve-out as internal-addons.
-	{Method: "GET", Pattern: "/clusters/{clusterName}/internal-workers/{accountId}", Action: "ListInternalWorkers",
-		Handler: func(ctx context.Context, nc *nats.Conn, acct, callerARN string, p []string, b []byte) (any, error) {
-			return ListInternalWorkers(ctx, nc, p[0], p[1])
-		}},
-
 	// Nodegroup
 	{Method: "POST", Pattern: "/clusters/{clusterName}/node-groups", Action: "CreateNodegroup",
 		Handler: func(ctx context.Context, nc *nats.Conn, acct, callerARN string, p []string, b []byte) (any, error) {

spinifex/domains/eks/awsapi/routes_test.go

@@ -108,7 +108,6 @@ func TestLookupEKSAction_CoversAllActions(t *testing.T) {
 		"WebhookTokenReview":           false,
 		"ListInternalAddons":           false,
 		"GetRecoveryDirective":         false,
-		"ListInternalWorkers":          false,
 		"DescribeCluster":              false,
 		"ListClusters":                 false,
 		"DeleteCluster":                false,

spinifex/domains/eks/cluster/k3s_server_role.go

@@ -22,10 +22,9 @@ const (
 	// WebhookTokenReview (the eks-token-webhook relays `aws eks get-token` bearer
 	// tokens to the token-review broker for host-side STS verification), and
 	// GetRecoveryDirective (the k3s-recovery agent pulls its per-member etcd
-	// recovery directive at boot), and ListInternalWorkers (the node-gc agent
-	// pulls the live nodegroup workers). The gateway evaluates these per request
-	// against the role's policies.
-	eksServerInlinePolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["eks:PublishInternal","eks:ListInternalAddons","eks:WebhookTokenReview","eks:GetRecoveryDirective","eks:ListInternalWorkers"],"Resource":"*"}]}`
+	// recovery directive at boot). The gateway evaluates these per request against
+	// the role's policies.
+	eksServerInlinePolicy = `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["eks:PublishInternal","eks:ListInternalAddons","eks:WebhookTokenReview","eks:GetRecoveryDirective"],"Resource":"*"}]}`
 )
 
 // ensureCPInstanceProfile returns the CP instance-profile ARN, or "" to signal
@@ -36,7 +35,7 @@ func (s *EKSServiceImpl) ensureCPInstanceProfile(accountID string) string {
 	iam := s.iamEnsurer()
 	if iam == nil {
 		slog.Warn("EKS: IAM service unwired; CP VM falls back to baked static gateway creds and "+
-			"will be denied eks:PublishInternal, eks:WebhookTokenReview, eks:ListInternalAddons, eks:GetRecoveryDirective and eks:ListInternalWorkers (no instance identity to bind)",
+			"will be denied eks:PublishInternal, eks:WebhookTokenReview, eks:ListInternalAddons and eks:GetRecoveryDirective (no instance identity to bind)",
 			"accountID", accountID)
 		return ""
 	}
@@ -44,7 +43,7 @@ func (s *EKSServiceImpl) ensureCPInstanceProfile(accountID string) string {
 		CPInstanceRoleName, eksServerInlinePolicyName, eksServerInlinePolicy)
 	if err != nil {
 		slog.Error("EKS: ensure CP instance profile failed; CP VM falls back to baked static gateway creds and "+
-			"will be denied eks:PublishInternal, eks:WebhookTokenReview, eks:ListInternalAddons, eks:GetRecoveryDirective and eks:ListInternalWorkers (no instance identity to bind)",
+			"will be denied eks:PublishInternal, eks:WebhookTokenReview, eks:ListInternalAddons and eks:GetRecoveryDirective (no instance identity to bind)",
 			"accountID", accountID, "role", CPInstanceRoleName, "err", err)
 		return ""
 	}

spinifex/domains/eks/cluster/k3s_server_role_test.go

@@ -38,9 +38,6 @@ func TestEnsureK3sServerInstanceProfile_CreatesRolePolicyProfile(t *testing.T) {
 	// these IMDS creds; without this action the gateway 403s the boot-time fetch
 	// and a wedged control plane can never be auto-reset.
 	assert.Contains(t, policy, "eks:GetRecoveryDirective")
-	// The node-gc agent pulls the live nodegroup workers with these IMDS creds;
-	// without this action a terminated worker's Node is never deleted.
-	assert.Contains(t, policy, "eks:ListInternalWorkers")
 
 	require.NotNil(t, f.Profiles[CPInstanceRoleName])
 	assert.Len(t, f.Profiles[CPInstanceRoleName].Roles, 1)

spinifex/domains/eks/cluster/service.go

@@ -44,10 +44,6 @@ type EKSService interface {
 	// AWS-SDK action): the on-VM addon-sync agent fetches staged manifests for a
 	// cluster via the internal-addons gateway route.
 	ListStagedAddonManifests(ctx context.Context, input *ListStagedAddonManifestsInput, accountID string) (*ListStagedAddonManifestsOutput, error)
-	// ListManagedWorkers is an internal control-plane method (not an AWS-SDK
-	// action): the on-VM node-gc agent fetches the cluster's live nodegroup
-	// workers via the internal-workers gateway route.
-	ListManagedWorkers(ctx context.Context, input *ListManagedWorkersInput, accountID string) (*ListManagedWorkersOutput, error)
 
 	// GetRecoveryDirective / SetRecoveryDirective are internal control-plane
 	// methods (not AWS-SDK actions): the on-VM k3s-recovery agent fetches its

spinifex/domains/eks/cluster/service_nats.go

@@ -129,10 +129,6 @@ func (s *NATSEKSService) ListStagedAddonManifests(ctx context.Context, input *Li
 	return natsmsg.NATSRequest[ListStagedAddonManifestsOutput](ctx, s.natsConn, "eks.ListStagedAddonManifests", input, defaultTimeout, accountID)
 }
 
-func (s *NATSEKSService) ListManagedWorkers(ctx context.Context, input *ListManagedWorkersInput, accountID string) (*ListManagedWorkersOutput, error) {
-	return natsmsg.NATSRequest[ListManagedWorkersOutput](ctx, s.natsConn, "eks.ListManagedWorkers", input, defaultTimeout, accountID)
-}
-
 func (s *NATSEKSService) UpdateAddon(ctx context.Context, input *eks.UpdateAddonInput, accountID string) (*eks.UpdateAddonOutput, error) {
 	return natsmsg.NATSRequest[eks.UpdateAddonOutput](ctx, s.natsConn, "eks.UpdateAddon", input, defaultTimeout, accountID)
 }

spinifex/domains/eks/cluster/worker_membership.go

@@ -1,46 +0,0 @@
-package cluster
-
-import (
-	"context"
-	"errors"
-	"slices"
-
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-)
-
-// ListManagedWorkersInput names the cluster whose managed-nodegroup workers to
-// list. It is an internal control-plane request (not an AWS-SDK shape), served
-// over NATS for the guest node-gc agent via the internal-workers route.
-type ListManagedWorkersInput struct {
-	ClusterName string `json:"clusterName"`
-}
-
-// ListManagedWorkersOutput carries the instance ID of every worker a managed
-// nodegroup of the cluster still owns, sorted.
-type ListManagedWorkersOutput struct {
-	InstanceIDs []string `json:"instanceIds"`
-}
-
-// ListManagedWorkers returns every worker instance ID recorded on the cluster's
-// nodegroups. A nodegroup drops an ID only after terminating it (DeleteNodegroup,
-// scale-down, reclaim), so the guest deletes a nodegroup Node whose instance is
-// absent here, as the AWS cloud controller deletes a Node whose instance is gone.
-func (s *EKSServiceImpl) ListManagedWorkers(ctx context.Context, input *ListManagedWorkersInput, accountID string) (*ListManagedWorkersOutput, error) {
-	if input == nil || input.ClusterName == "" {
-		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
-	}
-	acctKV, err := s.acctKVForCluster(ctx, accountID, input.ClusterName)
-	if err != nil {
-		return nil, err
-	}
-	recs, err := ListNodegroupRecords(ctx, acctKV, input.ClusterName)
-	if err != nil {
-		return nil, err
-	}
-	ids := make([]string, 0)
-	for _, rec := range recs {
-		ids = append(ids, rec.InstanceIDs...)
-	}
-	slices.Sort(ids)
-	return &ListManagedWorkersOutput{InstanceIDs: slices.Compact(ids)}, nil
-}

spinifex/domains/eks/cluster/worker_membership_test.go

@@ -1,74 +0,0 @@
-package cluster
-
-import (
-	"context"
-	"testing"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/eks"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func managedWorkers(t *testing.T, f *eksServiceFixture, cluster string) []string {
-	t.Helper()
-	out, err := f.svc.ListManagedWorkers(context.Background(), &ListManagedWorkersInput{ClusterName: cluster}, testAccountID)
-	require.NoError(t, err)
-	return out.InstanceIDs
-}
-
-// A worker leaves the managed set exactly when its nodegroup terminates it, by
-// scale-down or DeleteNodegroup, so the guest node-gc deletes its Node then and
-// leaves every worker still owned by a nodegroup alone.
-func TestListManagedWorkers_DropsWorkersTheNodegroupTerminated(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-	seedActiveClusterWithToken(t, f, "c1")
-	for _, ng := range []string{"ng1", "ng2"} {
-		_, err := f.svc.CreateNodegroup(context.Background(), createNGInput("c1", ng, 2), testAccountID)
-		require.NoError(t, err)
-		markWorkersReady(t, f, "c1", ng, 2)
-		f.svc.WaitLaunches()
-	}
-	ng1, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng1")
-	require.NoError(t, err)
-	ng2, err := GetNodegroupRecord(t.Context(), f.kv, "c1", "ng2")
-	require.NoError(t, err)
-	require.Len(t, ng1.InstanceIDs, 2)
-	assert.ElementsMatch(t, append(append([]string{}, ng1.InstanceIDs...), ng2.InstanceIDs...), managedWorkers(t, f, "c1"))
-
-	_, err = f.svc.UpdateNodegroupConfig(context.Background(), &eks.UpdateNodegroupConfigInput{
-		ClusterName:   aws.String("c1"),
-		NodegroupName: aws.String("ng1"),
-		ScalingConfig: &eks.NodegroupScalingConfig{DesiredSize: aws.Int64(1)},
-	}, testAccountID)
-	require.NoError(t, err)
-	require.Len(t, f.worker.terminateCalls, 1)
-	scaledIn := f.worker.terminateCalls[0]
-	got := managedWorkers(t, f, "c1")
-	assert.Len(t, got, 3)
-	for _, id := range scaledIn {
-		assert.NotContains(t, got, id, "a worker terminated by scale-down must leave the managed set")
-	}
-
-	_, err = f.svc.DeleteNodegroup(context.Background(), &eks.DeleteNodegroupInput{
-		ClusterName: aws.String("c1"), NodegroupName: aws.String("ng2"),
-	}, testAccountID)
-	require.NoError(t, err)
-	got = managedWorkers(t, f, "c1")
-	assert.Len(t, got, 1)
-	for _, id := range ng2.InstanceIDs {
-		assert.NotContains(t, got, id, "a deleted nodegroup's workers must leave the managed set")
-	}
-}
-
-func TestListManagedWorkers_UnknownClusterIsNotFound(t *testing.T) {
-	t.Parallel()
-	f := newEKSServiceFixture(t)
-
-	_, err := f.svc.ListManagedWorkers(context.Background(), &ListManagedWorkersInput{ClusterName: "ghost"}, testAccountID)
-	code, ok := awserrors.ResolveErrorCode(err)
-	require.True(t, ok)
-	assert.Equal(t, awserrors.ErrorEKSResourceNotFound, code)
-}

spinifex/domains/elbv2/system_instance_launcher_nats_test.go

@@ -294,8 +294,8 @@ func TestNATSSystemInstanceLauncher_RefusesNonLoadBalancerKindBeforeSending(t *t
 
 	launcher := NewNATSSystemInstanceLauncher(nc, 2*time.Second)
 	for name, input := range map[string]*SystemInstanceInput{
-		"kind":            {InstanceType: "sys.micro", Kind: systeminstance.KindSQSStore},
-		"sqs store input": {InstanceType: "sys.micro", SQSStore: &systeminstance.SQSStoreInput{DataVolumeID: "vol-0123456789abcdef0"}},
+		"kind":             {InstanceType: "sys.micro", Kind: systeminstance.KindSQSWorker},
+		"sqs worker input": {InstanceType: "sys.micro", SQSWorker: &systeminstance.SQSWorkerInput{}},
 	} {
 		out, err := launcher.LaunchSystemInstance(input)
 		require.Error(t, err, name)

spinifex/domains/sqs/awsapi/registration.go

@@ -141,6 +141,11 @@ func OperationNames() []string {
 	return names
 }
 
+// OperationInventory lists the AWS model actions the registration serves.
+func OperationInventory() dispatch.Inventory {
+	return dispatch.Inventory{Registered: OperationNames()}
+}
+
 // queryErrorCodes are the legacy query codes the SQS model gives error shapes
 // whose name differs, as aws-sdk-go v1 records them, and the Coral framework
 // errors' query codes as captured from AWS.

spinifex/domains/sqs/awsapi/worker/contract_test.go

@@ -9,7 +9,8 @@ import (
 
 // The registration dispatches exactly the actions the worker role is granted.
 func TestActionNames_MatchTheRoleGrant(t *testing.T) {
-	if got, want := ActionNames(), workerrole.Actions(); !slices.Equal(got, want) {
+	want := slices.Sorted(slices.Values(workerrole.Actions()))
+	if got := ActionNames(); !slices.Equal(got, want) {
 		t.Fatalf("dispatched actions %v, role grants %v", got, want)
 	}
 }

spinifex/domains/sqs/awsapi/worker/worker.go

@@ -6,6 +6,8 @@ package worker
 import (
 	"bytes"
 	"context"
+	"encoding/base64"
+	"encoding/json"
 	"encoding/xml"
 	"errors"
 	"fmt"
@@ -13,29 +15,44 @@ import (
 	"log/slog"
 	"maps"
 	"net/http"
+	"regexp"
 	"slices"
+	"time"
 
 	"github.com/mulgadc/bluebottle/pkg/sigv4"
-	"github.com/mulgadc/spinifex/spinifex/domains/sqs/firststore"
-	"github.com/mulgadc/spinifex/spinifex/domains/sqs/storelaunch"
+	"github.com/mulgadc/spinifex/spinifex/domains/sqs/relay"
 	"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerident"
 	"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerrole"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/query"
 	"github.com/nats-io/nats.go/jetstream"
-	"github.com/nats-io/nkeys"
 )
 
 // The contract lives in workerrole so the role grant can read it without
 // importing this adapter.
 const (
-	ServiceName                 = workerrole.ServiceName
-	WorkerRoleName              = workerrole.RoleName
-	ActionClaimFirstStore       = workerrole.ActionClaimFirstStore
-	ActionDescribeStore         = workerrole.ActionDescribeStore
-	ActionRecordWorkerPublicKey = workerrole.ActionRecordWorkerPublicKey
+	ServiceName            = workerrole.ServiceName
+	WorkerRoleName         = workerrole.RoleName
+	ActionClaimReceiptKey  = workerrole.ActionClaimReceiptKey
+	ActionRecordReceiptKey = workerrole.ActionRecordReceiptKey
+	ActionPollWork         = workerrole.ActionPollWork
+	ActionCompleteWork     = workerrole.ActionCompleteWork
+)
+
+// pollWait bounds one PollWork long poll; pollBatch and pollBytes bound the
+// items one poll hands out.
+const (
+	pollWait  = 20 * time.Second
+	pollBatch = 16
+	pollBytes = 16 << 20
+)
+
+var (
+	workIDPattern      = regexp.MustCompile(`^[0-9a-f]{32}$`)
+	fingerprintPattern = regexp.MustCompile(`^[0-9a-f]{64}$`)
 )
 
 const principalTypeAssumedRole = "assumed-role"
@@ -43,9 +60,10 @@ const principalTypeAssumedRole = "assumed-role"
 // params lists the only parameters each action accepts. Identity is never a
 // parameter: it is derived from the session and the current worker record.
 var params = map[string][]string{
-	ActionClaimFirstStore:       {},
-	ActionDescribeStore:         {},
-	ActionRecordWorkerPublicKey: {"PublicKey"},
+	ActionClaimReceiptKey:  {},
+	ActionRecordReceiptKey: {"Fingerprint"},
+	ActionPollWork:         {},
+	ActionCompleteWork:     {"WorkId"},
 }
 
 // ActionNames lists every action the registration dispatches, sorted.
@@ -53,34 +71,32 @@ func ActionNames() []string {
 	return slices.Sorted(maps.Keys(params))
 }
 
-// Identities resolves a worker session to its current authoritative identity.
+// Identities resolves a worker session to its current authoritative identity
+// and records its receipt key fingerprint.
 type Identities interface {
 	Authorize(ctx context.Context, instanceID string) (workerident.Identity, error)
+	RecordReceiptKey(ctx context.Context, id workerident.Identity, fingerprint string) error
 }
 
-// FirstStore is the first-store decision the worker's key handling consults.
-type FirstStore interface {
-	Claim(ctx context.Context, region string, id firststore.Identity) (firststore.Decision, error)
-	RecordPublicKey(ctx context.Context, region string, id firststore.Identity, pub string) error
+var _ Identities = workerident.Store{}
+
+// Work is the relay the worker pulls owner calls from and posts results to.
+type Work interface {
+	Take(ctx context.Context, taker relay.Taker, limit, maxBytes int) []relay.Work
+	Complete(taker relay.Taker, id string, res relay.Result) (relay.Outcome, error)
 }
 
-// Stores reads the Region's store launch record from the stream leader.
-type Stores interface {
-	Current(ctx context.Context, region string) (storelaunch.Record, error)
-}
+var _ Work = (*relay.Relay)(nil)
 
-var (
-	_ Stores     = storelaunch.Records{}
-	_ Identities = workerident.Store{}
-	_ FirstStore = firststore.Store{}
-)
-
-// Deps are the capabilities the actions run against. Either one nil answers
-// every authorized request with ServiceUnavailable, never a decision.
+// Deps are the capabilities the actions run against. Any one nil answers
+// every authorized request with ServiceUnavailable, never a decision; a nil
+// Work does so for the work actions only.
 type Deps struct {
 	Identities Identities
-	FirstStore FirstStore
-	Stores     Stores
+	Work       Work
+	// HostNodes reports this gateway's declared cluster node count; nil is
+	// clustersize.Nodes.
+	HostNodes func() int
 }
 
 // NewRegistration returns the internal SQS worker Query registration.
@@ -103,7 +119,12 @@ func resolveAction(r *http.Request) string {
 }
 
 // readQueryArgs parses the Query body and rewinds it for a second read.
+// CompleteWork carries its arguments in the signed query string instead,
+// because its body is the result.
 func readQueryArgs(r *http.Request) (map[string]string, error) {
+	if r.URL.Query().Get("Action") == ActionCompleteWork {
+		return query.ParseArgs(r.URL.RawQuery)
+	}
 	body, err := io.ReadAll(io.LimitReader(r.Body, sigv4.MaxPayloadLen+1))
 	if err != nil {
 		return nil, err
@@ -135,7 +156,7 @@ func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 	}
 
 	actor, _ := inv.Actor()
-	session, err := requireWorkerPrincipal(ctx, inv.AccountID, actor)
+	session, err := requirePrincipal(ctx, inv.AccountID, actor, WorkerRoleName)
 	if err != nil {
 		return err
 	}
@@ -149,8 +170,7 @@ func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 			return errors.New(awserrors.ErrorInvalidParameterValue)
 		}
 	}
-
-	if d.Identities == nil || d.FirstStore == nil || d.Stores == nil {
+	if d.Identities == nil {
 		return errors.New(awserrors.ErrorServiceUnavailable)
 	}
 	ident, err := d.Identities.Authorize(ctx, session)
@@ -163,40 +183,57 @@ func (d Deps) dispatch(w http.ResponseWriter, inv dispatch.Invocation) error {
 		slog.DebugContext(ctx, "SQS internal: worker bound to another Region", "instanceID", session)
 		return errors.New(awserrors.ErrorAccessDenied)
 	}
-	id := firststore.Identity{InstanceID: ident.InstanceID, ClaimID: ident.ClaimID, Generation: ident.Generation}
 
 	switch action {
-	case ActionDescribeStore:
-		return d.describeStore(ctx, w, ident.Region)
-	case ActionClaimFirstStore:
-		dec, err := d.FirstStore.Claim(ctx, ident.Region, id)
-		if err != nil {
-			return mapError(ctx, action, err)
+	case ActionPollWork, ActionCompleteWork:
+		if d.Work == nil {
+			return errors.New(awserrors.ErrorServiceUnavailable)
 		}
-		return writeXML(ctx, w, claimResponse{Result: claimResult{CreateKey: dec.CreateKey, PublicKey: dec.PublicKey}})
+		taker := relay.Taker{InstanceID: ident.InstanceID, Generation: ident.Generation, ClaimID: ident.ClaimID}
+		if action == ActionPollWork {
+			return d.pollWork(ctx, w, taker)
+		}
+		return d.completeWork(ctx, w, r, taker, args["WorkId"])
+	case ActionClaimReceiptKey:
+		return d.claimReceiptKey(ctx, w, ident)
 	default:
-		pub := args["PublicKey"]
-		if pub == "" {
+		fp := args["Fingerprint"]
+		if fp == "" {
 			return errors.New(awserrors.ErrorMissingParameter)
 		}
-		// A seed or any other value is refused without echoing it anywhere.
-		if !nkeys.IsValidPublicUserKey(pub) {
+		if !fingerprintPattern.MatchString(fp) {
 			return errors.New(awserrors.ErrorInvalidParameterValue)
 		}
-		if err := d.FirstStore.RecordPublicKey(ctx, ident.Region, id, pub); err != nil {
+		if err := d.Identities.RecordReceiptKey(ctx, ident, fp); err != nil {
 			return mapError(ctx, action, err)
 		}
-		return writeXML(ctx, w, recordResponse{})
+		return writeXML(ctx, w, recordReceiptResponse{})
 	}
 }
 
-// requireWorkerPrincipal admits only a session of the worker instance role in
-// the system account, and returns its session name: the instance ID STS set.
-// It says the caller is an SQS worker VM, not that it is the current one.
-func requireWorkerPrincipal(ctx context.Context, accountID string, actor dispatch.Actor) (string, error) {
+// claimReceiptKey reports the worker volume's receipt key initialization. A
+// record with none never authorizes creating a key: its volume is not proven
+// new. HostNodes lets the guest refuse anything but a single-node cluster.
+func (d Deps) claimReceiptKey(ctx context.Context, w http.ResponseWriter, ident workerident.Identity) error {
+	res := claimReceiptResult{HostNodes: d.hostNodes()}
+	switch ident.ReceiptInit {
+	case workerident.ReceiptPending:
+		res.Pending = true
+	case workerident.ReceiptRecorded:
+		res.Fingerprint = ident.ReceiptFingerprint
+	default:
+		return mapError(ctx, ActionClaimReceiptKey, workerident.ErrReceiptUnknown)
+	}
+	return writeXML(ctx, w, claimReceiptResponse{Result: res})
+}
+
+// requirePrincipal admits only a session of role, a system-account instance
+// role, and returns its session name: the instance ID STS set. It says the
+// caller is a VM under that role, not that it is the current one.
+func requirePrincipal(ctx context.Context, accountID string, actor dispatch.Actor, role string) (string, error) {
 	if actor.PrincipalType != principalTypeAssumedRole ||
 		accountID != awsidentifiers.GlobalAccountID || actor.AccountID != awsidentifiers.GlobalAccountID ||
-		actor.RoleName() != WorkerRoleName || actor.SessionName == "" {
+		actor.RoleName() != role || actor.SessionName == "" {
 		slog.DebugContext(ctx, "SQS internal: rejected non-worker caller",
 			"principalType", actor.PrincipalType, "accountID", accountID)
 		return "", errors.New(awserrors.ErrorAccessDenied)
@@ -208,14 +245,13 @@ func requireWorkerPrincipal(ctx context.Context, accountID string, actor dispatc
 // refusals are AccessDenied, contention is retryable, the rest are faults.
 func mapError(ctx context.Context, action string, err error) error {
 	switch {
-	case errors.Is(err, workerident.ErrUnknownInstance), errors.Is(err, workerident.ErrNotCurrent),
-		errors.Is(err, firststore.ErrClaimHeld), errors.Is(err, firststore.ErrStaleGeneration):
+	case errors.Is(err, workerident.ErrUnknownInstance), errors.Is(err, workerident.ErrNotCurrent):
 		return errors.New(awserrors.ErrorAccessDenied)
-	case errors.Is(err, firststore.ErrKeyMismatch):
+	case errors.Is(err, workerident.ErrReceiptRecorded):
 		return errors.New(awserrors.ErrorInvalidParameterValue)
-	case errors.Is(err, firststore.ErrNotClaimed):
+	case errors.Is(err, workerident.ErrReceiptUnknown):
 		return errors.New(awserrors.ErrorInvalidState)
-	case errors.Is(err, workerident.ErrContended), errors.Is(err, firststore.ErrContended):
+	case errors.Is(err, workerident.ErrContended):
 		return errors.New(awserrors.ErrorServiceUnavailable)
 	case errors.Is(err, errUnavailable):
 		slog.WarnContext(ctx, "SQS internal: worker stores unavailable", "action", action, "err", err)
@@ -225,55 +261,110 @@ func mapError(ctx context.Context, action string, err error) error {
 	return errors.New(awserrors.ErrorServerInternal)
 }
 
-// describeStore reports the Region's store launch: pending until it is bound,
-// then the address and the public key it was launched for. Bound is not
-// ready; the worker checks the store itself with its own key.
-func (d Deps) describeStore(ctx context.Context, w http.ResponseWriter, region string) error {
-	rec, err := d.Stores.Current(ctx, region)
-	switch {
-	case errors.Is(err, jetstream.ErrKeyNotFound):
-		return writeXML(ctx, w, describeResponse{Result: describeResult{State: StorePending}})
-	case errors.Is(err, storelaunch.ErrUnreadable), errors.Is(err, storelaunch.ErrInvalidRecording):
-		return mapError(ctx, ActionDescribeStore, err)
-	case err != nil:
-		return mapError(ctx, ActionDescribeStore, fmt.Errorf("%w: %w", errUnavailable, err))
-	case rec.State != storelaunch.StateBound:
-		return writeXML(ctx, w, describeResponse{Result: describeResult{State: StorePending}})
+func (d Deps) hostNodes() int {
+	if d.HostNodes != nil {
+		return d.HostNodes()
 	}
-	return writeXML(ctx, w, describeResponse{Result: describeResult{
-		State: StoreBound, Address: rec.Address, PublicKey: rec.PublicKey,
-	}})
+	return clustersize.Nodes()
 }
 
-// DescribeStore states.
+// pollWork hands this worker queued owner calls, waiting up to pollWait.
+// Each item is the Work JSON, base64 encoded.
+func (d Deps) pollWork(ctx context.Context, w http.ResponseWriter, taker relay.Taker) error {
+	pollCtx, cancel := context.WithTimeout(ctx, pollWait)
+	defer cancel()
+	works := d.Work.Take(pollCtx, taker, pollBatch, pollBytes)
+	res := pollResult{Items: make([]string, 0, len(works))}
+	for _, wk := range works {
+		b, err := json.Marshal(wk)
+		if err != nil {
+			return errors.New(awserrors.ErrorServerInternal)
+		}
+		res.Items = append(res.Items, base64.StdEncoding.EncodeToString(b))
+	}
+	return writeXML(ctx, w, pollResponse{Result: res})
+}
+
+// completeWork posts the result in the body for the item the same worker
+// identity took. The outcome tells the worker whether delivery is disproved.
+func (d Deps) completeWork(ctx context.Context, w http.ResponseWriter, r *http.Request, taker relay.Taker, id string) error {
+	if !workIDPattern.MatchString(id) {
+		return errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	body, err := io.ReadAll(io.LimitReader(r.Body, sigv4.MaxPayloadLen+1))
+	if err != nil {
+		return errors.New(awserrors.ErrorMalformedQueryString)
+	}
+	if int64(len(body)) > sigv4.MaxPayloadLen {
+		return errors.New(awserrors.ErrorRequestEntityTooLarge)
+	}
+	var res relay.Result
+	dec := json.NewDecoder(bytes.NewReader(body))
+	dec.DisallowUnknownFields()
+	if err := dec.Decode(&res); err != nil || (res.Error == nil) == (res.Output == nil) {
+		return errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	outcome, err := d.Work.Complete(taker, id, res)
+	if errors.Is(err, relay.ErrNotTaker) {
+		return errors.New(awserrors.ErrorAccessDenied)
+	}
+	if err != nil {
+		return errors.New(awserrors.ErrorServerInternal)
+	}
+	return writeXML(ctx, w, completeResponse{Result: completeResult{Outcome: OutcomeName(outcome)}})
+}
+
+// CompleteWork outcomes on the wire.
 const (
-	StorePending = "pending"
-	StoreBound   = "bound"
+	OutcomeDelivered  = "delivered"
+	OutcomeCallerGone = "caller-gone"
+	OutcomeUnknown    = "unknown"
 )
 
-type describeResult struct {
-	State     string `xml:"State"`
-	Address   string `xml:"Address,omitempty"`
-	PublicKey string `xml:"PublicKey,omitempty"`
+// OutcomeName is the wire name of o.
+func OutcomeName(o relay.Outcome) string {
+	switch o {
+	case relay.Delivered:
+		return OutcomeDelivered
+	case relay.CallerGone:
+		return OutcomeCallerGone
+	}
+	return OutcomeUnknown
 }
 
-type describeResponse struct {
-	XMLName xml.Name       `xml:"DescribeStoreResponse"`
-	Result  describeResult `xml:"DescribeStoreResult"`
+type pollResult struct {
+	Items []string `xml:"Items>member"`
 }
 
-type claimResult struct {
-	CreateKey bool   `xml:"CreateKey"`
-	PublicKey string `xml:"PublicKey,omitempty"`
+type pollResponse struct {
+	XMLName xml.Name   `xml:"PollWorkResponse"`
+	Result  pollResult `xml:"PollWorkResult"`
 }
 
-type claimResponse struct {
-	XMLName xml.Name    `xml:"ClaimFirstStoreResponse"`
-	Result  claimResult `xml:"ClaimFirstStoreResult"`
+type completeResult struct {
+	Outcome string `xml:"Outcome"`
 }
 
-type recordResponse struct {
-	XMLName xml.Name `xml:"RecordWorkerPublicKeyResponse"`
+type completeResponse struct {
+	XMLName xml.Name       `xml:"CompleteWorkResponse"`
+	Result  completeResult `xml:"CompleteWorkResult"`
+}
+
+type claimReceiptResult struct {
+	Pending     bool   `xml:"Pending"`
+	Fingerprint string `xml:"Fingerprint,omitempty"`
+	// HostNodes is this gateway's declared cluster node count, zero when
+	// undeclared; the guest serves only when it is exactly one.
+	HostNodes int `xml:"HostNodes"`
+}
+
+type claimReceiptResponse struct {
+	XMLName xml.Name           `xml:"ClaimReceiptKeyResponse"`
+	Result  claimReceiptResult `xml:"ClaimReceiptKeyResult"`
+}
+
+type recordReceiptResponse struct {
+	XMLName xml.Name `xml:"RecordReceiptKeyResponse"`
 }
 
 func writeXML(ctx context.Context, w http.ResponseWriter, v any) error {
@@ -289,14 +380,14 @@ func writeXML(ctx context.Context, w http.ResponseWriter, v any) error {
 	return nil
 }
 
-// JetStreamDeps binds the existing worker-identity and first-store buckets per
-// request and never creates them, so a gateway provisions nothing and refuses
-// every request until the worker launch path has created them durably.
+// JetStreamDeps binds the existing worker-identity bucket per request and
+// never creates it, so a gateway provisions nothing and refuses every request
+// until the worker launch path has created it durably.
 func JetStreamDeps(js jetstream.JetStream) Deps {
 	if js == nil {
 		return Deps{}
 	}
-	return Deps{Identities: boundIdentities{js}, FirstStore: boundFirstStore{js}, Stores: boundStores{js}}
+	return Deps{Identities: boundIdentities{js}}
 }
 
 // errUnavailable marks a bucket that could not be bound: absent, unreachable
@@ -313,36 +404,12 @@ func (b boundIdentities) Authorize(ctx context.Context, instanceID string) (work
 	return s.Authorize(ctx, instanceID)
 }
 
-type boundFirstStore struct{ js jetstream.JetStream }
-
-func (b boundFirstStore) Claim(ctx context.Context, region string, id firststore.Identity) (firststore.Decision, error) {
-	s, err := firststore.Bind(ctx, b.js)
-	if err != nil {
-		return firststore.Decision{}, fmt.Errorf("%w: %w", errUnavailable, err)
-	}
-	return s.Claim(ctx, region, id)
-}
-
-func (b boundFirstStore) RecordPublicKey(ctx context.Context, region string, id firststore.Identity, pub string) error {
-	s, err := firststore.Bind(ctx, b.js)
+func (b boundIdentities) RecordReceiptKey(ctx context.Context, id workerident.Identity, fingerprint string) error {
+	s, err := workerident.Bind(ctx, b.js)
 	if err != nil {
 		return fmt.Errorf("%w: %w", errUnavailable, err)
 	}
-	return s.RecordPublicKey(ctx, region, id, pub)
+	return s.RecordReceiptKey(ctx, id, fingerprint)
 }
 
-type boundStores struct{ js jetstream.JetStream }
-
-func (b boundStores) Current(ctx context.Context, region string) (storelaunch.Record, error) {
-	r, err := storelaunch.Bind(ctx, b.js)
-	if err != nil {
-		return storelaunch.Record{}, fmt.Errorf("%w: %w", errUnavailable, err)
-	}
-	return r.Current(ctx, region)
-}
-
-var (
-	_ Identities = boundIdentities{}
-	_ FirstStore = boundFirstStore{}
-	_ Stores     = boundStores{}
-)
+var _ Identities = boundIdentities{}

spinifex/domains/sqs/awsapi/worker/worker_test.go

@@ -8,20 +8,15 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"strings"
-	"sync"
 	"testing"
 
 	"github.com/mulgadc/bluebottle/pkg/iampolicy"
 	testutil "github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/spinifex/domains/sqs/firststore"
-	"github.com/mulgadc/spinifex/spinifex/domains/sqs/storelaunch"
 	"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerident"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
-	"github.com/nats-io/nats.go/jetstream"
-	"github.com/nats-io/nkeys"
 )
 
 const (
@@ -29,6 +24,8 @@ const (
 	instance  = "i-0abc12345def67890"
 	successor = "i-0fed98765cba43210"
 	volume    = "vol-0123456789abcdef0"
+	fpA       = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+	fpB       = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
 )
 
 func workerActor(session string) dispatch.Actor {
@@ -73,9 +70,11 @@ func assertCode(t *testing.T, err error, code string) {
 }
 
 type fakeIdentities struct {
-	ident workerident.Identity
-	err   error
-	asked []string
+	ident    workerident.Identity
+	err      error
+	asked    []string
+	recorded []string
+	recErr   error
 }
 
 func (f *fakeIdentities) Authorize(_ context.Context, instanceID string) (workerident.Identity, error) {
@@ -83,99 +82,87 @@ func (f *fakeIdentities) Authorize(_ context.Context, instanceID string) (worker
 	return f.ident, f.err
 }
 
-type fakeFirstStore struct {
-	decision firststore.Decision
-	err      error
-	regions  []string
-	ids      []firststore.Identity
-	keys     []string
-}
-
-func (f *fakeFirstStore) Claim(_ context.Context, region string, id firststore.Identity) (firststore.Decision, error) {
-	f.regions, f.ids = append(f.regions, region), append(f.ids, id)
-	return f.decision, f.err
-}
-
-func (f *fakeFirstStore) RecordPublicKey(_ context.Context, region string, id firststore.Identity, pub string) error {
-	f.regions, f.ids, f.keys = append(f.regions, region), append(f.ids, id), append(f.keys, pub)
-	return f.err
+func (f *fakeIdentities) RecordReceiptKey(_ context.Context, _ workerident.Identity, fp string) error {
+	f.recorded = append(f.recorded, fp)
+	return f.recErr
 }
 
 func current() workerident.Identity {
-	return workerident.Identity{Region: region, InstanceID: instance, Generation: 3, ClaimID: "claim-0003", NodeID: "node-a", DataVolumeID: volume}
+	return workerident.Identity{Region: region, InstanceID: instance, Generation: 3, ClaimID: "claim-0003",
+		NodeID: "node-a", DataVolumeID: volume, ReceiptInit: workerident.ReceiptPending}
 }
 
-func fakes() (*fakeIdentities, *fakeFirstStore, Deps) {
-	ids, fs := &fakeIdentities{ident: current()}, &fakeFirstStore{decision: firststore.Decision{CreateKey: true}}
-	return ids, fs, Deps{Identities: ids, FirstStore: fs, Stores: &fakeStores{err: jetstream.ErrKeyNotFound}}
+func fakes() (*fakeIdentities, Deps) {
+	ids := &fakeIdentities{ident: current()}
+	return ids, Deps{Identities: ids, HostNodes: func() int { return 1 }}
 }
 
-type fakeStores struct {
-	rec     storelaunch.Record
-	err     error
-	regions []string
-}
-
-func (f *fakeStores) Current(_ context.Context, region string) (storelaunch.Record, error) {
-	f.regions = append(f.regions, region)
-	return f.rec, f.err
-}
-
-// The first-store identity is the authoritative record's, keyed by the session
-// STS set, never anything the request carried.
-func TestClaim_UsesTheCurrentRecordIdentity(t *testing.T) {
-	ids, fs, deps := fakes()
-	rec, err := invoke(deps, "Action=ClaimFirstStore", workerActor(instance), &call{})
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(ids.asked) != 1 || ids.asked[0] != instance {
-		t.Fatalf("authorized %v, want the session's instance", ids.asked)
-	}
-	want := firststore.Identity{InstanceID: instance, ClaimID: "claim-0003", Generation: 3}
-	if len(fs.ids) != 1 || fs.ids[0] != want || fs.regions[0] != region {
-		t.Fatalf("claimed %v in %v, want %v in %s", fs.ids, fs.regions, want, region)
-	}
-	if !strings.Contains(rec.Body.String(), "<CreateKey>true</CreateKey>") {
-		t.Fatalf("response %q does not authorize key creation", rec.Body.String())
+// The receipt key claim reports only what the current record proves: pending
+// for a new volume, the recorded fingerprint, or a refusal for neither.
+func TestClaimReceiptKey_ReportsTheRecordsInitialization(t *testing.T) {
+	for name, tc := range map[string]struct {
+		init workerident.ReceiptInit
+		fp   string
+		want []string
+		code string
+	}{
+		"pending":  {init: workerident.ReceiptPending, want: []string{"<Pending>true</Pending>", "<HostNodes>1</HostNodes>"}},
+		"recorded": {init: workerident.ReceiptRecorded, fp: fpA, want: []string{"<Pending>false</Pending>", fpA}},
+		"unknown":  {code: awserrors.ErrorInvalidState},
+	} {
+		t.Run(name, func(t *testing.T) {
+			ids, deps := fakes()
+			ids.ident.ReceiptInit, ids.ident.ReceiptFingerprint = tc.init, tc.fp
+			rec, err := invoke(deps, "Action=ClaimReceiptKey", workerActor(instance), &call{})
+			if tc.code != "" {
+				assertCode(t, err, tc.code)
+				return
+			}
+			if err != nil {
+				t.Fatal(err)
+			}
+			for _, w := range tc.want {
+				if !strings.Contains(rec.Body.String(), w) {
+					t.Fatalf("body %q lacks %q", rec.Body.String(), w)
+				}
+			}
+		})
 	}
 }
 
-func TestClaim_ReturnsTheRecordedKey(t *testing.T) {
-	_, fs, deps := fakes()
-	pub := newUserKey(t).pub
-	fs.decision = firststore.Decision{PublicKey: pub}
-	rec, err := invoke(deps, "Action=ClaimFirstStore", workerActor(instance), &call{})
-	if err != nil {
-		t.Fatal(err)
-	}
-	body := rec.Body.String()
-	if !strings.Contains(body, "<CreateKey>false</CreateKey>") || !strings.Contains(body, "<PublicKey>"+pub+"</PublicKey>") {
-		t.Fatalf("response %q does not carry exactly the recorded key", body)
+func TestRecordReceiptKey_ValidatesTheFingerprintBeforeTheStore(t *testing.T) {
+	for body, code := range map[string]string{
+		"Action=RecordReceiptKey":                                     awserrors.ErrorMissingParameter,
+		"Action=RecordReceiptKey&Fingerprint=" + fpA[:63]:             awserrors.ErrorInvalidParameterValue,
+		"Action=RecordReceiptKey&Fingerprint=" + strings.ToUpper(fpA): awserrors.ErrorInvalidParameterValue,
+	} {
+		ids, deps := fakes()
+		_, err := invoke(deps, body, workerActor(instance), &call{})
+		assertCode(t, err, code)
+		if len(ids.recorded) != 0 {
+			t.Fatalf("%s: reached the store", body)
+		}
 	}
 }
 
-// Identity fields in the request are refused outright rather than ignored, and
-// before any store is consulted.
 func TestRequestSuppliedIdentityIsRefused(t *testing.T) {
 	for _, body := range []string{
-		"Action=ClaimFirstStore&InstanceId=" + successor,
-		"Action=ClaimFirstStore&Generation=99",
-		"Action=ClaimFirstStore&ClaimId=claim-9999",
-		"Action=ClaimFirstStore&Region=us-east-1",
-		"Action=RecordWorkerPublicKey&PublicKey=" + newUserKey(t).pub + "&Generation=99",
+		"Action=ClaimReceiptKey&InstanceId=" + successor,
+		"Action=ClaimReceiptKey&Generation=99",
+		"Action=ClaimReceiptKey&Region=us-east-1",
+		"Action=RecordReceiptKey&Fingerprint=" + fpA + "&Generation=99",
 	} {
-		ids, fs, deps := fakes()
+		ids, deps := fakes()
 		_, err := invoke(deps, body, workerActor(instance), &call{})
 		assertCode(t, err, awserrors.ErrorInvalidParameterValue)
-		if len(ids.asked)+len(fs.ids) != 0 {
-			t.Fatalf("%s: consulted the stores", body)
+		if len(ids.asked)+len(ids.recorded) != 0 {
+			t.Fatalf("%s: consulted the store", body)
 		}
 	}
 }
 
 // Nothing but the worker role's own session in the system account passes, and
-// a refused caller never reaches the policy evaluator or the stores.
+// a refused caller never reaches the policy evaluator or the store.
 func TestPrincipalGate(t *testing.T) {
 	const customer = "123456789012"
 	roleIn := func(acct, role string) string { return "arn:aws:iam::" + acct + ":role/" + role }
@@ -193,42 +180,41 @@ func TestPrincipalGate(t *testing.T) {
 	}
 	for name, actor := range cases {
 		t.Run(name, func(t *testing.T) {
-			ids, fs, deps := fakes()
+			ids, deps := fakes()
 			c := &call{}
-			_, err := invoke(deps, "Action=ClaimFirstStore", actor, c)
+			_, err := invoke(deps, "Action=ClaimReceiptKey", actor, c)
 			assertCode(t, err, awserrors.ErrorAccessDenied)
-			if c.policyChecks != 0 || len(ids.asked)+len(fs.ids) != 0 {
-				t.Fatal("a refused caller reached the policy check or the stores")
+			if c.policyChecks != 0 || len(ids.asked) != 0 {
+				t.Fatal("a refused caller reached the policy check or the store")
 			}
 		})
 	}
 	t.Run("invocation account differs from the actor", func(t *testing.T) {
-		ids, fs, deps := fakes()
-		req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("Action=ClaimFirstStore"))
+		ids, deps := fakes()
+		req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader("Action=ClaimReceiptKey"))
 		err := deps.dispatch(httptest.NewRecorder(), dispatch.Invocation{
 			Request: req, AccountID: customer, Region: region,
 			Authorize: func(string, string, []string, iampolicy.ConditionKeys) error { return nil },
 			Actor:     func() (dispatch.Actor, error) { return workerActor(instance), nil },
 		})
 		assertCode(t, err, awserrors.ErrorAccessDenied)
-		if len(ids.asked)+len(fs.ids) != 0 {
-			t.Fatal("consulted the stores")
+		if len(ids.asked) != 0 {
+			t.Fatal("consulted the store")
 		}
 	})
 }
 
 func TestPolicyDenialStops(t *testing.T) {
-	ids, fs, deps := fakes()
+	ids, deps := fakes()
 	c := &call{policyErr: errors.New(awserrors.ErrorAccessDenied)}
-	_, err := invoke(deps, "Action=ClaimFirstStore", workerActor(instance), c)
+	_, err := invoke(deps, "Action=ClaimReceiptKey", workerActor(instance), c)
 	assertCode(t, err, awserrors.ErrorAccessDenied)
-	if c.policyChecks != 1 || len(ids.asked)+len(fs.ids) != 0 {
-		t.Fatal("the role policy was not the gate before the stores")
+	if c.policyChecks != 1 || len(ids.asked) != 0 {
+		t.Fatal("the role policy was not the gate before the store")
 	}
 }
 
-// An unknown, superseded or foreign-Region worker never reaches the
-// first-store decision.
+// An unknown, superseded or foreign-Region worker never records a key.
 func TestNonCurrentWorkerIsRefused(t *testing.T) {
 	for name, set := range map[string]func(*fakeIdentities){
 		"unknown instance":    func(f *fakeIdentities) { f.err = workerident.ErrUnknownInstance },
@@ -236,37 +222,62 @@ func TestNonCurrentWorkerIsRefused(t *testing.T) {
 		"another Region":      func(f *fakeIdentities) { f.ident.Region = "us-east-1" },
 	} {
 		t.Run(name, func(t *testing.T) {
-			ids, fs, deps := fakes()
+			ids, deps := fakes()
 			set(ids)
-			_, err := invoke(deps, "Action=ClaimFirstStore", workerActor(instance), &call{})
+			_, err := invoke(deps, "Action=RecordReceiptKey&Fingerprint="+fpA, workerActor(instance), &call{})
 			assertCode(t, err, awserrors.ErrorAccessDenied)
-			if len(fs.ids) != 0 {
-				t.Fatal("a non-current worker reached the first-store decision")
+			if len(ids.recorded) != 0 {
+				t.Fatal("a non-current worker recorded a key")
 			}
 		})
 	}
 }
 
-type userKey struct{ pub, seed string }
-
-func newUserKey(t *testing.T) userKey {
-	t.Helper()
-	kp, err := nkeys.CreateUser()
-	if err != nil {
-		t.Fatal(err)
+func TestStoreErrorsMapToWorkerCodes(t *testing.T) {
+	for err, code := range map[error]string{
+		workerident.ErrNotCurrent:        awserrors.ErrorAccessDenied,
+		workerident.ErrReceiptRecorded:   awserrors.ErrorInvalidParameterValue,
+		workerident.ErrReceiptUnknown:    awserrors.ErrorInvalidState,
+		workerident.ErrContended:         awserrors.ErrorServiceUnavailable,
+		errors.New("leader unreachable"): awserrors.ErrorServerInternal,
+	} {
+		ids, deps := fakes()
+		ids.recErr = err
+		_, got := invoke(deps, "Action=RecordReceiptKey&Fingerprint="+fpA, workerActor(instance), &call{})
+		assertCode(t, got, code)
 	}
-	pub, _ := kp.PublicKey()
-	seed, _ := kp.Seed()
-	return userKey{pub: pub, seed: string(seed)}
 }
 
-// corrupt always changes the last character, which the key's checksum covers.
-func corrupt(pub string) string {
-	last := "A"
-	if strings.HasSuffix(pub, "A") {
-		last = "B"
+// Without its store the surface refuses every worker, after the gate has
+// refused everyone else.
+func TestMissingDependenciesFailClosed(t *testing.T) {
+	for name, deps := range map[string]Deps{
+		"no deps":       {},
+		"nil JetStream": JetStreamDeps(nil),
+	} {
+		t.Run(name, func(t *testing.T) {
+			_, err := invoke(deps, "Action=ClaimReceiptKey", workerActor(instance), &call{})
+			assertCode(t, err, awserrors.ErrorServiceUnavailable)
+			_, err = invoke(deps, "Action=ClaimReceiptKey", dispatch.Actor{}, &call{})
+			assertCode(t, err, awserrors.ErrorAccessDenied)
+		})
+	}
+	_, deps := fakes()
+	_, err := invoke(deps, "Action=PollWork", workerActor(instance), &call{})
+	assertCode(t, err, awserrors.ErrorServiceUnavailable)
+}
+
+func TestUnknownAndMissingActions(t *testing.T) {
+	_, deps := fakes()
+	for body, code := range map[string]string{
+		"Action=SendMessage":     awserrors.ErrorInvalidAction,
+		"Action=ClaimFirstStore": awserrors.ErrorInvalidAction,
+		"Action=DescribeStore":   awserrors.ErrorInvalidAction,
+		"":                       awserrors.ErrorMissingAction,
+	} {
+		_, err := invoke(deps, body, workerActor(instance), &call{})
+		assertCode(t, err, code)
 	}
-	return pub[:len(pub)-1] + last
 }
 
 func captureLogs(t *testing.T) *bytes.Buffer {
@@ -278,106 +289,25 @@ func captureLogs(t *testing.T) *bytes.Buffer {
 	return &buf
 }
 
-// A seed, or anything else that is not a public user key, is refused without
-// being recorded, logged or echoed.
-func TestRecordPublicKey_RefusesSeedsAndNonKeys(t *testing.T) {
-	k := newUserKey(t)
-	acct, _ := nkeys.CreateAccount()
-	acctPub, _ := acct.PublicKey()
-	for name, value := range map[string]string{
-		"user seed":          k.seed,
-		"account public key": acctPub,
-		"corrupt key":        corrupt(k.pub),
-	} {
-		t.Run(name, func(t *testing.T) {
-			logs := captureLogs(t)
-			_, fs, deps := fakes()
-			rec, err := invoke(deps, "Action=RecordWorkerPublicKey&PublicKey="+value, workerActor(instance), &call{})
-			assertCode(t, err, awserrors.ErrorInvalidParameterValue)
-			if len(fs.keys) != 0 {
-				t.Fatal("recorded a value that is not a public user key")
-			}
-			if strings.Contains(rec.Body.String()+err.Error()+logs.String(), value) {
-				t.Fatal("the refused value was echoed or logged")
-			}
-		})
+func TestUnexpectedParameterNameIsNotLogged(t *testing.T) {
+	logs := captureLogs(t)
+	_, deps := fakes()
+	const secret = "SUAKSECRETPARAMETERNAME0123456789"
+	_, err := invoke(deps, "Action=ClaimReceiptKey&"+secret+"=1", workerActor(instance), &call{})
+	assertCode(t, err, awserrors.ErrorInvalidParameterValue)
+	if !strings.Contains(logs.String(), "unexpected parameter refused") {
+		t.Fatalf("refusal not logged: %q", logs.String())
 	}
-	t.Run("missing key", func(t *testing.T) {
-		_, fs, deps := fakes()
-		_, err := invoke(deps, "Action=RecordWorkerPublicKey", workerActor(instance), &call{})
-		assertCode(t, err, awserrors.ErrorMissingParameter)
-		if len(fs.keys) != 0 {
-			t.Fatal("recorded an empty key")
-		}
-	})
-	t.Run("public key is recorded under the current identity", func(t *testing.T) {
-		_, fs, deps := fakes()
-		if _, err := invoke(deps, "Action=RecordWorkerPublicKey&PublicKey="+k.pub, workerActor(instance), &call{}); err != nil {
-			t.Fatal(err)
-		}
-		want := firststore.Identity{InstanceID: instance, ClaimID: "claim-0003", Generation: 3}
-		if len(fs.keys) != 1 || fs.keys[0] != k.pub || fs.ids[0] != want {
-			t.Fatalf("recorded %v for %v", fs.keys, fs.ids)
-		}
-	})
-}
-
-func TestStoreErrorsMapToWorkerCodes(t *testing.T) {
-	for err, code := range map[error]string{
-		firststore.ErrClaimHeld:          awserrors.ErrorAccessDenied,
-		firststore.ErrStaleGeneration:    awserrors.ErrorAccessDenied,
-		firststore.ErrKeyMismatch:        awserrors.ErrorInvalidParameterValue,
-		firststore.ErrNotClaimed:         awserrors.ErrorInvalidState,
-		firststore.ErrContended:          awserrors.ErrorServiceUnavailable,
-		workerident.ErrContended:         awserrors.ErrorServiceUnavailable,
-		firststore.ErrUnreadable:         awserrors.ErrorServerInternal,
-		errors.New("leader unreachable"): awserrors.ErrorServerInternal,
-	} {
-		_, fs, deps := fakes()
-		fs.err = err
-		_, got := invoke(deps, "Action=ClaimFirstStore", workerActor(instance), &call{})
-		assertCode(t, got, code)
+	if strings.Contains(logs.String(), secret[2:12]) {
+		t.Fatal("parameter name reached the log")
 	}
 }
 
-// Without its stores the surface refuses every worker, after the gate has
-// refused everyone else.
-func TestMissingDependenciesFailClosed(t *testing.T) {
-	for name, deps := range map[string]Deps{
-		"no deps":        {},
-		"no first store": {Identities: &fakeIdentities{ident: current()}, Stores: &fakeStores{}},
-		"no identities":  {FirstStore: &fakeFirstStore{decision: firststore.Decision{CreateKey: true}}, Stores: &fakeStores{}},
-		"no stores":      {Identities: &fakeIdentities{ident: current()}, FirstStore: &fakeFirstStore{}},
-		"nil JetStream":  JetStreamDeps(nil),
-	} {
-		t.Run(name, func(t *testing.T) {
-			_, err := invoke(deps, "Action=ClaimFirstStore", workerActor(instance), &call{})
-			assertCode(t, err, awserrors.ErrorServiceUnavailable)
-			_, err = invoke(deps, "Action=ClaimFirstStore", dispatch.Actor{}, &call{})
-			assertCode(t, err, awserrors.ErrorAccessDenied)
-		})
-	}
-}
+// --- Against the real identity store on embedded JetStream. ---
 
-func TestUnknownAndMissingActions(t *testing.T) {
-	_, _, deps := fakes()
-	_, err := invoke(deps, "Action=SendMessage", workerActor(instance), &call{})
-	assertCode(t, err, awserrors.ErrorInvalidAction)
-	_, err = invoke(deps, "", workerActor(instance), &call{})
-	assertCode(t, err, awserrors.ErrorMissingAction)
-}
-
-// --- Against the real stores on embedded JetStream. ---
-
-type cluster struct {
-	js    jetstream.JetStream
-	ids   workerident.Store
-	fence firststore.Store
-	deps  Deps
-}
-
-func newCluster(t *testing.T) cluster {
-	t.Helper()
+// A worker on a newly created volume is told to create its key, records it
+// once, and its successor on the same volume is given only that fingerprint.
+func TestRealStore_PendingRecordAndReplace(t *testing.T) {
 	clustersize.DeclareForTest(t, 1)
 	_, _, js := testutil.StartTestJetStream(t)
 	ctx := context.Background()
@@ -385,283 +315,82 @@ func newCluster(t *testing.T) cluster {
 	if err != nil {
 		t.Fatal(err)
 	}
-	fence, err := firststore.Open(ctx, js)
+	deps := JetStreamDeps(js)
+	deps.HostNodes = func() int { return 1 }
+
+	first, err := ids.BeginLaunch(ctx, region, workerident.LaunchRequest{InstanceType: "t3.small",
+		ClaimID: "claim-0001", NodeID: "node-a", SizeGiB: 8})
 	if err != nil {
 		t.Fatal(err)
 	}
-	return cluster{js: js, ids: ids, fence: fence, deps: JetStreamDeps(js)}
-}
-
-func (c cluster) launch(t *testing.T, claim string, supersedes uint64, inst string) workerident.Intent {
-	t.Helper()
-	ctx := context.Background()
-	in, err := c.ids.BeginLaunch(ctx, region, workerident.LaunchRequest{InstanceType: "t3.small",
-		ClaimID: claim, Supersedes: supersedes, NodeID: "node-a", DataVolumeID: volume, DataVolumeEncrypted: true,
-	})
-	if err != nil {
+	if err := ids.MarkVolumeAttempted(ctx, first); err != nil {
 		t.Fatal(err)
 	}
-	if err := c.ids.BindInstance(ctx, in, inst); err != nil {
+	if err := ids.RecordVolume(ctx, first, volume); err != nil {
 		t.Fatal(err)
 	}
-	return in
-}
-
-// replace follows the launch path's order once old is confirmed stopped: raise
-// the first-store fence past old's generation, then begin and bind the successor.
-func (c cluster) replace(t *testing.T, old workerident.Intent, claim, inst string) workerident.Intent {
-	t.Helper()
-	if err := c.fence.Fence(context.Background(), region, old.Generation+1); err != nil {
+	if err := ids.BindInstance(ctx, first, instance); err != nil {
 		t.Fatal(err)
 	}
-	return c.launch(t, claim, old.Generation, inst)
-}
 
-// The full path: a bound worker claims, records its key, and after a confirmed
-// replacement the old instance is refused while the new one gets only the key.
-func TestRealStores_ClaimRecordAndReplace(t *testing.T) {
-	c := newCluster(t)
-	k := newUserKey(t)
-	first := c.launch(t, "claim-0001", 0, instance)
-
-	rec, err := invoke(c.deps, "Action=ClaimFirstStore", workerActor(instance), &call{})
-	if err != nil || !strings.Contains(rec.Body.String(), "<CreateKey>true</CreateKey>") {
+	rec, err := invoke(deps, "Action=ClaimReceiptKey", workerActor(instance), &call{})
+	if err != nil || !strings.Contains(rec.Body.String(), "<Pending>true</Pending>") {
 		t.Fatalf("first claim: %v %q", err, rec.Body.String())
 	}
-	if _, err := invoke(c.deps, "Action=RecordWorkerPublicKey&PublicKey="+k.pub, workerActor(instance), &call{}); err != nil {
+	if _, err := invoke(deps, "Action=RecordReceiptKey&Fingerprint="+fpA, workerActor(instance), &call{}); err != nil {
 		t.Fatal(err)
 	}
-	// A lost reply is retried with the same key.
-	if _, err := invoke(c.deps, "Action=RecordWorkerPublicKey&PublicKey="+k.pub, workerActor(instance), &call{}); err != nil {
+	// A lost reply is retried with the same fingerprint; any other is refused.
+	if _, err := invoke(deps, "Action=RecordReceiptKey&Fingerprint="+fpA, workerActor(instance), &call{}); err != nil {
 		t.Fatalf("retried record: %v", err)
 	}
-	other := newUserKey(t)
-	_, err = invoke(c.deps, "Action=RecordWorkerPublicKey&PublicKey="+other.pub, workerActor(instance), &call{})
+	_, err = invoke(deps, "Action=RecordReceiptKey&Fingerprint="+fpB, workerActor(instance), &call{})
 	assertCode(t, err, awserrors.ErrorInvalidParameterValue)
 
-	// An instance that was never bound is refused.
-	_, err = invoke(c.deps, "Action=ClaimFirstStore", workerActor(successor), &call{})
+	_, err = invoke(deps, "Action=ClaimReceiptKey", workerActor(successor), &call{})
 	assertCode(t, err, awserrors.ErrorAccessDenied)
 
-	if err := c.ids.RecordStopped(context.Background(), first, instance); err != nil {
+	if err := ids.RecordStopped(ctx, first, instance); err != nil {
 		t.Fatal(err)
 	}
-	// Stopped but not yet replaced: the old instance is no longer current.
-	_, err = invoke(c.deps, "Action=ClaimFirstStore", workerActor(instance), &call{})
-	assertCode(t, err, awserrors.ErrorAccessDenied)
-
-	c.replace(t, first, "claim-0002", successor)
-	_, err = invoke(c.deps, "Action=ClaimFirstStore", workerActor(instance), &call{})
-	assertCode(t, err, awserrors.ErrorAccessDenied)
-	rec, err = invoke(c.deps, "Action=ClaimFirstStore", workerActor(successor), &call{})
+	next, err := ids.BeginLaunch(ctx, region, workerident.LaunchRequest{InstanceType: "t3.small",
+		ClaimID: "claim-0002", Supersedes: first.Generation, NodeID: "node-a",
+		DataVolumeID: volume, DataVolumeEncrypted: true, SizeGiB: 8})
 	if err != nil {
 		t.Fatal(err)
 	}
-	body := rec.Body.String()
-	if !strings.Contains(body, "<CreateKey>false</CreateKey>") || !strings.Contains(body, k.pub) {
-		t.Fatalf("successor got %q, want only the recorded key", body)
+	if err := ids.BindInstance(ctx, next, successor); err != nil {
+		t.Fatal(err)
 	}
-}
-
-// The race between authorization and the first-store call: a request that
-// passed Authorize just before a replacement still carries the old generation,
-// and once the successor has claimed, firststore refuses it.
-func TestRealStores_InFlightOldGenerationLosesToTheSuccessor(t *testing.T) {
-	c := newCluster(t)
-	first := c.launch(t, "claim-0001", 0, instance)
-	stale, err := c.ids.Authorize(context.Background(), instance)
+	_, err = invoke(deps, "Action=ClaimReceiptKey", workerActor(instance), &call{})
+	assertCode(t, err, awserrors.ErrorAccessDenied)
+	rec, err = invoke(deps, "Action=ClaimReceiptKey", workerActor(successor), &call{})
 	if err != nil {
 		t.Fatal(err)
 	}
-	if err := c.ids.RecordStopped(context.Background(), first, instance); err != nil {
-		t.Fatal(err)
-	}
-	c.replace(t, first, "claim-0002", successor)
-	if _, err := invoke(c.deps, "Action=ClaimFirstStore", workerActor(successor), &call{}); err != nil {
-		t.Fatal(err)
-	}
-
-	inFlight := Deps{Identities: &fakeIdentities{ident: stale}, FirstStore: c.deps.FirstStore, Stores: c.deps.Stores}
-	_, err = invoke(inFlight, "Action=ClaimFirstStore", workerActor(instance), &call{})
-	assertCode(t, err, awserrors.ErrorAccessDenied)
-	_, err = invoke(inFlight, "Action=RecordWorkerPublicKey&PublicKey="+newUserKey(t).pub, workerActor(instance), &call{})
-	assertCode(t, err, awserrors.ErrorAccessDenied)
-}
-
-// Concurrent claims from the one current worker decide once.
-func TestRealStores_ConcurrentClaimsAgree(t *testing.T) {
-	c := newCluster(t)
-	c.launch(t, "claim-0001", 0, instance)
-	var wg sync.WaitGroup
-	bodies := make([]string, 8)
-	errs := make([]error, 8)
-	for i := range bodies {
-		wg.Go(func() {
-			rec, err := invoke(c.deps, "Action=ClaimFirstStore", workerActor(instance), &call{})
-			bodies[i], errs[i] = rec.Body.String(), err
-		})
-	}
-	wg.Wait()
-	for i := range bodies {
-		if errs[i] != nil && errs[i].Error() != awserrors.ErrorServiceUnavailable {
-			t.Fatalf("claim %d: %v", i, errs[i])
-		}
-		if errs[i] == nil && !strings.Contains(bodies[i], "<CreateKey>true</CreateKey>") {
-			t.Fatalf("claim %d: %q", i, bodies[i])
-		}
+	if body := rec.Body.String(); !strings.Contains(body, "<Pending>false</Pending>") || !strings.Contains(body, fpA) {
+		t.Fatalf("successor got %q, want only the recorded fingerprint", body)
 	}
 }
 
-// Absent buckets refuse the worker and stay absent: the gateway provisions
-// nothing.
-func TestRealStores_AbsentBucketsFailClosedAndStayAbsent(t *testing.T) {
+// A record created on a supplied volume is never pending: its volume is not
+// proven new, so no key is created for it.
+func TestRealStore_SuppliedVolumeIsNotPending(t *testing.T) {
 	clustersize.DeclareForTest(t, 1)
 	_, _, js := testutil.StartTestJetStream(t)
-	deps := JetStreamDeps(js)
-	_, err := invoke(deps, "Action=ClaimFirstStore", workerActor(instance), &call{})
-	assertCode(t, err, awserrors.ErrorServiceUnavailable)
-	for _, name := range []string{workerident.Bucket, firststore.Bucket} {
-		if _, err := js.KeyValue(context.Background(), name); !errors.Is(err, jetstream.ErrBucketNotFound) {
-			t.Fatalf("the gateway provisioned %s: %v", name, err)
-		}
-	}
-}
-
-// pausingIdentities runs the real Authorize, then runs between before the
-// request reaches firststore, standing in for a request delayed in flight.
-type pausingIdentities struct {
-	real    Identities
-	between func()
-}
-
-func (p pausingIdentities) Authorize(ctx context.Context, instanceID string) (workerident.Identity, error) {
-	id, err := p.real.Authorize(ctx, instanceID)
-	if err == nil {
-		p.between()
-	}
-	return id, err
-}
-
-// The other order: the old request passes Authorize, the old worker is stopped
-// and its successor bound, and the old request reaches firststore before the
-// successor has claimed. The fence refuses it, so the successor still decides.
-func TestRealStores_InFlightOldGenerationCannotPreemptTheSuccessor(t *testing.T) {
-	for _, action := range []string{"Action=ClaimFirstStore", "Action=RecordWorkerPublicKey&PublicKey="} {
-		t.Run(strings.TrimSuffix(strings.TrimPrefix(action, "Action="), "&PublicKey="), func(t *testing.T) {
-			c := newCluster(t)
-			oldKey := newUserKey(t)
-			first := c.launch(t, "claim-0001", 0, instance)
-			if strings.Contains(action, "Record") {
-				// The old worker has claimed, so only its key record is in flight.
-				if _, err := invoke(c.deps, "Action=ClaimFirstStore", workerActor(instance), &call{}); err != nil {
-					t.Fatal(err)
-				}
-				action += oldKey.pub
-			}
-			replaced := false
-			inFlight := Deps{
-				Identities: pausingIdentities{real: c.deps.Identities, between: func() {
-					if err := c.ids.RecordStopped(context.Background(), first, instance); err != nil {
-						t.Fatal(err)
-					}
-					c.replace(t, first, "claim-0002", successor)
-					replaced = true
-				}},
-				FirstStore: c.deps.FirstStore,
-				Stores:     c.deps.Stores,
-			}
-			_, err := invoke(inFlight, action, workerActor(instance), &call{})
-			if !replaced {
-				t.Fatal("the replacement did not run inside the request")
-			}
-			assertCode(t, err, awserrors.ErrorAccessDenied)
-
-			rec, err := invoke(c.deps, "Action=ClaimFirstStore", workerActor(successor), &call{})
-			if err != nil {
-				t.Fatal(err)
-			}
-			body := rec.Body.String()
-			if !strings.Contains(body, "<CreateKey>true</CreateKey>") || strings.Contains(body, oldKey.pub) {
-				t.Fatalf("successor got %q, want to create its own key", body)
-			}
-		})
-	}
-}
-
-// A caller-chosen parameter name never reaches the log.
-func TestUnexpectedParameterNameIsNotLogged(t *testing.T) {
-	logs := captureLogs(t)
-	_, _, deps := fakes()
-	secret := newUserKey(t).seed
-	_, err := invoke(deps, "Action=ClaimFirstStore&"+secret+"=1", workerActor(instance), &call{})
-	assertCode(t, err, awserrors.ErrorInvalidParameterValue)
-	if !strings.Contains(logs.String(), "unexpected parameter refused") {
-		t.Fatalf("refusal not logged: %q", logs.String())
-	}
-	if strings.Contains(logs.String(), secret) || strings.Contains(logs.String(), secret[2:12]) {
-		t.Fatal("parameter name reached the log")
-	}
-}
-
-// DescribeStore reports only the current worker's own Region, says pending
-// until the store is bound, and never echoes anything but the public key.
-func TestDescribeStore(t *testing.T) {
-	k := newUserKey(t)
-	bound := storelaunch.Record{Region: region, State: storelaunch.StateBound, PublicKey: k.pub,
-		InstanceID: successor, Address: "10.240.0.9"}
-	for _, tc := range []struct {
-		name    string
-		stores  *fakeStores
-		want    []string
-		wantErr string
-	}{
-		{"no record", &fakeStores{err: jetstream.ErrKeyNotFound}, []string{"<State>pending</State>"}, ""},
-		{"launching", &fakeStores{rec: storelaunch.Record{Region: region, State: storelaunch.StateLaunching}},
-			[]string{"<State>pending</State>"}, ""},
-		{"bound", &fakeStores{rec: bound},
-			[]string{"<State>bound</State>", "<Address>10.240.0.9</Address>", "<PublicKey>" + k.pub + "</PublicKey>"}, ""},
-		{"leader unavailable", &fakeStores{err: errors.New("leader lost")}, nil, awserrors.ErrorServiceUnavailable},
-		{"corrupt record", &fakeStores{err: storelaunch.ErrUnreadable}, nil, awserrors.ErrorServerInternal},
-	} {
-		t.Run(tc.name, func(t *testing.T) {
-			ids, _, deps := fakes()
-			deps.Stores = tc.stores
-			rec, err := invoke(deps, "Action=DescribeStore", workerActor(instance), &call{})
-			if tc.wantErr != "" {
-				assertCode(t, err, tc.wantErr)
-				return
-			}
-			if err != nil {
-				t.Fatal(err)
-			}
-			for _, w := range tc.want {
-				if !strings.Contains(rec.Body.String(), w) {
-					t.Fatalf("body %q lacks %q", rec.Body.String(), w)
-				}
-			}
-			if tc.name != "bound" && strings.Contains(rec.Body.String(), "<Address>") {
-				t.Fatalf("an unbound store reported an address: %q", rec.Body.String())
-			}
-			if len(tc.stores.regions) != 1 || tc.stores.regions[0] != ids.ident.Region {
-				t.Fatalf("read Regions %v, want only the bound worker's", tc.stores.regions)
-			}
-		})
-	}
-}
-
-// A caller that is not the current worker never reads the store record.
-func TestDescribeStore_RefusesNonCurrentWorkers(t *testing.T) {
-	ids, _, deps := fakes()
-	ids.err = workerident.ErrNotCurrent
-	stores := &fakeStores{}
-	deps.Stores = stores
-	_, err := invoke(deps, "Action=DescribeStore", workerActor(instance), &call{})
-	assertCode(t, err, awserrors.ErrorAccessDenied)
-	_, err = invoke(deps, "Action=DescribeStore&Region=ap-southeast-2", workerActor(instance), &call{})
-	if err == nil {
-		t.Fatal("an identity parameter was accepted")
-	}
-	if len(stores.regions) != 0 {
-		t.Fatalf("store record read for a refused caller: %v", stores.regions)
+	ctx := context.Background()
+	ids, err := workerident.Open(ctx, js)
+	if err != nil {
+		t.Fatal(err)
 	}
+	in, err := ids.BeginLaunch(ctx, region, workerident.LaunchRequest{InstanceType: "t3.small",
+		ClaimID: "claim-0001", NodeID: "node-a", DataVolumeID: volume, DataVolumeEncrypted: true, SizeGiB: 8})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := ids.BindInstance(ctx, in, instance); err != nil {
+		t.Fatal(err)
+	}
+	_, err = invoke(JetStreamDeps(js), "Action=ClaimReceiptKey", workerActor(instance), &call{})
+	assertCode(t, err, awserrors.ErrorInvalidState)
 }

spinifex/domains/sqs/firststore/firststore.go

@@ -1,390 +0,0 @@
-// Package firststore is the SQS domain's durable decision on whether the shared
-// SQS worker may create the store's client key. One record per Region moves
-// claimed -> key-recorded -> launch-intent under compare-and-swap, behind a
-// generation fence the launch path raises before it replaces a worker.
-package firststore
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"regexp"
-
-	sqsjs "github.com/mulgadc/spinifex/spinifex/domains/sqs/message/jetstream"
-	"github.com/nats-io/nats.go/jetstream"
-	"github.com/nats-io/nkeys"
-)
-
-// Bucket is the SQS domain's KV bucket for first-store records.
-const Bucket = "spinifex-sqs-first-store"
-
-// BucketConfig is the bucket's configuration for test fixtures; production
-// opens it through Open, which takes the replica count from the cluster.
-func BucketConfig(replicas int) jetstream.KeyValueConfig {
-	return jetstream.KeyValueConfig{
-		Bucket:   Bucket,
-		History:  1,
-		Storage:  jetstream.FileStorage,
-		Replicas: replicas,
-	}
-}
-
-type State string
-
-const (
-	// Only a fence is recorded: no worker has claimed since it was raised.
-	StateOpen State = "open"
-	// A worker identity holds the claim; no key is recorded, so no store can
-	// have been launched with any key.
-	StateClaimed State = "claimed"
-	// The worker's public key is recorded; from here a missing or different
-	// worker key fails closed.
-	StateKeyRecorded State = "key-recorded"
-	// A store launch may use the recorded key.
-	StateLaunchIntent State = "launch-intent"
-)
-
-// Identity is one worker VM incarnation. Generation increases each time the
-// worker is relaunched. A caller must derive it from authoritative instance
-// state, never from the worker's request: a newer generation may take over.
-type Identity struct {
-	InstanceID string `json:"instanceID"`
-	ClaimID    string `json:"claimID"`
-	Generation uint64 `json:"generation"`
-}
-
-// Record is the stored decision. It never holds private key material.
-type Record struct {
-	Region    string   `json:"region"`
-	State     State    `json:"state"`
-	Worker    Identity `json:"worker"`
-	PublicKey string   `json:"publicKey,omitempty"`
-	// Fence is the lowest generation still accepted. It only rises, so a
-	// request from a replaced worker that was already in flight is refused.
-	Fence uint64 `json:"fence,omitempty"`
-}
-
-// Decision is what a claim authorizes: create a key, or use exactly the
-// recorded one.
-type Decision struct {
-	CreateKey bool
-	PublicKey string
-}
-
-var (
-	ErrClaimHeld       = errors.New("first-store claim is held by another worker identity")
-	ErrStaleGeneration = errors.New("worker generation is older than the recorded claim")
-	ErrNotClaimed      = errors.New("no first-store claim is recorded")
-	ErrKeyNotRecorded  = errors.New("worker public key is not recorded")
-	ErrKeyMismatch     = errors.New("worker public key does not match the recorded key")
-	ErrUnreadable      = errors.New("first-store record is unreadable")
-	ErrContended       = errors.New("first-store record changed concurrently; retry")
-)
-
-const maxAttempts = 8
-
-var (
-	regionPattern   = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`)
-	instancePattern = regexp.MustCompile(`^i-[0-9a-f]{8,17}$`)
-	claimPattern    = regexp.MustCompile(`^[A-Za-z0-9-]{8,64}$`)
-)
-
-// Store is the first-store state machine over the domain's KV bucket. Reads
-// go to the stream leader, and every authorizing answer is also confirmed by a
-// compare-and-swap at the revision read, so a stale read authorizes nothing.
-type Store struct {
-	JS jetstream.JetStream
-	KV jetstream.KeyValue
-}
-
-// Open opens or creates the bucket and refuses one that could lose records.
-func Open(ctx context.Context, js jetstream.JetStream) (Store, error) {
-	kv, err := sqsjs.OpenDurableKV(ctx, js, Bucket, "SQS first-store decisions: who may create the store key per Region")
-	if err != nil {
-		return Store{}, err
-	}
-	return Store{JS: js, KV: kv}, nil
-}
-
-// Bind binds the existing bucket without creating it, refusing one that could
-// lose records; a reader that must not provision state uses it.
-func Bind(ctx context.Context, js jetstream.JetStream) (Store, error) {
-	kv, err := sqsjs.BindDurableKV(ctx, js, Bucket)
-	if err != nil {
-		return Store{}, err
-	}
-	return Store{JS: js, KV: kv}, nil
-}
-
-func key(region string) string { return "region." + region }
-
-func validate(region string, id Identity) error {
-	if !regionPattern.MatchString(region) {
-		return errors.New("invalid region")
-	}
-	if !instancePattern.MatchString(id.InstanceID) {
-		return errors.New("invalid worker instance ID")
-	}
-	if !claimPattern.MatchString(id.ClaimID) {
-		return errors.New("invalid claim ID")
-	}
-	if id.Generation == 0 {
-		return errors.New("worker generation must be positive")
-	}
-	return nil
-}
-
-// Claim decides for this worker identity. Absent a record it creates a claim
-// and authorizes key creation. A claim with no recorded key may pass to a newer
-// generation; once a key is recorded only that key is ever authorized.
-func (s Store) Claim(ctx context.Context, region string, id Identity) (Decision, error) {
-	if err := validate(region, id); err != nil {
-		return Decision{}, err
-	}
-	for range maxAttempts {
-		rec, rev, err := s.read(ctx, region)
-		if errors.Is(err, ErrNotClaimed) {
-			// A failed create may still have applied; the re-read decides.
-			if s.create(ctx, region, Record{Region: region, State: StateClaimed, Worker: id}) == nil {
-				return Decision{CreateKey: true}, nil
-			}
-			continue
-		}
-		if err != nil {
-			return Decision{}, err
-		}
-		if id.Generation < rec.Fence {
-			return Decision{}, ErrStaleGeneration
-		}
-		if rec.State == StateOpen {
-			next := *rec
-			next.State, next.Worker = StateClaimed, id
-			if s.update(ctx, region, next, rev) == nil {
-				return Decision{CreateKey: true}, nil
-			}
-			continue
-		}
-		if err := checkGeneration(rec.Worker, id); err != nil {
-			return Decision{}, err
-		}
-		// Rebinds a strictly newer generation, keeping state and key; for the
-		// same identity it rewrites the record to prove the read was current.
-		next := *rec
-		next.Worker = id
-		if s.update(ctx, region, next, rev) != nil {
-			continue
-		}
-		if rec.State == StateClaimed {
-			return Decision{CreateKey: true}, nil
-		}
-		return Decision{PublicKey: rec.PublicKey}, nil
-	}
-	return Decision{}, ErrContended
-}
-
-// RecordPublicKey binds the worker's public key to its current claim. It is
-// idempotent for the same key, so a lost reply can be retried.
-func (s Store) RecordPublicKey(ctx context.Context, region string, id Identity, pub string) error {
-	return s.advance(ctx, region, id, pub, func(rec *Record) (*Record, error) {
-		switch rec.State {
-		case StateClaimed:
-			next := *rec
-			next.State, next.PublicKey = StateKeyRecorded, pub
-			return &next, nil
-		default:
-			return nil, nil
-		}
-	})
-}
-
-// RecordLaunchIntent permits a store launch with the recorded key. It refuses
-// until the worker's public key is recorded.
-func (s Store) RecordLaunchIntent(ctx context.Context, region string, id Identity, pub string) error {
-	return s.advance(ctx, region, id, pub, func(rec *Record) (*Record, error) {
-		switch rec.State {
-		case StateClaimed:
-			return nil, ErrKeyNotRecorded
-		case StateKeyRecorded:
-			next := *rec
-			next.State = StateLaunchIntent
-			return &next, nil
-		default:
-			return nil, nil
-		}
-	})
-}
-
-// RecordedKey is the Region's recorded worker public key and the identity
-// that recorded it, read from the stream leader. It authorizes nothing by
-// itself: RecordLaunchIntent confirms the same key under compare-and-swap.
-func (s Store) RecordedKey(ctx context.Context, region string) (Identity, string, error) {
-	if !regionPattern.MatchString(region) {
-		return Identity{}, "", errors.New("invalid region")
-	}
-	rec, _, err := s.read(ctx, region)
-	if err != nil {
-		return Identity{}, "", err
-	}
-	if rec.State != StateKeyRecorded && rec.State != StateLaunchIntent {
-		return Identity{}, "", ErrKeyNotRecorded
-	}
-	return rec.Worker, rec.PublicKey, nil
-}
-
-// advance applies step to the current record of the exact claiming identity.
-// A nil next record means the transition has already happened.
-func (s Store) advance(ctx context.Context, region string, id Identity, pub string, step func(*Record) (*Record, error)) error {
-	if err := validate(region, id); err != nil {
-		return err
-	}
-	if !nkeys.IsValidPublicUserKey(pub) {
-		return errors.New("worker key is not a user public NKey")
-	}
-	for range maxAttempts {
-		rec, rev, err := s.read(ctx, region)
-		if err != nil {
-			return err
-		}
-		if id.Generation < rec.Fence {
-			return ErrStaleGeneration
-		}
-		if rec.State == StateOpen {
-			return ErrNotClaimed
-		}
-		if rec.Worker != id {
-			if err := checkGeneration(rec.Worker, id); err != nil {
-				return err
-			}
-			// A newer generation must claim before it records.
-			return ErrClaimHeld
-		}
-		if rec.State != StateClaimed && rec.PublicKey != pub {
-			return ErrKeyMismatch
-		}
-		next, err := step(rec)
-		if err != nil {
-			return err
-		}
-		if next == nil {
-			// Already applied: rewrite unchanged to prove the read was current.
-			next = rec
-		}
-		if s.update(ctx, region, *next, rev) == nil {
-			return nil
-		}
-	}
-	return ErrContended
-}
-
-// Fence raises the lowest accepted generation to gen and never lowers it. The
-// launch path calls it before it replaces a worker, so a request from the old
-// generation that is still in flight decides nothing once the fence is up.
-func (s Store) Fence(ctx context.Context, region string, gen uint64) error {
-	if !regionPattern.MatchString(region) || gen == 0 {
-		return errors.New("invalid fence")
-	}
-	for range maxAttempts {
-		rec, rev, err := s.read(ctx, region)
-		if errors.Is(err, ErrNotClaimed) {
-			if s.create(ctx, region, Record{Region: region, State: StateOpen, Fence: gen}) == nil {
-				return nil
-			}
-			continue
-		}
-		if err != nil {
-			return err
-		}
-		next := *rec
-		next.Fence = max(rec.Fence, gen)
-		// Unchanged when already raised: the rewrite proves the read was current.
-		if s.update(ctx, region, next, rev) == nil {
-			return nil
-		}
-	}
-	return ErrContended
-}
-
-// FencedAt is the Region's current fence, zero when none is recorded. The
-// fence only rises, so a value read from the leader stays a lower bound.
-func (s Store) FencedAt(ctx context.Context, region string) (uint64, error) {
-	rec, _, err := s.read(ctx, region)
-	if errors.Is(err, ErrNotClaimed) {
-		return 0, nil
-	}
-	if err != nil {
-		return 0, err
-	}
-	return rec.Fence, nil
-}
-
-// checkGeneration refuses an older generation, and a different identity at
-// the same generation; only a strictly newer generation may supersede.
-func checkGeneration(held, id Identity) error {
-	switch {
-	case held == id:
-		return nil
-	case id.Generation < held.Generation:
-		return ErrStaleGeneration
-	case id.Generation == held.Generation:
-		return ErrClaimHeld
-	default:
-		return nil
-	}
-}
-
-func (s Store) read(ctx context.Context, region string) (*Record, uint64, error) {
-	value, rev, err := sqsjs.LeaderKVGet(ctx, s.JS, Bucket, key(region))
-	if errors.Is(err, jetstream.ErrKeyNotFound) {
-		return nil, 0, ErrNotClaimed
-	}
-	if err != nil {
-		return nil, 0, fmt.Errorf("read first-store record: %w", err)
-	}
-	var rec Record
-	if err := json.Unmarshal(value, &rec); err != nil {
-		return nil, 0, ErrUnreadable
-	}
-	if !wellFormed(&rec, region) {
-		return nil, 0, ErrUnreadable
-	}
-	return &rec, rev, nil
-}
-
-func wellFormed(rec *Record, region string) bool {
-	if rec.Region != region {
-		return false
-	}
-	if rec.State == StateOpen {
-		return rec.Worker == Identity{} && rec.PublicKey == "" && rec.Fence > 0
-	}
-	if validate(region, rec.Worker) != nil {
-		return false
-	}
-	switch rec.State {
-	case StateClaimed:
-		return rec.PublicKey == ""
-	case StateKeyRecorded, StateLaunchIntent:
-		return nkeys.IsValidPublicUserKey(rec.PublicKey)
-	default:
-		return false
-	}
-}
-
-func (s Store) create(ctx context.Context, region string, rec Record) error {
-	b, err := json.Marshal(rec)
-	if err != nil {
-		return err
-	}
-	_, err = s.KV.Create(ctx, key(region), b)
-	return err
-}
-
-func (s Store) update(ctx context.Context, region string, rec Record, rev uint64) error {
-	b, err := json.Marshal(rec)
-	if err != nil {
-		return err
-	}
-	_, err = s.KV.Update(ctx, key(region), b, rev)
-	return err
-}

spinifex/domains/sqs/firststore/firststore_test.go

@@ -1,602 +0,0 @@
-package firststore
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"testing"
-
-	testutil "github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/nats-io/nats.go"
-	"github.com/nats-io/nats.go/jetstream"
-	"github.com/nats-io/nkeys"
-)
-
-const region = "ap-southeast-2"
-
-func newStore(t *testing.T) (Store, jetstream.KeyValue) {
-	t.Helper()
-	_, _, js := testutil.StartTestJetStream(t)
-	kv, err := js.CreateKeyValue(context.Background(), BucketConfig(1))
-	if err != nil {
-		t.Fatal(err)
-	}
-	return Store{JS: js, KV: kv}, kv
-}
-
-func ident(n int, gen uint64) Identity {
-	return Identity{InstanceID: fmt.Sprintf("i-%017x", n), ClaimID: fmt.Sprintf("claim-%04d-%d", n, gen), Generation: gen}
-}
-
-func userPub(t *testing.T) (pub, seed string) {
-	t.Helper()
-	kp, err := nkeys.CreateUser()
-	if err != nil {
-		t.Fatal(err)
-	}
-	p, _ := kp.PublicKey()
-	s, _ := kp.Seed()
-	return p, string(s)
-}
-
-func stored(t *testing.T, kv jetstream.KeyValue) string {
-	t.Helper()
-	e, err := kv.Get(context.Background(), key(region))
-	if err != nil {
-		t.Fatal(err)
-	}
-	return string(e.Value())
-}
-
-func TestClaim_FirstClaimCreatesKeyThenOnlyTheRecordedKey(t *testing.T) {
-	s, _ := newStore(t)
-	ctx := context.Background()
-	w := ident(1, 1)
-	d, err := s.Claim(ctx, region, w)
-	if err != nil || !d.CreateKey {
-		t.Fatalf("first claim = %+v, %v; want CreateKey", d, err)
-	}
-	pub, _ := userPub(t)
-	if err := s.RecordPublicKey(ctx, region, w, pub); err != nil {
-		t.Fatal(err)
-	}
-	d, err = s.Claim(ctx, region, w)
-	if err != nil || d.CreateKey || d.PublicKey != pub {
-		t.Fatalf("after record = %+v, %v; want only the recorded key", d, err)
-	}
-}
-
-func TestClaim_ConcurrentClaimsHaveOneWinner(t *testing.T) {
-	s, _ := newStore(t)
-	const n = 16
-	var created, held atomic.Int32
-	var wg sync.WaitGroup
-	for i := range n {
-		wg.Go(func() {
-			d, err := s.Claim(context.Background(), region, ident(i+1, 1))
-			switch {
-			case err == nil && d.CreateKey:
-				created.Add(1)
-			case errors.Is(err, ErrClaimHeld):
-				held.Add(1)
-			default:
-				t.Errorf("claim %d = %+v, %v", i, d, err)
-			}
-		})
-	}
-	wg.Wait()
-	if created.Load() != 1 || held.Load() != n-1 {
-		t.Fatalf("created=%d held=%d, want exactly one winner", created.Load(), held.Load())
-	}
-}
-
-func TestClaim_GenerationRules(t *testing.T) {
-	ctx := context.Background()
-	t.Run("newer generation takes over a claim with no key", func(t *testing.T) {
-		s, _ := newStore(t)
-		old, next := ident(1, 1), ident(2, 2)
-		if _, err := s.Claim(ctx, region, old); err != nil {
-			t.Fatal(err)
-		}
-		d, err := s.Claim(ctx, region, next)
-		if err != nil || !d.CreateKey {
-			t.Fatalf("takeover = %+v, %v", d, err)
-		}
-		pub, _ := userPub(t)
-		if err := s.RecordPublicKey(ctx, region, old, pub); !errors.Is(err, ErrStaleGeneration) {
-			t.Fatalf("superseded record = %v, want ErrStaleGeneration", err)
-		}
-		if _, err := s.Claim(ctx, region, old); !errors.Is(err, ErrStaleGeneration) {
-			t.Fatalf("superseded claim = %v, want ErrStaleGeneration", err)
-		}
-	})
-	t.Run("newer generation after recording gets only the recorded key", func(t *testing.T) {
-		s, _ := newStore(t)
-		old, next := ident(1, 1), ident(2, 2)
-		s.Claim(ctx, region, old)
-		pub, _ := userPub(t)
-		if err := s.RecordPublicKey(ctx, region, old, pub); err != nil {
-			t.Fatal(err)
-		}
-		d, err := s.Claim(ctx, region, next)
-		if err != nil || d.CreateKey || d.PublicKey != pub {
-			t.Fatalf("relaunched worker = %+v, %v; want the recorded key", d, err)
-		}
-		other, _ := userPub(t)
-		if err := s.RecordPublicKey(ctx, region, next, other); !errors.Is(err, ErrKeyMismatch) {
-			t.Fatalf("new key after recording = %v, want ErrKeyMismatch", err)
-		}
-		if _, err := s.Claim(ctx, region, old); !errors.Is(err, ErrStaleGeneration) {
-			t.Fatalf("replayed old boot = %v, want ErrStaleGeneration", err)
-		}
-	})
-	t.Run("different identity at the same generation is refused", func(t *testing.T) {
-		s, _ := newStore(t)
-		s.Claim(ctx, region, ident(1, 3))
-		rival := ident(2, 3)
-		if _, err := s.Claim(ctx, region, rival); !errors.Is(err, ErrClaimHeld) {
-			t.Fatalf("rival claim = %v, want ErrClaimHeld", err)
-		}
-		pub, _ := userPub(t)
-		if err := s.RecordPublicKey(ctx, region, rival, pub); !errors.Is(err, ErrClaimHeld) {
-			t.Fatalf("rival record = %v, want ErrClaimHeld", err)
-		}
-		sameInstance := ident(1, 3)
-		sameInstance.ClaimID = "claim-replayed-x"
-		if _, err := s.Claim(ctx, region, sameInstance); !errors.Is(err, ErrClaimHeld) {
-			t.Fatalf("replayed claim ID = %v, want ErrClaimHeld", err)
-		}
-	})
-	t.Run("newer generation must claim before recording", func(t *testing.T) {
-		s, _ := newStore(t)
-		s.Claim(ctx, region, ident(1, 1))
-		pub, _ := userPub(t)
-		if err := s.RecordPublicKey(ctx, region, ident(2, 2), pub); !errors.Is(err, ErrClaimHeld) {
-			t.Fatalf("unclaimed newer record = %v, want ErrClaimHeld", err)
-		}
-	})
-}
-
-func TestRecordPublicKey_IdempotentAndFailClosed(t *testing.T) {
-	ctx := context.Background()
-	s, kv := newStore(t)
-	w := ident(1, 1)
-	pub, seed := userPub(t)
-	if err := s.RecordPublicKey(ctx, region, w, pub); !errors.Is(err, ErrNotClaimed) {
-		t.Fatalf("record without claim = %v, want ErrNotClaimed", err)
-	}
-	s.Claim(ctx, region, w)
-	err := s.RecordPublicKey(ctx, region, w, seed)
-	if err == nil || strings.Contains(err.Error(), seed) {
-		t.Fatalf("seed as public key = %v; want a refusal without the seed", err)
-	}
-	if err := s.RecordPublicKey(ctx, region, w, pub); err != nil {
-		t.Fatal(err)
-	}
-	if err := s.RecordPublicKey(ctx, region, w, pub); err != nil {
-		t.Fatalf("retried record = %v, want idempotent success", err)
-	}
-	other, _ := userPub(t)
-	if err := s.RecordPublicKey(ctx, region, w, other); !errors.Is(err, ErrKeyMismatch) {
-		t.Fatalf("second key = %v, want ErrKeyMismatch", err)
-	}
-	if strings.Contains(stored(t, kv), seed) {
-		t.Fatal("the record holds a seed")
-	}
-}
-
-func TestRecordLaunchIntent_NeedsRecordedKey(t *testing.T) {
-	ctx := context.Background()
-	s, _ := newStore(t)
-	w := ident(1, 1)
-	pub, _ := userPub(t)
-	if err := s.RecordLaunchIntent(ctx, region, w, pub); !errors.Is(err, ErrNotClaimed) {
-		t.Fatalf("launch without claim = %v", err)
-	}
-	s.Claim(ctx, region, w)
-	if err := s.RecordLaunchIntent(ctx, region, w, pub); !errors.Is(err, ErrKeyNotRecorded) {
-		t.Fatalf("launch before key = %v, want ErrKeyNotRecorded", err)
-	}
-	s.RecordPublicKey(ctx, region, w, pub)
-	other, _ := userPub(t)
-	if err := s.RecordLaunchIntent(ctx, region, w, other); !errors.Is(err, ErrKeyMismatch) {
-		t.Fatalf("launch with another key = %v, want ErrKeyMismatch", err)
-	}
-	for range 2 {
-		if err := s.RecordLaunchIntent(ctx, region, w, pub); err != nil {
-			t.Fatalf("launch intent = %v", err)
-		}
-	}
-	d, err := s.Claim(ctx, region, w)
-	if err != nil || d.CreateKey || d.PublicKey != pub {
-		t.Fatalf("claim after launch intent = %+v, %v", d, err)
-	}
-	if err := s.RecordPublicKey(ctx, region, w, other); !errors.Is(err, ErrKeyMismatch) {
-		t.Fatalf("new key after launch intent = %v, want ErrKeyMismatch", err)
-	}
-}
-
-func TestClaim_UnreadableRecordAuthorizesNothing(t *testing.T) {
-	ctx := context.Background()
-	pub, _ := userPub(t)
-	for name, raw := range map[string]string{
-		"not json":           "{",
-		"unknown state":      `{"region":"ap-southeast-2","state":"ready","worker":{"instanceID":"i-00000000000000001","claimID":"claim-0001-1","generation":1}}`,
-		"keyed without key":  `{"region":"ap-southeast-2","state":"key-recorded","worker":{"instanceID":"i-00000000000000001","claimID":"claim-0001-1","generation":1}}`,
-		"claimed with a key": `{"region":"ap-southeast-2","state":"claimed","worker":{"instanceID":"i-00000000000000001","claimID":"claim-0001-1","generation":1},"publicKey":"` + pub + `"}`,
-		"other region":       `{"region":"us-east-1","state":"claimed","worker":{"instanceID":"i-00000000000000001","claimID":"claim-0001-1","generation":1}}`,
-	} {
-		t.Run(name, func(t *testing.T) {
-			s, kv := newStore(t)
-			if _, err := kv.Put(ctx, key(region), []byte(raw)); err != nil {
-				t.Fatal(err)
-			}
-			d, err := s.Claim(ctx, region, ident(1, 1))
-			if !errors.Is(err, ErrUnreadable) || d.CreateKey {
-				t.Fatalf("claim = %+v, %v; want ErrUnreadable", d, err)
-			}
-			if err := s.RecordPublicKey(ctx, region, ident(1, 1), pub); !errors.Is(err, ErrUnreadable) {
-				t.Fatalf("record = %v, want ErrUnreadable", err)
-			}
-			if stored(t, kv) != raw {
-				t.Fatal("an unreadable record was overwritten")
-			}
-		})
-	}
-}
-
-// lossyKV applies or drops a write and then reports failure, as a write whose
-// reply was lost or that never reached the server.
-type lossyKV struct {
-	jetstream.KeyValue
-
-	failCreate, failUpdate int
-	apply                  bool
-	beforeCreate           func()
-}
-
-func (k *lossyKV) Create(ctx context.Context, key string, value []byte, opts ...jetstream.KVCreateOpt) (uint64, error) {
-	if k.beforeCreate != nil {
-		k.beforeCreate()
-		k.beforeCreate = nil
-	}
-	if k.failCreate > 0 {
-		k.failCreate--
-		if k.apply {
-			k.KeyValue.Create(ctx, key, value, opts...)
-		}
-		return 0, errors.New("nats: timeout")
-	}
-	return k.KeyValue.Create(ctx, key, value, opts...)
-}
-
-func (k *lossyKV) Update(ctx context.Context, key string, value []byte, rev uint64) (uint64, error) {
-	if k.failUpdate > 0 {
-		k.failUpdate--
-		if k.apply {
-			k.KeyValue.Update(ctx, key, value, rev)
-		}
-		return 0, errors.New("nats: timeout")
-	}
-	return k.KeyValue.Update(ctx, key, value, rev)
-}
-
-func TestAmbiguousWrites(t *testing.T) {
-	ctx := context.Background()
-	t.Run("applied claim with a lost reply is recovered", func(t *testing.T) {
-		base, kv := newStore(t)
-		s := Store{JS: base.JS, KV: &lossyKV{KeyValue: kv, failCreate: 1, apply: true}}
-		d, err := s.Claim(ctx, region, ident(1, 1))
-		if err != nil || !d.CreateKey {
-			t.Fatalf("claim = %+v, %v", d, err)
-		}
-	})
-	t.Run("dropped claim that a rival won is refused", func(t *testing.T) {
-		base, kv := newStore(t)
-		lossy := &lossyKV{KeyValue: kv, failCreate: 1}
-		lossy.beforeCreate = func() { base.Claim(ctx, region, ident(2, 1)) }
-		if _, err := (Store{JS: base.JS, KV: lossy}).Claim(ctx, region, ident(1, 1)); !errors.Is(err, ErrClaimHeld) {
-			t.Fatalf("claim = %v, want ErrClaimHeld", err)
-		}
-	})
-	t.Run("applied key record with a lost reply is confirmed on retry", func(t *testing.T) {
-		base, kv := newStore(t)
-		w := ident(1, 1)
-		base.Claim(ctx, region, w)
-		pub, _ := userPub(t)
-		lossy := &lossyKV{KeyValue: kv, failUpdate: 1, apply: true}
-		s := Store{JS: base.JS, KV: lossy}
-		// The update applies but its reply is lost; the re-read sees it recorded.
-		if err := s.RecordPublicKey(ctx, region, w, pub); err != nil {
-			t.Fatalf("record = %v, want recovered success", err)
-		}
-		d, err := s.Claim(ctx, region, w)
-		if err != nil || d.CreateKey || d.PublicKey != pub {
-			t.Fatalf("claim after lost reply = %+v, %v", d, err)
-		}
-	})
-	t.Run("persistently dropped writes authorize nothing", func(t *testing.T) {
-		base, kv := newStore(t)
-		s := Store{JS: base.JS, KV: &lossyKV{KeyValue: kv, failCreate: maxAttempts}}
-		d, err := s.Claim(ctx, region, ident(1, 1))
-		if !errors.Is(err, ErrContended) || d.CreateKey {
-			t.Fatalf("claim = %+v, %v; want ErrContended", d, err)
-		}
-	})
-}
-
-func TestValidation(t *testing.T) {
-	s, _ := newStore(t)
-	ctx := context.Background()
-	bad := []Identity{
-		{InstanceID: "", ClaimID: "claim-0001", Generation: 1},
-		{InstanceID: "i-00000000000000001", ClaimID: "", Generation: 1},
-		{InstanceID: "i-00000000000000001", ClaimID: "claim-0001", Generation: 0},
-		{InstanceID: "i-00000000000000001.x", ClaimID: "claim-0001", Generation: 1},
-	}
-	for _, id := range bad {
-		if _, err := s.Claim(ctx, region, id); err == nil {
-			t.Errorf("claim accepted %+v", id)
-		}
-	}
-	if _, err := s.Claim(ctx, "Bad.Region", ident(1, 1)); err == nil {
-		t.Error("claim accepted an invalid region")
-	}
-}
-
-// laggingReplica serves reads from a second server whose bucket holds an
-// older history of the record, so its revisions match the leader's past ones,
-// while writes still go to the real leader.
-func laggingReplica(t *testing.T, leader Store, history ...string) Store {
-	t.Helper()
-	_, _, js := testutil.StartTestJetStream(t)
-	kv, err := js.CreateKeyValue(context.Background(), BucketConfig(1))
-	if err != nil {
-		t.Fatal(err)
-	}
-	for _, v := range history {
-		if _, err := kv.Put(context.Background(), key(region), []byte(v)); err != nil {
-			t.Fatal(err)
-		}
-	}
-	return Store{JS: js, KV: leader.KV}
-}
-
-func TestStaleRead_RecordedKeyNeverReauthorizesCreation(t *testing.T) {
-	ctx := context.Background()
-	s, kv := newStore(t)
-	w := ident(1, 1)
-	s.Claim(ctx, region, w)
-	claimed := stored(t, kv)
-	pub, _ := userPub(t)
-	if err := s.RecordPublicKey(ctx, region, w, pub); err != nil {
-		t.Fatal(err)
-	}
-	// The replica has applied the claim but not the key record.
-	stale := laggingReplica(t, s, claimed)
-	d, err := stale.Claim(ctx, region, w)
-	if d.CreateKey || !errors.Is(err, ErrContended) {
-		t.Fatalf("stale claim = %+v, %v; want no authorization", d, err)
-	}
-	if err := stale.RecordLaunchIntent(ctx, region, w, pub); !errors.Is(err, ErrKeyNotRecorded) {
-		t.Fatalf("stale launch intent = %v, want a refusal", err)
-	}
-	if d, err := s.Claim(ctx, region, w); err != nil || d.CreateKey || d.PublicKey != pub {
-		t.Fatalf("leader claim = %+v, %v; want the recorded key", d, err)
-	}
-}
-
-func TestStaleRead_OldGenerationIsNotRevivedAfterRebind(t *testing.T) {
-	ctx := context.Background()
-	for _, keyed := range []bool{false, true} {
-		t.Run(map[bool]string{false: "claimed", true: "key recorded"}[keyed], func(t *testing.T) {
-			s, kv := newStore(t)
-			old, next := ident(1, 1), ident(2, 2)
-			s.Claim(ctx, region, old)
-			history := []string{stored(t, kv)}
-			pub, _ := userPub(t)
-			if keyed {
-				s.RecordPublicKey(ctx, region, old, pub)
-				history = append(history, stored(t, kv))
-			}
-			if _, err := s.Claim(ctx, region, next); err != nil {
-				t.Fatal(err)
-			}
-			// The replica still shows the old generation as the holder.
-			stale := laggingReplica(t, s, history...)
-			if d, err := stale.Claim(ctx, region, old); err == nil || d.CreateKey || d.PublicKey != "" {
-				t.Fatalf("stale old-generation claim = %+v, %v; want refusal", d, err)
-			}
-			if err := stale.RecordPublicKey(ctx, region, old, pub); err == nil {
-				t.Fatal("stale old-generation record was accepted")
-			}
-			if err := stale.RecordLaunchIntent(ctx, region, old, pub); err == nil {
-				t.Fatal("stale old-generation launch intent was accepted")
-			}
-			var rec Record
-			json.Unmarshal([]byte(stored(t, kv)), &rec)
-			if rec.Worker != next {
-				t.Fatalf("leader record holder = %+v, want the newer generation", rec.Worker)
-			}
-		})
-	}
-}
-
-// Reads must be stream message gets, which only the leader answers, never
-// direct gets, which any replica answers.
-func TestReads_UseLeaderMessageGet(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
-	kv, err := js.CreateKeyValue(context.Background(), BucketConfig(1))
-	if err != nil {
-		t.Fatal(err)
-	}
-	var direct, leader atomic.Int32
-	sub1, _ := nc.Subscribe("$JS.API.DIRECT.GET.KV_"+Bucket+".>", func(*nats.Msg) { direct.Add(1) })
-	sub2, _ := nc.Subscribe("$JS.API.DIRECT.GET.KV_"+Bucket, func(*nats.Msg) { direct.Add(1) })
-	sub3, _ := nc.Subscribe("$JS.API.STREAM.MSG.GET.KV_"+Bucket, func(*nats.Msg) { leader.Add(1) })
-	defer sub1.Unsubscribe()
-	defer sub2.Unsubscribe()
-	defer sub3.Unsubscribe()
-	nc.Flush()
-	s := Store{JS: js, KV: kv}
-	ctx := context.Background()
-	w := ident(1, 1)
-	pub, _ := userPub(t)
-	s.Claim(ctx, region, w)
-	s.RecordPublicKey(ctx, region, w, pub)
-	s.Claim(ctx, region, w)
-	nc.Flush()
-	if direct.Load() != 0 || leader.Load() < 3 {
-		t.Fatalf("direct gets=%d leader gets=%d; want only leader reads", direct.Load(), leader.Load())
-	}
-}
-
-func mustFence(t *testing.T, s Store, gen uint64) {
-	t.Helper()
-	if err := s.Fence(context.Background(), region, gen); err != nil {
-		t.Fatalf("fence %d: %v", gen, err)
-	}
-}
-
-func fencedAt(t *testing.T, s Store) uint64 {
-	t.Helper()
-	f, err := s.FencedAt(context.Background(), region)
-	if err != nil {
-		t.Fatal(err)
-	}
-	return f
-}
-
-// A fence raised before any claim leaves an open record: an older generation
-// can neither claim nor record, and the fenced generation claims as first.
-func TestFence_BeforeAnyClaim(t *testing.T) {
-	s, _ := newStore(t)
-	ctx := context.Background()
-	if f := fencedAt(t, s); f != 0 {
-		t.Fatalf("fence with no record = %d, want 0", f)
-	}
-	mustFence(t, s, 2)
-	pub, _ := userPub(t)
-	if _, err := s.Claim(ctx, region, ident(1, 1)); !errors.Is(err, ErrStaleGeneration) {
-		t.Fatalf("old claim behind the fence = %v, want ErrStaleGeneration", err)
-	}
-	if err := s.RecordPublicKey(ctx, region, ident(1, 1), pub); !errors.Is(err, ErrStaleGeneration) {
-		t.Fatalf("old record behind the fence = %v, want ErrStaleGeneration", err)
-	}
-	if err := s.RecordPublicKey(ctx, region, ident(2, 2), pub); !errors.Is(err, ErrNotClaimed) {
-		t.Fatalf("record before any claim = %v, want ErrNotClaimed", err)
-	}
-	d, err := s.Claim(ctx, region, ident(2, 2))
-	if err != nil || !d.CreateKey {
-		t.Fatalf("fenced generation's claim = %+v, %v; want CreateKey", d, err)
-	}
-	if f := fencedAt(t, s); f != 2 {
-		t.Fatalf("claim moved the fence to %d", f)
-	}
-}
-
-// Raised over a recorded key, the fence refuses the old generation's every
-// call and keeps the key, which the successor alone receives.
-func TestFence_OverARecordedKey(t *testing.T) {
-	s, _ := newStore(t)
-	ctx := context.Background()
-	old, next := ident(1, 1), ident(2, 2)
-	pub, _ := userPub(t)
-	if _, err := s.Claim(ctx, region, old); err != nil {
-		t.Fatal(err)
-	}
-	if err := s.RecordPublicKey(ctx, region, old, pub); err != nil {
-		t.Fatal(err)
-	}
-	mustFence(t, s, 2)
-	if _, err := s.Claim(ctx, region, old); !errors.Is(err, ErrStaleGeneration) {
-		t.Fatalf("old claim = %v, want ErrStaleGeneration", err)
-	}
-	if err := s.RecordPublicKey(ctx, region, old, pub); !errors.Is(err, ErrStaleGeneration) {
-		t.Fatalf("old retried record = %v, want ErrStaleGeneration", err)
-	}
-	if err := s.RecordLaunchIntent(ctx, region, old, pub); !errors.Is(err, ErrStaleGeneration) {
-		t.Fatalf("old launch intent = %v, want ErrStaleGeneration", err)
-	}
-	d, err := s.Claim(ctx, region, next)
-	if err != nil || d.CreateKey || d.PublicKey != pub {
-		t.Fatalf("successor = %+v, %v; want only the recorded key", d, err)
-	}
-}
-
-// The fence never falls, and repeating it is a no-op, so a retried or late
-// lower fence cannot readmit a replaced generation.
-func TestFence_OnlyRises(t *testing.T) {
-	s, _ := newStore(t)
-	for _, gen := range []uint64{3, 3, 1, 2} {
-		mustFence(t, s, gen)
-	}
-	if f := fencedAt(t, s); f != 3 {
-		t.Fatalf("fence = %d, want 3", f)
-	}
-	if _, err := s.Claim(context.Background(), region, ident(1, 2)); !errors.Is(err, ErrStaleGeneration) {
-		t.Fatalf("claim below the fence = %v, want ErrStaleGeneration", err)
-	}
-	if err := s.Fence(context.Background(), region, 0); err == nil {
-		t.Fatal("accepted a zero fence")
-	}
-}
-
-// An old-generation call racing the fence either lands before it, as a
-// completed write the successor inherits, or is refused; never both.
-func TestFence_RacesOldGenerationCalls(t *testing.T) {
-	for range 20 {
-		s, _ := newStore(t)
-		ctx := context.Background()
-		old := ident(1, 1)
-		pub, _ := userPub(t)
-		if _, err := s.Claim(ctx, region, old); err != nil {
-			t.Fatal(err)
-		}
-		var wg sync.WaitGroup
-		var recErr error
-		wg.Go(func() { recErr = s.RecordPublicKey(ctx, region, old, pub) })
-		wg.Go(func() { mustFence(t, s, 2) })
-		wg.Wait()
-		d, err := s.Claim(ctx, region, ident(2, 2))
-		if err != nil {
-			t.Fatal(err)
-		}
-		switch {
-		case recErr == nil && (d.CreateKey || d.PublicKey != pub):
-			t.Fatalf("old key recorded but successor got %+v", d)
-		case recErr != nil && (!errors.Is(recErr, ErrStaleGeneration) || !d.CreateKey):
-			t.Fatalf("old record refused with %v but successor got %+v", recErr, d)
-		}
-	}
-}
-
-// RecordedKey reports the recorded key and its recorder, before and after the
-// launch intent, and refuses when no key is recorded.
-func TestRecordedKey_ReportsTheRecorderAndKey(t *testing.T) {
-	ctx := context.Background()
-	s, _ := newStore(t)
-	w := ident(1, 1)
-	if _, _, err := s.RecordedKey(ctx, region); !errors.Is(err, ErrNotClaimed) {
-		t.Fatalf("before claim = %v", err)
-	}
-	s.Claim(ctx, region, w)
-	if _, _, err := s.RecordedKey(ctx, region); !errors.Is(err, ErrKeyNotRecorded) {
-		t.Fatalf("before key = %v, want ErrKeyNotRecorded", err)
-	}
-	pub, _ := userPub(t)
-	s.RecordPublicKey(ctx, region, w, pub)
-	for range 2 {
-		holder, got, err := s.RecordedKey(ctx, region)
-		if err != nil || holder != w || got != pub {
-			t.Fatalf("recorded key = %+v %q %v", holder, got, err)
-		}
-		if err := s.RecordLaunchIntent(ctx, region, w, pub); err != nil {
-			t.Fatal(err)
-		}
-	}
-}

spinifex/domains/sqs/message/jetstream/guestconfig_test.go

@@ -15,35 +15,29 @@ import (
 	"github.com/nats-io/nats-server/v2/server"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
-	"github.com/nats-io/nkeys"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-// guestLib is the store guest's shell library, from the repository root.
+// guestLib is the SQS guest's shell library, from the repository root.
 func guestLib(t *testing.T) string {
 	t.Helper()
 	_, here, _, ok := runtime.Caller(0)
 	require.True(t, ok)
-	p := filepath.Join(filepath.Dir(here), "..", "..", "..", "..", "..", "build", "microvm", "sqs-store", "sqs-store-lib.sh")
+	p := filepath.Join(filepath.Dir(here), "..", "..", "..", "..", "..", "build", "microvm", "sqs-worker", "sqs-worker-lib.sh")
 	_, err := os.Stat(p)
 	require.NoError(t, err)
 	return p
 }
 
-// guestStore starts a server from the config the store guest itself writes,
-// changing only the port, and connects with the client NKey it admits.
+// guestStore starts a server from the config the SQS guest itself writes,
+// changing only the port, and connects as the co-located worker does.
 func guestStore(t *testing.T) (message.Store, *server.Options) {
 	t.Helper()
-	kp, err := nkeys.CreateUser()
-	require.NoError(t, err)
-	pub, err := kp.PublicKey()
-	require.NoError(t, err)
-
 	dir := t.TempDir()
 	conf := filepath.Join(dir, "nats.conf")
-	cmd := exec.Command("bash", "-c", `. "$0" && sqs_write_config 127.0.0.1 "$1" "$2"`, guestLib(t), pub, conf)
-	cmd.Env = append(os.Environ(), "SQS_DATA="+filepath.Join(dir, "data"), "SQS_PORT=4222")
+	cmd := exec.Command("bash", "-c", `. "$0" && sqsw_write_config "$1"`, guestLib(t), conf)
+	cmd.Env = append(os.Environ(), "SQSW_DATA="+filepath.Join(dir, "data"))
 	out, err := cmd.CombinedOutput()
 	require.NoError(t, err, "%s", out)
 
@@ -57,7 +51,7 @@ func guestStore(t *testing.T) (message.Store, *server.Options) {
 	require.True(t, ns.ReadyForConnections(10*time.Second))
 	t.Cleanup(ns.Shutdown)
 
-	nc, err := nats.Connect(ns.ClientURL(), nats.Nkey(pub, kp.Sign))
+	nc, err := nats.Connect(ns.ClientURL())
 	require.NoError(t, err)
 	t.Cleanup(nc.Close)
 	js, err := jetstream.New(nc)

spinifex/domains/sqs/relay/exec.go

@@ -0,0 +1,186 @@
+package relay
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/mulgadc/bluebottle/pkg/sigv4"
+	"github.com/mulgadc/spinifex/spinifex/domains/sqs/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
+	"github.com/mulgadc/spinifex/spinifex/domains/sqs/queue"
+)
+
+// A posted result is a SigV4 payload of at most sigv4.MaxPayloadLen. Bodies
+// and string values travel as base64, so a message encodes to at most 4/3 of
+// the largest SQS message plus messageAllowance.
+const (
+	messageAllowance = 16 << 10
+	resultAllowance  = 64 << 10
+)
+
+// ReceiveLimit is the most messages one receive returns, so its result fits
+// a post whatever the messages hold. A claimed message is never dropped.
+var ReceiveLimit = min(message.MaxReceiveMessages,
+	(sigv4.MaxPayloadLen-resultAllowance)/((message.MaxMessageSize*4+2)/3+messageAllowance))
+
+// ErrResultTooLarge is a result that would not fit one post, or could not be
+// encoded. ReceiveLimit makes it unreachable for any result the owners return.
+var ErrResultTooLarge = errors.New("relay: result exceeds the post limit")
+
+// Owners are the worker's queue and message owners.
+type Owners struct {
+	Queues     awsapi.Queues
+	Messages   awsapi.Messages
+	Receives   awsapi.Receives
+	Deletes    awsapi.Deletes
+	Visibility awsapi.Visibility
+}
+
+// Claim is a message a receive took in flight, for release if the result is
+// proven undelivered. It holds a receipt handle and is never logged.
+type Claim struct {
+	Account, Region, Queue string
+	ReceiptHandle          string
+}
+
+// Execute runs w against the owners and returns its encoded result and the
+// claims it took. A result that cannot be posted whole is replaced by an
+// unavailable error and its claims are withheld, so they are never released.
+func (o Owners) Execute(ctx context.Context, w Work) ([]byte, []Claim) {
+	out, claims, err := o.run(ctx, w)
+	res := Result{Output: out}
+	if err != nil {
+		res = Result{Error: encodeError(err)}
+	}
+	body, mErr := MarshalResult(res)
+	if mErr == nil && len(body) <= sigv4.MaxPayloadLen {
+		return body, claims
+	}
+	body, _ = MarshalResult(Result{Error: encodeError(fmt.Errorf("%w: %w", message.ErrUnavailable, ErrResultTooLarge))})
+	return body, nil
+}
+
+// Release returns claims to visibility at once. Call it only when the
+// ingress proved the result was not delivered.
+func (o Owners) Release(ctx context.Context, claims []Claim) error {
+	var errs []error
+	for _, c := range claims {
+		errs = append(errs, o.Visibility.ChangeVisibility(ctx, message.ChangeVisibilityInput{
+			Account: c.Account, Region: c.Region, Queue: c.Queue, ReceiptHandle: c.ReceiptHandle,
+		}))
+	}
+	return errors.Join(errs...)
+}
+
+func decodeIn[T any](raw json.RawMessage) (T, error) {
+	var v T
+	if err := json.Unmarshal(raw, &v); err != nil {
+		return v, fmt.Errorf("%w: undecodable work input", errWorkerFault)
+	}
+	return v, nil
+}
+
+func encodeOut(v any, err error) (json.RawMessage, []Claim, error) {
+	if err != nil {
+		return nil, nil, err
+	}
+	b, err := marshal(v)
+	return b, nil, err
+}
+
+func (o Owners) run(ctx context.Context, w Work) (json.RawMessage, []Claim, error) {
+	switch w.Op {
+	case OpCreateQueue:
+		in, err := decodeIn[queue.CreateRequest](w.Input)
+		if err != nil {
+			return nil, nil, err
+		}
+		return encodeOut(o.Queues.Create(ctx, in))
+	case OpResolveQueue:
+		in, err := decodeIn[nameInput](w.Input)
+		if err != nil {
+			return nil, nil, err
+		}
+		return encodeOut(o.Queues.Resolve(ctx, in.Account, in.Region, in.Name))
+	case OpListQueues:
+		in, err := decodeIn[queue.ListRequest](w.Input)
+		if err != nil {
+			return nil, nil, err
+		}
+		return encodeOut(o.Queues.List(ctx, in))
+	case OpSetQueueAttributes:
+		in, err := decodeIn[attrsInput](w.Input)
+		if err != nil {
+			return nil, nil, err
+		}
+		return encodeOut(o.Queues.SetAttributes(ctx, in.Account, in.Region, in.Name, in.Attributes))
+	case OpTagQueue:
+		in, err := decodeIn[tagsInput](w.Input)
+		if err != nil {
+			return nil, nil, err
+		}
+		return encodeOut(o.Queues.Tag(ctx, in.Account, in.Region, in.Name, in.Tags))
+	case OpUntagQueue:
+		in, err := decodeIn[untagInput](w.Input)
+		if err != nil {
+			return nil, nil, err
+		}
+		return encodeOut(o.Queues.Untag(ctx, in.Account, in.Region, in.Name, in.Keys))
+	case OpListQueueTags:
+		in, err := decodeIn[nameInput](w.Input)
+		if err != nil {
+			return nil, nil, err
+		}
+		tags, err := o.Queues.Tags(ctx, in.Account, in.Region, in.Name)
+		if tags == nil {
+			tags = map[string]string{}
+		}
+		return encodeOut(tags, err)
+	case OpSendMessage:
+		in, err := decodeIn[message.SendInput](w.Input)
+		if err != nil {
+			return nil, nil, err
+		}
+		return encodeOut(o.Messages.Send(ctx, in))
+	case OpReceiveMessage:
+		return o.receive(ctx, w.Input)
+	case OpDeleteMessage:
+		in, err := decodeIn[message.DeleteInput](w.Input)
+		if err != nil {
+			return nil, nil, err
+		}
+		return encodeOut(struct{}{}, o.Deletes.Delete(ctx, in))
+	case OpChangeVisibility:
+		in, err := decodeIn[message.ChangeVisibilityInput](w.Input)
+		if err != nil {
+			return nil, nil, err
+		}
+		return encodeOut(struct{}{}, o.Visibility.ChangeVisibility(ctx, in))
+	}
+	return nil, nil, fmt.Errorf("%w: unknown operation", errWorkerFault)
+}
+
+// receive lowers a valid request above ReceiveLimit before anything is
+// claimed; an out-of-range request reaches the owner and is refused there.
+func (o Owners) receive(ctx context.Context, raw json.RawMessage) (json.RawMessage, []Claim, error) {
+	in, err := decodeIn[message.ReceiveInput](raw)
+	if err != nil {
+		return nil, nil, err
+	}
+	if n := in.MaxNumberOfMessages; n != nil && *n > ReceiveLimit && *n <= message.MaxReceiveMessages {
+		limit := ReceiveLimit
+		in.MaxNumberOfMessages = &limit
+	}
+	msgs, err := o.Receives.Receive(ctx, in)
+	if err != nil {
+		return nil, nil, err
+	}
+	claims := make([]Claim, len(msgs))
+	for i, m := range msgs {
+		claims[i] = Claim{Account: in.Account, Region: in.Region, Queue: in.Queue, ReceiptHandle: m.ReceiptHandle}
+	}
+	b, err := marshal(toWireReceived(msgs))
+	return b, claims, err
+}

spinifex/domains/sqs/relay/owners_test.go

@@ -0,0 +1,172 @@
+package relay_test
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
+	"github.com/mulgadc/spinifex/spinifex/domains/sqs/queue"
+	"github.com/mulgadc/spinifex/spinifex/domains/sqs/relay"
+	"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerserve"
+)
+
+const (
+	acct   = "123456789012"
+	region = "ap-southeast-2"
+)
+
+var taker = relay.Taker{InstanceID: "i-0abc12345def67890", Generation: 1, ClaimID: "claim-0001"}
+
+// served starts real owners on an embedded store behind r, posting each
+// result back as the worker loop does.
+func served(t *testing.T) (*relay.Relay, relay.Owners) {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	key := make([]byte, 32)
+	if _, err := rand.Read(key); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	t.Cleanup(cancel)
+	owners, err := workerserve.OpenOwners(ctx, js, key)
+	if err != nil {
+		t.Fatalf("open owners: %v", err)
+	}
+	r := &relay.Relay{}
+	polled, done := context.WithCancel(ctx)
+	done()
+	r.Take(polled, taker, 1, 1)
+	go func() {
+		for ctx.Err() == nil {
+			for _, w := range r.Take(ctx, taker, 10, 10<<20) {
+				body, _ := owners.Execute(ctx, w)
+				var res relay.Result
+				if err := json.Unmarshal(body, &res); err != nil {
+					panic(err)
+				}
+				_, _ = r.Complete(taker, w.ID, res)
+			}
+		}
+	}()
+	return r, owners
+}
+
+func callCtx(t *testing.T) context.Context {
+	t.Helper()
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	t.Cleanup(cancel)
+	return ctx
+}
+
+// Every queue action crosses the wire into the real owner and back, and its
+// effect is visible to the next call.
+func TestOwners_QueueActionsRoundTrip(t *testing.T) {
+	r, _ := served(t)
+	ctx := callCtx(t)
+
+	rec, err := r.Create(ctx, queue.CreateRequest{Account: acct, Region: region, Name: "orders", Tags: map[string]string{"team": "a"}})
+	if err != nil {
+		t.Fatalf("create: %v", err)
+	}
+	if got, err := r.Resolve(ctx, acct, region, "orders"); err != nil || got.QueueID != rec.QueueID {
+		t.Fatalf("resolve = %v, %v; want %s", got.QueueID, err, rec.QueueID)
+	}
+	if _, err := r.SetAttributes(ctx, acct, region, "orders", map[string]string{"VisibilityTimeout": "45"}); err != nil {
+		t.Fatalf("set attributes: %v", err)
+	}
+	if got, _ := r.Resolve(ctx, acct, region, "orders"); got.Attributes["VisibilityTimeout"] != "45" {
+		t.Fatalf("VisibilityTimeout = %q after set", got.Attributes["VisibilityTimeout"])
+	}
+	if _, err := r.Tag(ctx, acct, region, "orders", map[string]string{"env": "dev"}); err != nil {
+		t.Fatalf("tag: %v", err)
+	}
+	if _, err := r.Untag(ctx, acct, region, "orders", []string{"team"}); err != nil {
+		t.Fatalf("untag: %v", err)
+	}
+	tags, err := r.Tags(ctx, acct, region, "orders")
+	if err != nil || len(tags) != 1 || tags["env"] != "dev" {
+		t.Fatalf("tags = %v, %v; want only env=dev", tags, err)
+	}
+	page, err := r.List(ctx, queue.ListRequest{Account: acct, Region: region})
+	if err != nil || len(page.Queues) != 1 || page.Queues[0].Name != "orders" {
+		t.Fatalf("list = %+v, %v", page, err)
+	}
+	if _, err := r.Resolve(ctx, acct, region, "missing"); !errors.Is(err, queue.ErrQueueDoesNotExist) {
+		t.Fatalf("resolve missing = %v; want QueueDoesNotExist", err)
+	}
+}
+
+// A receive asking for the AWS maximum claims at most ReceiveLimit, leaving
+// the rest visible; Release and ChangeVisibility return claims at once.
+func TestOwners_ReceiveClaimsAtMostTheLimitAndReleaseReturnsThem(t *testing.T) {
+	r, owners := served(t)
+	ctx := callCtx(t)
+	if _, err := r.Create(ctx, queue.CreateRequest{Account: acct, Region: region, Name: "jobs"}); err != nil {
+		t.Fatalf("create: %v", err)
+	}
+	for i := range message.MaxReceiveMessages {
+		if _, err := r.Send(ctx, message.SendInput{Account: acct, Region: region, Queue: "jobs", Sender: "AIDATEST", Body: fmt.Appendf(nil, "m%d", i)}); err != nil {
+			t.Fatalf("send %d: %v", i, err)
+		}
+	}
+	maxN, hide := message.MaxReceiveMessages, 300
+	in := message.ReceiveInput{Account: acct, Region: region, Queue: "jobs", MaxNumberOfMessages: &maxN, VisibilityTimeout: &hide}
+	first, err := r.Receive(ctx, in)
+	if err != nil || len(first) == 0 || len(first) > relay.ReceiveLimit {
+		t.Fatalf("receive = %d messages, %v; want 1..%d", len(first), err, relay.ReceiveLimit)
+	}
+	rest, err := r.Receive(ctx, in)
+	if err != nil || len(first)+len(rest) != message.MaxReceiveMessages {
+		t.Fatalf("second receive = %d, %v; want the %d unclaimed", len(rest), err, message.MaxReceiveMessages-len(first))
+	}
+
+	// Everything is now hidden for 300s. Release the first batch's claims
+	// directly and the second through ChangeMessageVisibility over the wire.
+	claims := make([]relay.Claim, len(first))
+	for i, m := range first {
+		claims[i] = relay.Claim{Account: acct, Region: region, Queue: "jobs", ReceiptHandle: m.ReceiptHandle}
+	}
+	if err := owners.Release(ctx, claims); err != nil {
+		t.Fatalf("release: %v", err)
+	}
+	for _, m := range rest {
+		if err := r.ChangeVisibility(ctx, message.ChangeVisibilityInput{Account: acct, Region: region, Queue: "jobs", ReceiptHandle: m.ReceiptHandle}); err != nil {
+			t.Fatalf("change visibility: %v", err)
+		}
+	}
+	seen := 0
+	for seen < message.MaxReceiveMessages {
+		got, err := r.Receive(ctx, in)
+		if err != nil || len(got) == 0 {
+			t.Fatalf("after release saw %d of %d, then %v", seen, message.MaxReceiveMessages, err)
+		}
+		seen += len(got)
+	}
+}
+
+// Work the worker cannot decode or does not know is a fault, never a
+// success, and claims nothing.
+func TestOwners_UnknownOrUndecodableWorkFaults(t *testing.T) {
+	_, owners := served(t)
+	ctx := callCtx(t)
+	for _, w := range []relay.Work{
+		{ID: "a", Op: "PurgeEverything", Input: json.RawMessage(`{}`)},
+		{ID: "b", Op: relay.OpSendMessage, Input: json.RawMessage(`[`)},
+		{ID: "c", Op: relay.OpReceiveMessage, Input: json.RawMessage(`"x"`)},
+	} {
+		body, claims := owners.Execute(ctx, w)
+		var res relay.Result
+		if err := json.Unmarshal(body, &res); err != nil {
+			t.Fatalf("%s: result undecodable: %v", w.Op, err)
+		}
+		if res.Error == nil || res.Error.Kind != "fault" || res.Output != nil || claims != nil {
+			t.Fatalf("%s: result %+v claims %d; want a fault and no claims", w.Op, res, len(claims))
+		}
+	}
+}

spinifex/domains/sqs/relay/relay.go

@@ -0,0 +1,480 @@
+package relay
+
+import (
+	"context"
+	"crypto/md5" //nolint:gosec // SQS defines MD5OfBody.
+	"crypto/rand"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"regexp"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/sqs/awsapi"
+	"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
+	"github.com/mulgadc/spinifex/spinifex/domains/sqs/queue"
+)
+
+// Defaults for a Relay whose fields are zero.
+const (
+	// DefaultCallLimit bounds a call end to end; it exceeds the 20-second
+	// long poll with room for the worker's poll and post.
+	DefaultCallLimit = 35 * time.Second
+	// DefaultPollerStale is how long after the worker's last poll calls are
+	// still queued; past it no worker is assumed and calls fail at once.
+	DefaultPollerStale = 30 * time.Second
+	// DefaultMaxPending bounds work queued or held by the worker.
+	DefaultMaxPending = 512
+	// DefaultMaxPendingBytes bounds the encoded inputs of that work.
+	DefaultMaxPendingBytes = 64 << 20
+	// takenRetention keeps a taken item after its caller left, so a late
+	// post is answered as undelivered rather than unknown.
+	takenRetention = 2 * time.Minute
+)
+
+// Taker is the worker identity that took an item, as the ingress authorized
+// it on that poll. Only the same identity may post its result.
+type Taker struct {
+	InstanceID string
+	Generation uint64
+	ClaimID    string
+}
+
+// Outcome is what became of a posted result.
+type Outcome int
+
+const (
+	// Delivered: the result was handed to the waiting caller.
+	Delivered Outcome = iota + 1
+	// CallerGone: the caller had stopped waiting before the result arrived,
+	// so nothing from this result reached it.
+	CallerGone
+	// Unknown: the item is not held; whether a result reached the caller
+	// cannot be said, so the worker must not act on non-delivery.
+	Unknown
+)
+
+var (
+	// ErrNotTaker is a post from an identity other than the one that took
+	// the item.
+	ErrNotTaker = errors.New("relay: work was taken by another worker identity")
+	// ErrNoWorker is a call made while no worker is polling, or with the
+	// queue full; nothing was handed out.
+	ErrNoWorker = errors.New("relay: no SQS worker is accepting work")
+	// ErrNotAdmitted is a call made while the service is not enabled.
+	ErrNotAdmitted = errors.New("relay: SQS is not enabled in this Region")
+)
+
+type item struct {
+	work  Work
+	taken *Taker
+	// gone is set, under the lock, once the caller stops waiting.
+	gone   bool
+	result chan Result
+}
+
+// Relay is the ingress end: it queues owner calls and hands them to the
+// polling worker. It holds work in memory only, so it serves the worker
+// polling this ingress and nothing else.
+type Relay struct {
+	// Now is the clock; nil means time.Now.
+	Now func() time.Time
+	// CallLimit, PollerStale and MaxPending default when zero.
+	CallLimit       time.Duration
+	PollerStale     time.Duration
+	MaxPending      int
+	MaxPendingBytes int
+	// Admit reports whether a tenant call may be queued; nil admits every
+	// call. A refused call is unavailable and never reaches a worker.
+	Admit func(context.Context) bool
+
+	mu    sync.Mutex
+	items map[string]*item
+	queue []string
+	// bytes is the encoded input size of every held item.
+	bytes    int
+	lastPoll time.Time
+	// polling counts polls waiting now; a waiting poll is a present worker.
+	polling int
+	wake    chan struct{}
+}
+
+func (r *Relay) now() time.Time {
+	if r.Now != nil {
+		return r.Now()
+	}
+	return time.Now()
+}
+
+func (r *Relay) init() {
+	if r.items == nil {
+		r.items = map[string]*item{}
+		r.wake = make(chan struct{})
+	}
+}
+
+// call queues one owner call and waits for its result, the caller's context
+// or the call limit. Every failure to obtain a result is unavailable.
+func (r *Relay) call(ctx context.Context, op string, in any, unavailable error) (json.RawMessage, error) {
+	if r.Admit != nil && !r.Admit(ctx) {
+		return nil, errors.Join(unavailable, ErrNotAdmitted)
+	}
+	input, err := marshal(in)
+	if err != nil {
+		return nil, unavailable
+	}
+	limit := r.CallLimit
+	if limit <= 0 {
+		limit = DefaultCallLimit
+	}
+	ctx, cancel := context.WithTimeout(ctx, limit)
+	defer cancel()
+	deadline, _ := ctx.Deadline()
+	id, err := newID()
+	if err != nil {
+		return nil, unavailable
+	}
+	it := &item{work: Work{ID: id, Op: op, Deadline: deadline, Input: input}, result: make(chan Result, 1)}
+	if err := r.enqueue(it); err != nil {
+		return nil, errors.Join(unavailable, err)
+	}
+	select {
+	case res := <-it.result:
+		if res.Error != nil {
+			return nil, decodeError(res.Error)
+		}
+		return res.Output, nil
+	case <-ctx.Done():
+	}
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	select {
+	case res := <-it.result:
+		// Posted while the caller was leaving: still delivered.
+		if res.Error != nil {
+			return nil, decodeError(res.Error)
+		}
+		return res.Output, nil
+	default:
+	}
+	it.gone = true
+	if it.taken == nil {
+		r.drop(id)
+		r.removeQueued(id)
+	}
+	return nil, unavailable
+}
+
+func (r *Relay) enqueue(it *item) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.init()
+	r.expire()
+	stale := r.stale()
+	limit := r.MaxPending
+	if limit <= 0 {
+		limit = DefaultMaxPending
+	}
+	maxBytes := r.MaxPendingBytes
+	if maxBytes <= 0 {
+		maxBytes = DefaultMaxPendingBytes
+	}
+	if !r.presentLocked(stale) || len(r.items) >= limit || r.bytes+len(it.work.Input) > maxBytes {
+		return ErrNoWorker
+	}
+	r.bytes += len(it.work.Input)
+	r.items[it.work.ID] = it
+	r.queue = append(r.queue, it.work.ID)
+	close(r.wake)
+	r.wake = make(chan struct{})
+	return nil
+}
+
+func (r *Relay) stale() time.Duration {
+	if r.PollerStale <= 0 {
+		return DefaultPollerStale
+	}
+	return r.PollerStale
+}
+
+func (r *Relay) presentLocked(stale time.Duration) bool {
+	return r.polling > 0 || !r.lastPoll.IsZero() && r.now().Sub(r.lastPoll) <= stale
+}
+
+// WorkerPresent reports whether a worker is polling this ingress now or did
+// within the poller staleness window.
+func (r *Relay) WorkerPresent() bool {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return r.presentLocked(r.stale())
+}
+
+// drop forgets a held item and its bytes.
+func (r *Relay) drop(id string) {
+	if it, ok := r.items[id]; ok {
+		r.bytes -= len(it.work.Input)
+		delete(r.items, id)
+	}
+}
+
+func (r *Relay) removeQueued(id string) {
+	for i, q := range r.queue {
+		if q == id {
+			r.queue = append(r.queue[:i], r.queue[i+1:]...)
+			return
+		}
+	}
+}
+
+// expire drops taken items whose caller left long enough ago that no post
+// for them is still expected.
+func (r *Relay) expire() {
+	now := r.now()
+	for id, it := range r.items {
+		if it.taken != nil && now.After(it.work.Deadline.Add(takenRetention)) {
+			r.drop(id)
+		}
+	}
+}
+
+// Take hands taker up to limit queued items whose inputs total at most
+// maxBytes, but always at least one, waiting until one is queued or ctx
+// ends. The poll itself marks a worker present.
+func (r *Relay) Take(ctx context.Context, taker Taker, limit, maxBytes int) []Work {
+	r.mu.Lock()
+	r.polling++
+	r.mu.Unlock()
+	defer func() {
+		r.mu.Lock()
+		r.polling--
+		r.lastPoll = r.now()
+		r.mu.Unlock()
+	}()
+	for {
+		r.mu.Lock()
+		r.init()
+		r.lastPoll = r.now()
+		var out []Work
+		size := 0
+		for len(r.queue) > 0 && len(out) < limit {
+			id := r.queue[0]
+			it, ok := r.items[id]
+			if !ok || it.gone {
+				r.queue = r.queue[1:]
+				continue
+			}
+			if len(out) > 0 && size+len(it.work.Input) > maxBytes {
+				break
+			}
+			r.queue = r.queue[1:]
+			size += len(it.work.Input)
+			t := taker
+			it.taken = &t
+			out = append(out, it.work)
+		}
+		wake := r.wake
+		r.mu.Unlock()
+		if len(out) > 0 {
+			return out
+		}
+		select {
+		case <-ctx.Done():
+			return nil
+		case <-wake:
+		}
+	}
+}
+
+// Complete delivers res for id if taker took it and its caller still waits.
+// CallerGone is the only outcome that proves nothing reached the caller.
+func (r *Relay) Complete(taker Taker, id string, res Result) (Outcome, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.init()
+	it, ok := r.items[id]
+	if !ok || it.taken == nil {
+		return Unknown, nil
+	}
+	if *it.taken != taker {
+		return 0, ErrNotTaker
+	}
+	r.drop(id)
+	if it.gone {
+		return CallerGone, nil
+	}
+	it.result <- res
+	return Delivered, nil
+}
+
+func newID() (string, error) {
+	b := make([]byte, 16)
+	if _, err := rand.Read(b); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(b), nil
+}
+
+// Owner capabilities the public adapter runs against, carried to the worker.
+// A success is reported only for a complete result that names the queue the
+// caller asked about; anything else is unavailable, never a settled answer.
+var (
+	_ awsapi.Queues     = (*Relay)(nil)
+	_ awsapi.Messages   = (*Relay)(nil)
+	_ awsapi.Receives   = (*Relay)(nil)
+	_ awsapi.Deletes    = (*Relay)(nil)
+	_ awsapi.Visibility = (*Relay)(nil)
+)
+
+var md5Hex = regexp.MustCompile(`^[0-9a-f]{32}$`)
+
+func decodeOut[T any](raw json.RawMessage, unavailable error) (T, error) {
+	var v T
+	if len(raw) == 0 || string(raw) == "null" {
+		return v, unavailable
+	}
+	if err := json.Unmarshal(raw, &v); err != nil {
+		return v, unavailable
+	}
+	return v, nil
+}
+
+// validRecord requires an active queue with a usable namespace, since the
+// adapter pins sends and batches to it.
+func validRecord(r queue.Record, account, region, name string) bool {
+	return r.QueueID != "" && r.State == queue.StateActive && r.Namespace().Validate() == nil &&
+		r.Account == account && r.Region == region && (name == "" || r.Name == name)
+}
+
+func (r *Relay) record(ctx context.Context, op string, in any, account, region, name string) (queue.Record, error) {
+	raw, err := r.call(ctx, op, in, queue.ErrUnavailable)
+	if err != nil {
+		return queue.Record{}, err
+	}
+	rec, err := decodeOut[queue.Record](raw, queue.ErrUnavailable)
+	if err != nil || !validRecord(rec, account, region, name) {
+		return queue.Record{}, queue.ErrUnavailable
+	}
+	return rec, nil
+}
+
+func (r *Relay) Create(ctx context.Context, req queue.CreateRequest) (queue.Record, error) {
+	return r.record(ctx, OpCreateQueue, req, req.Account, req.Region, req.Name)
+}
+
+func (r *Relay) Resolve(ctx context.Context, account, region, name string) (queue.Record, error) {
+	return r.record(ctx, OpResolveQueue, nameInput{account, region, name}, account, region, name)
+}
+
+func (r *Relay) List(ctx context.Context, req queue.ListRequest) (queue.ListResult, error) {
+	raw, err := r.call(ctx, OpListQueues, req, queue.ErrUnavailable)
+	if err != nil {
+		return queue.ListResult{}, err
+	}
+	res, err := decodeOut[queue.ListResult](raw, queue.ErrUnavailable)
+	if err != nil {
+		return queue.ListResult{}, err
+	}
+	for _, q := range res.Queues {
+		if !validRecord(q, req.Account, req.Region, "") || !strings.HasPrefix(q.Name, req.Prefix) {
+			return queue.ListResult{}, queue.ErrUnavailable
+		}
+	}
+	return res, nil
+}
+
+func (r *Relay) SetAttributes(ctx context.Context, account, region, name string, attrs map[string]string) (queue.Record, error) {
+	return r.record(ctx, OpSetQueueAttributes, attrsInput{account, region, name, attrs}, account, region, name)
+}
+
+func (r *Relay) Tag(ctx context.Context, account, region, name string, tags map[string]string) (queue.Record, error) {
+	return r.record(ctx, OpTagQueue, tagsInput{account, region, name, tags}, account, region, name)
+}
+
+func (r *Relay) Untag(ctx context.Context, account, region, name string, keys []string) (queue.Record, error) {
+	return r.record(ctx, OpUntagQueue, untagInput{account, region, name, keys}, account, region, name)
+}
+
+func (r *Relay) Tags(ctx context.Context, account, region, name string) (map[string]string, error) {
+	raw, err := r.call(ctx, OpListQueueTags, nameInput{account, region, name}, queue.ErrUnavailable)
+	if err != nil {
+		return nil, err
+	}
+	return decodeOut[map[string]string](raw, queue.ErrUnavailable)
+}
+
+func (r *Relay) Send(ctx context.Context, in message.SendInput) (message.SendResult, error) {
+	raw, err := r.call(ctx, OpSendMessage, in, message.ErrUnavailable)
+	if err != nil {
+		return message.SendResult{}, err
+	}
+	res, err := decodeOut[message.SendResult](raw, message.ErrUnavailable)
+	// A position and enqueue time are the evidence the append completed.
+	if err != nil || res.MessageID == "" || res.Position == 0 || res.EnqueuedAt.IsZero() || !md5Hex.MatchString(res.MD5OfBody) ||
+		res.MD5OfBody != md5Of(in.Body) || res.MD5OfMessageAttributes != sentAttributesMD5(in) {
+		return message.SendResult{}, message.ErrUnavailable
+	}
+	return res, nil
+}
+
+func (r *Relay) Receive(ctx context.Context, in message.ReceiveInput) ([]message.Received, error) {
+	raw, err := r.call(ctx, OpReceiveMessage, in, message.ErrUnavailable)
+	if err != nil {
+		return nil, err
+	}
+	want := 1
+	if in.MaxNumberOfMessages != nil {
+		want = *in.MaxNumberOfMessages
+	}
+	wire, err := decodeOut[[]received](raw, message.ErrUnavailable)
+	if err != nil || len(wire) > min(want, ReceiveLimit) {
+		return nil, message.ErrUnavailable
+	}
+	out, err := fromWireReceived(wire)
+	if err != nil {
+		return nil, message.ErrUnavailable
+	}
+	for _, m := range out {
+		if m.MessageID == "" || m.ReceiptHandle == "" || m.MD5OfBody != md5Of(m.Body) || m.SenderID == "" ||
+			m.SentAt.IsZero() || m.FirstReceive.IsZero() || m.Deadline.IsZero() || m.ReceiveCount == 0 ||
+			m.Attributes.Len() > 0 && m.MD5OfMessageAttributes != m.Attributes.MD5() {
+			return nil, message.ErrUnavailable
+		}
+	}
+	return out, nil
+}
+
+// ack accepts only the empty object the worker answers a settled call with.
+func ack(raw json.RawMessage, err error) error {
+	if err != nil {
+		return err
+	}
+	if string(raw) != "{}" {
+		return message.ErrUnavailable
+	}
+	return nil
+}
+
+func (r *Relay) Delete(ctx context.Context, in message.DeleteInput) error {
+	return ack(r.call(ctx, OpDeleteMessage, in, message.ErrUnavailable))
+}
+
+func (r *Relay) ChangeVisibility(ctx context.Context, in message.ChangeVisibilityInput) error {
+	return ack(r.call(ctx, OpChangeVisibility, in, message.ErrUnavailable))
+}
+
+// sentAttributesMD5 is the digest the worker must return for in's attributes,
+// empty when it has none.
+func sentAttributesMD5(in message.SendInput) string {
+	attrs, err := message.NewAttributes(in.MessageAttributes)
+	if err != nil || attrs.Len() == 0 {
+		return ""
+	}
+	return attrs.MD5()
+}
+
+func md5Of(b []byte) string {
+	sum := md5.Sum(b) //nolint:gosec // SQS defines MD5OfBody; this is an integrity check, not security.
+	return hex.EncodeToString(sum[:])
+}

spinifex/domains/sqs/relay/relay_test.go

@@ -0,0 +1,303 @@
+package relay
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/bluebottle/pkg/sigv4"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
+	"github.com/mulgadc/spinifex/spinifex/domains/sqs/queue"
+)
+
+var worker = Taker{InstanceID: "i-0abc12345def67890", Generation: 1, ClaimID: "claim-0001"}
+
+// takeOne waits for the caller's item to be queued and takes it.
+func takeOne(t *testing.T, r *Relay, as Taker) Work {
+	t.Helper()
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	got := r.Take(ctx, as, 1, 1<<20)
+	if len(got) != 1 {
+		t.Fatalf("took %d items", len(got))
+	}
+	return got[0]
+}
+
+// present marks a worker as having polled, without leaving a poll waiting.
+func present(r *Relay) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	r.Take(ctx, worker, 1, 1)
+}
+
+// Only the identity that took an item may post it, and the caller gets the
+// owner's error with its sentinel intact.
+func TestComplete_DeliversOnlyFromTheTaker(t *testing.T) {
+	r := &Relay{}
+	present(r)
+	done := make(chan error, 1)
+	go func() {
+		_, err := r.Resolve(context.Background(), "123456789012", "ap-southeast-2", "orders")
+		done <- err
+	}()
+	w := takeOne(t, r, worker)
+	res := Result{Error: encodeError(queue.ErrQueueDoesNotExist)}
+
+	successor := worker
+	successor.Generation = 2
+	if _, err := r.Complete(successor, w.ID, res); !errors.Is(err, ErrNotTaker) {
+		t.Fatalf("post by another identity = %v", err)
+	}
+	if out, err := r.Complete(worker, w.ID, res); err != nil || out != Delivered {
+		t.Fatalf("post = %v, %v", out, err)
+	}
+	if err := <-done; !errors.Is(err, queue.ErrQueueDoesNotExist) {
+		t.Fatalf("caller got %v", err)
+	}
+	if out, _ := r.Complete(worker, w.ID, res); out != Unknown {
+		t.Fatalf("second post = %v, want Unknown", out)
+	}
+}
+
+// A post after the caller left is CallerGone, the one outcome that lets the
+// worker release what it claimed; an id never handed out is Unknown.
+func TestComplete_CallerGoneOnlyAfterTheCallerLeft(t *testing.T) {
+	r := &Relay{}
+	present(r)
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() {
+		_, err := r.Receive(ctx, message.ReceiveInput{Account: "123456789012", Region: "ap-southeast-2", Queue: "orders"})
+		done <- err
+	}()
+	w := takeOne(t, r, worker)
+	cancel()
+	if err := <-done; !errors.Is(err, message.ErrUnavailable) {
+		t.Fatalf("caller got %v", err)
+	}
+	out, err := r.Complete(worker, w.ID, Result{Output: []byte("[]")})
+	if err != nil || out != CallerGone {
+		t.Fatalf("late post = %v, %v", out, err)
+	}
+	if out, _ := r.Complete(worker, strings.Repeat("0", 32), Result{Output: []byte("[]")}); out != Unknown {
+		t.Fatalf("unknown id = %v", out)
+	}
+}
+
+// With no worker polling, a call fails at once and queues nothing.
+func TestCall_NoWorkerFailsAtOnce(t *testing.T) {
+	r := &Relay{PollerStale: time.Minute}
+	start := time.Now()
+	_, err := r.Send(context.Background(), message.SendInput{Account: "123456789012", Region: "ap-southeast-2", Queue: "orders"})
+	if !errors.Is(err, message.ErrUnavailable) || !errors.Is(err, ErrNoWorker) || time.Since(start) > time.Second {
+		t.Fatalf("send = %v after %s", err, time.Since(start))
+	}
+	if len(r.items) != 0 {
+		t.Fatal("a call was queued with no worker")
+	}
+}
+
+// While the service is not admitted, a call fails at once even with a worker
+// polling, and nothing reaches the worker.
+func TestCall_NotAdmittedNeverReachesTheWorker(t *testing.T) {
+	r := &Relay{PollerStale: time.Minute, Admit: func(context.Context) bool { return false }}
+	present(r)
+	_, err := r.Send(context.Background(), message.SendInput{Account: "123456789012", Region: "ap-southeast-2", Queue: "orders"})
+	if !errors.Is(err, message.ErrUnavailable) || !errors.Is(err, ErrNotAdmitted) {
+		t.Fatalf("send = %v", err)
+	}
+	if len(r.items) != 0 {
+		t.Fatal("a call was queued while not admitted")
+	}
+}
+
+// ReceiveLimit messages of the largest size, every byte in values that JSON
+// escapes, still fit one post.
+func TestReceiveLimit_WorstCaseFitsOnePost(t *testing.T) {
+	if ReceiveLimit < 1 {
+		t.Fatalf("ReceiveLimit = %d", ReceiveLimit)
+	}
+	var attrs []message.Attribute
+	used := 0
+	for i := range 10 {
+		v := strings.Repeat("\"
", 4096)
+		name := "attr" + string(rune('a'+i))
+		attrs = append(attrs, message.Attribute{Name: name, Value: message.AttributeValue{DataType: "String", StringValue: &v}})
+		used += len(name) + len("String") + len(v)
+	}
+	set, err := message.NewAttributes(attrs)
+	if err != nil {
+		t.Fatal(err)
+	}
+	msgs := make([]message.Received, ReceiveLimit)
+	for i := range msgs {
+		msgs[i] = message.Received{
+			MessageID: strings.Repeat("m", 36), ReceiptHandle: strings.Repeat("h", 1024),
+			Body: []byte(strings.Repeat("\"", message.MaxMessageSize-used)), MD5OfBody: strings.Repeat("0", 32),
+			Attributes: set, MD5OfMessageAttributes: strings.Repeat("0", 32), SenderID: strings.Repeat("s", 128),
+			SentAt: time.Now(), FirstReceive: time.Now(), Deadline: time.Now(), ReceiveCount: 1 << 62,
+		}
+	}
+	out, err := marshal(toWireReceived(msgs))
+	if err != nil {
+		t.Fatal(err)
+	}
+	body, err := MarshalResult(Result{Output: out})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(body) > sigv4.MaxPayloadLen {
+		t.Fatalf("%d messages encode to %d bytes, over %d", ReceiveLimit, len(body), sigv4.MaxPayloadLen)
+	}
+	back, err := fromWireReceived(toWireReceived(msgs))
+	if err != nil || string(back[0].Body) != string(msgs[0].Body) || back[0].Attributes.MD5() != set.MD5() {
+		t.Fatalf("round trip changed the message: %v", err)
+	}
+}
+
+// An uncertain write wrapped with a client fault stays unavailable, for both
+// owners, so the caller is never told a write was settled when it was not.
+func TestEncodeError_UnavailableWinsOverClientFaults(t *testing.T) {
+	for _, tc := range []struct{ unavailable, fault error }{
+		{message.ErrUnavailable, message.ErrInvalidParameter},
+		{message.ErrUnavailable, message.ErrReceiptInvalid},
+		{queue.ErrUnavailable, queue.ErrInvalidAttributeValue},
+		{queue.ErrUnavailable, queue.ErrQueueDoesNotExist},
+	} {
+		got := decodeError(encodeError(errors.Join(tc.fault, tc.unavailable)))
+		if !errors.Is(got, tc.unavailable) || errors.Is(got, tc.fault) {
+			t.Fatalf("%v + %v decoded as %v", tc.fault, tc.unavailable, got)
+		}
+	}
+}
+
+// Owner and backend text never crosses the wire: the caller sees only the
+// fixed text for the kind.
+func TestEncodeError_CarriesNoOwnerText(t *testing.T) {
+	const secret = "AQEBsecretReceiptHandle"
+	wire := encodeError(fmt.Errorf("%w: handle %s rejected", message.ErrReceiptInvalid, secret))
+	body, err := MarshalResult(Result{Error: wire})
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := decodeError(wire)
+	if strings.Contains(string(body), secret) || strings.Contains(got.Error(), secret) || !errors.Is(got, message.ErrReceiptInvalid) {
+		t.Fatalf("wire %s, decoded %q", body, got)
+	}
+}
+
+// answer runs one call through the relay with the worker posting output.
+func answer(t *testing.T, output string, call func(r *Relay) error) error {
+	t.Helper()
+	r := &Relay{}
+	present(r)
+	done := make(chan error, 1)
+	go func() { done <- call(r) }()
+	w := takeOne(t, r, worker)
+	if _, err := r.Complete(worker, w.ID, Result{Output: []byte(output)}); err != nil {
+		t.Fatal(err)
+	}
+	return <-done
+}
+
+// A malformed, empty or mismatched success from the worker is reported as
+// unavailable, never as a confirmed send, receive, delete or queue.
+func TestRelay_IncompleteSuccessIsUnavailable(t *testing.T) {
+	const acct, region = "123456789012", "ap-southeast-2"
+	send := func(r *Relay) error {
+		_, err := r.Send(context.Background(), message.SendInput{Account: acct, Region: region, Queue: "orders", Body: []byte("hi")})
+		return err
+	}
+	goodMD5 := md5Of([]byte("hi"))
+	for name, out := range map[string]string{
+		"null":         "null",
+		"empty":        "{}",
+		"no id":        `{"MD5OfBody":"` + goodMD5 + `"}`,
+		"wrong md5":    `{"MessageID":"m","MD5OfBody":"` + md5Of([]byte("other")) + `"}`,
+		"attrs md5":    `{"MessageID":"m","MD5OfBody":"` + goodMD5 + `","MD5OfMessageAttributes":"` + goodMD5 + `"}`,
+		"not an array": `{"MessageID":"m"`,
+		"no position":  `{"MessageID":"m","MD5OfBody":"` + goodMD5 + `","EnqueuedAt":"2026-10-31T00:00:00Z"}`,
+		"no enqueue":   `{"MessageID":"m","MD5OfBody":"` + goodMD5 + `","Position":7}`,
+	} {
+		if err := answer(t, out, send); !errors.Is(err, message.ErrUnavailable) {
+			t.Fatalf("send %s = %v", name, err)
+		}
+	}
+	const done = `,"Position":7,"EnqueuedAt":"2026-10-31T00:00:00Z"}`
+	if err := answer(t, `{"MessageID":"m","MD5OfBody":"`+goodMD5+`"`+done, send); err != nil {
+		t.Fatalf("complete send = %v", err)
+	}
+
+	const times = `,"SentAt":"2026-10-31T00:00:00Z","FirstReceive":"2026-10-31T00:00:00Z","Deadline":"2026-10-31T00:01:00Z","ReceiveCount":1`
+	full := `{"MessageID":"m","ReceiptHandle":"h","Body":"aGk=","MD5OfBody":"` + goodMD5 + `","SenderID":"AIDA"` + times + `}`
+	recv := func(r *Relay) error {
+		_, err := r.Receive(context.Background(), message.ReceiveInput{Account: acct, Region: region, Queue: "orders"})
+		return err
+	}
+	for name, out := range map[string]string{
+		"null":      "null",
+		"no handle": `[{"MessageID":"m","Body":"aGk=","MD5OfBody":"` + goodMD5 + `"}]`,
+		"bad md5":   `[{"MessageID":"m","ReceiptHandle":"h","Body":"aGk=","MD5OfBody":"` + md5Of(nil) + `"}]`,
+		"no sender": `[{"MessageID":"m","ReceiptHandle":"h","Body":"aGk=","MD5OfBody":"` + goodMD5 + `"` + times + `}]`,
+		"too many":  `[` + full + `,` + full + `]`,
+	} {
+		if err := answer(t, out, recv); !errors.Is(err, message.ErrUnavailable) {
+			t.Fatalf("receive %s = %v", name, err)
+		}
+	}
+
+	if err := answer(t, `[`+full+`]`, recv); err != nil {
+		t.Fatalf("complete receive = %v", err)
+	}
+
+	del := func(r *Relay) error {
+		return r.Delete(context.Background(), message.DeleteInput{Account: acct, Region: region, Queue: "orders", ReceiptHandle: "h"})
+	}
+	if err := answer(t, "null", del); !errors.Is(err, message.ErrUnavailable) {
+		t.Fatalf("delete null = %v", err)
+	}
+
+	resolve := func(r *Relay) error {
+		_, err := r.Resolve(context.Background(), acct, region, "orders")
+		return err
+	}
+	for name, out := range map[string]string{
+		"null":          "null",
+		"no queue id":   `{"account":"` + acct + `","region":"` + region + `","name":"orders"}`,
+		"other account": `{"queueId":"q1","account":"210987654321","region":"` + region + `","name":"orders"}`,
+		"other name":    `{"queueId":"q1","account":"` + acct + `","region":"` + region + `","name":"payments"}`,
+		"deleting":      `{"queueId":"q1","account":"` + acct + `","region":"` + region + `","name":"orders","state":"deleting"}`,
+	} {
+		if err := answer(t, out, resolve); !errors.Is(err, queue.ErrUnavailable) {
+			t.Fatalf("resolve %s = %v", name, err)
+		}
+	}
+}
+
+// Pending work is bounded by bytes as well as count: a call that would push
+// the held inputs past the bound fails at once and holds nothing.
+func TestCall_PendingBytesAreBounded(t *testing.T) {
+	r := &Relay{MaxPendingBytes: 64 << 10, CallLimit: time.Second}
+	present(r)
+	big := message.SendInput{Account: "123456789012", Region: "ap-southeast-2", Queue: "orders", Body: make([]byte, 40<<10)}
+	go func() { _, _ = r.Send(context.Background(), big) }()
+	first := takeOne(t, r, worker)
+	start := time.Now()
+	if _, err := r.Send(context.Background(), big); !errors.Is(err, ErrNoWorker) || time.Since(start) > 500*time.Millisecond {
+		t.Fatalf("second send = %v after %s", err, time.Since(start))
+	}
+	if _, err := r.Complete(worker, first.ID, Result{Error: encodeError(message.ErrUnavailable)}); err != nil {
+		t.Fatal(err)
+	}
+	r.mu.Lock()
+	held := r.bytes
+	r.mu.Unlock()
+	if held != 0 {
+		t.Fatalf("bytes held after completion = %d", held)
+	}
+}

spinifex/domains/sqs/relay/wire.go

@@ -0,0 +1,221 @@
+// Package relay carries SQS owner calls from the ingress to the SQS worker,
+// which pulls work over its instance-role channel and posts each result back;
+// no message payload crosses platform NATS.
+package relay
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/sqs/message"
+	"github.com/mulgadc/spinifex/spinifex/domains/sqs/queue"
+)
+
+// Operations the relay carries, one per owner method the adapter calls.
+const (
+	OpCreateQueue        = "CreateQueue"
+	OpResolveQueue       = "ResolveQueue"
+	OpListQueues         = "ListQueues"
+	OpSetQueueAttributes = "SetQueueAttributes"
+	OpTagQueue           = "TagQueue"
+	OpUntagQueue         = "UntagQueue"
+	OpListQueueTags      = "ListQueueTags"
+	OpSendMessage        = "SendMessage"
+	OpReceiveMessage     = "ReceiveMessage"
+	OpDeleteMessage      = "DeleteMessage"
+	OpChangeVisibility   = "ChangeMessageVisibility"
+)
+
+// Work is one owner call handed to the worker. Deadline is when the caller
+// stops waiting; a result posted after it is not delivered.
+type Work struct {
+	ID       string          `json:"id"`
+	Op       string          `json:"op"`
+	Deadline time.Time       `json:"deadline"`
+	Input    json.RawMessage `json:"input"`
+}
+
+// Result is the worker's answer to one Work: an output or an owner error.
+type Result struct {
+	Output json.RawMessage `json:"output,omitempty"`
+	Error  *WireError      `json:"error,omitempty"`
+}
+
+// WireError is an owner error by kind alone. Owner and backend text never
+// crosses the wire, since a wrapped error can carry a handle, body or secret.
+type WireError struct {
+	Kind string `json:"kind"`
+}
+
+// kinds lists every owner sentinel the adapter distinguishes with the fixed
+// text the ingress answers it with. The first match names the error, so
+// Unavailable comes first: an uncertain write is never reported as settled.
+var kinds = []struct {
+	name string
+	err  error
+	msg  string
+}{
+	{"queue.Unavailable", queue.ErrUnavailable, "The queue service is temporarily unavailable."},
+	{"message.Unavailable", message.ErrUnavailable, "The queue service is temporarily unavailable."},
+	{"queue.QueueDoesNotExist", queue.ErrQueueDoesNotExist, "The specified queue does not exist."},
+	{"queue.QueueNameExists", queue.ErrQueueNameExists, "A queue already exists with the same name and a different value for attribute(s)."},
+	{"queue.QueueDeletedRecently", queue.ErrQueueDeletedRecently, "You must wait 60 seconds after deleting a queue before you can create another with the same name."},
+	{"queue.InvalidIdentity", queue.ErrInvalidIdentity, "The request identity is not valid."},
+	{"queue.InvalidAttributeName", queue.ErrInvalidAttributeName, "An attribute name is not valid."},
+	{"queue.InvalidAttributeValue", queue.ErrInvalidAttributeValue, "An attribute value is not valid."},
+	{"queue.UnsupportedAttribute", queue.ErrUnsupportedAttribute, "An attribute is not supported."},
+	{"queue.InvalidQueueName", queue.ErrInvalidQueueName, "Can only include alphanumeric characters, hyphens, or underscores. 1 to 80 in length."},
+	{"queue.InvalidTag", queue.ErrInvalidTag, "A tag is not valid."},
+	{"queue.InvalidListRequest", queue.ErrInvalidListRequest, "The list request is not valid."},
+	{"message.QueueDoesNotExist", message.ErrQueueDoesNotExist, "The specified queue does not exist."},
+	{"message.BodyRequired", message.ErrBodyRequired, "The request must contain the parameter MessageBody."},
+	{"message.InvalidContents", message.ErrInvalidContents, "The message contains characters outside the allowed set."},
+	{"message.InvalidParameter", message.ErrInvalidParameter, "A parameter value is not valid."},
+	{"message.Unsupported", message.ErrUnsupported, "A parameter or operation is not supported."},
+	{"message.ReceiptInvalid", message.ErrReceiptInvalid, "The receipt handle is not valid."},
+	{"message.NotInflight", message.ErrNotInflight, "The message is not in flight."},
+	{"message.VisibilityCap", message.ErrVisibilityCap, "The total visibility timeout for the message is beyond the 12-hour limit."},
+}
+
+// errWorkerFault is a worker error that is not an owner sentinel; the
+// adapter answers it as an internal failure.
+var errWorkerFault = errors.New("relay: SQS worker failed the request")
+
+// encodeError names err by the first owner sentinel it wraps.
+func encodeError(err error) *WireError {
+	for _, k := range kinds {
+		if errors.Is(err, k.err) {
+			return &WireError{Kind: k.name}
+		}
+	}
+	return &WireError{Kind: "fault"}
+}
+
+// remoteError is a decoded owner error: fixed text that unwraps to the
+// owner's sentinel.
+type remoteError struct {
+	sentinel error
+	msg      string
+}
+
+func (e *remoteError) Error() string { return e.msg }
+func (e *remoteError) Unwrap() error { return e.sentinel }
+
+func decodeError(w *WireError) error {
+	for _, k := range kinds {
+		if k.name == w.Kind {
+			return &remoteError{sentinel: k.err, msg: k.msg}
+		}
+	}
+	return errWorkerFault
+}
+
+// marshal encodes without HTML escaping, so a body's worst-case growth is
+// the bound the receive budget assumes.
+func marshal(v any) ([]byte, error) {
+	var buf bytes.Buffer
+	enc := json.NewEncoder(&buf)
+	enc.SetEscapeHTML(false)
+	if err := enc.Encode(v); err != nil {
+		return nil, err
+	}
+	return bytes.TrimSuffix(buf.Bytes(), []byte("\n")), nil
+}
+
+// MarshalResult is the CompleteWork body for r.
+func MarshalResult(r Result) ([]byte, error) { return marshal(r) }
+
+// Wire forms of owner inputs and outputs that are not plain JSON values.
+
+type nameInput struct {
+	Account, Region, Name string
+}
+
+type attrsInput struct {
+	Account, Region, Name string
+	Attributes            map[string]string
+}
+
+type tagsInput struct {
+	Account, Region, Name string
+	Tags                  map[string]string
+}
+
+type untagInput struct {
+	Account, Region, Name string
+	Keys                  []string
+}
+
+// received is a message on the wire. Body and string values travel as
+// base64, so a message's encoded size is at most 4/3 of its SQS size plus
+// a bounded allowance, whatever characters it holds.
+type received struct {
+	MessageID              string
+	ReceiptHandle          string
+	Body                   []byte
+	MD5OfBody              string
+	Attributes             []wireAttribute
+	MD5OfMessageAttributes string
+	SenderID               string
+	SentAt                 time.Time
+	ReceiveCount           uint64
+	FirstReceive           time.Time
+	Deadline               time.Time
+}
+
+type wireAttribute struct {
+	Name        string
+	DataType    string
+	StringValue []byte `json:",omitempty"`
+	HasString   bool   `json:",omitempty"`
+	BinaryValue []byte `json:",omitempty"`
+}
+
+func toWireReceived(in []message.Received) []received {
+	out := make([]received, len(in))
+	for i, m := range in {
+		attrs := make([]wireAttribute, 0, m.Attributes.Len())
+		for _, a := range m.Attributes.All() {
+			w := wireAttribute{Name: a.Name, DataType: a.Value.DataType, BinaryValue: a.Value.BinaryValue}
+			if a.Value.StringValue != nil {
+				w.StringValue, w.HasString = []byte(*a.Value.StringValue), true
+			}
+			attrs = append(attrs, w)
+		}
+		out[i] = received{
+			MessageID: m.MessageID, ReceiptHandle: m.ReceiptHandle, Body: m.Body, MD5OfBody: m.MD5OfBody,
+			Attributes: attrs, MD5OfMessageAttributes: m.MD5OfMessageAttributes,
+			SenderID: m.SenderID, SentAt: m.SentAt, ReceiveCount: m.ReceiveCount,
+			FirstReceive: m.FirstReceive, Deadline: m.Deadline,
+		}
+	}
+	return out
+}
+
+func fromWireReceived(in []received) ([]message.Received, error) {
+	out := make([]message.Received, len(in))
+	for i, m := range in {
+		list := make([]message.Attribute, 0, len(m.Attributes))
+		for _, w := range m.Attributes {
+			v := message.AttributeValue{DataType: w.DataType, BinaryValue: w.BinaryValue}
+			if w.HasString {
+				s := string(w.StringValue)
+				v.StringValue = &s
+			}
+			list = append(list, message.Attribute{Name: w.Name, Value: v})
+		}
+		attrs, err := message.NewAttributes(list)
+		if err != nil {
+			return nil, err
+		}
+		out[i] = message.Received{
+			MessageID: m.MessageID, ReceiptHandle: m.ReceiptHandle, Body: m.Body, MD5OfBody: m.MD5OfBody,
+			Attributes: attrs, MD5OfMessageAttributes: m.MD5OfMessageAttributes,
+			SenderID: m.SenderID, SentAt: m.SentAt, ReceiveCount: m.ReceiveCount,
+			FirstReceive: m.FirstReceive, Deadline: m.Deadline,
+		}
+	}
+	return out, nil
+}

spinifex/domains/sqs/servicestate/servicestate.go

@@ -0,0 +1,172 @@
+// Package servicestate is the operator's durable per-Region choice of whether
+// SQS should serve. It is platform service configuration only: no queue or
+// message data is kept here, and an absent record means disabled.
+package servicestate
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"regexp"
+	"sync"
+	"time"
+
+	sqsjs "github.com/mulgadc/spinifex/spinifex/domains/sqs/message/jetstream"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+// Bucket is the SQS domain's KV bucket for per-Region desired state.
+const Bucket = "spinifex-sqs-service"
+
+// Desired is what the operator asked for, not what is running.
+type Desired string
+
+const (
+	Enabled  Desired = "enabled"
+	Disabled Desired = "disabled"
+)
+
+// Record is one Region's desired state. Generation rises on every change.
+type Record struct {
+	Region     string    `json:"region"`
+	Desired    Desired   `json:"desired"`
+	Node       string    `json:"node,omitempty"`
+	Generation uint64    `json:"generation"`
+	UpdatedAt  time.Time `json:"updatedAt"`
+}
+
+var (
+	// ErrNodeConflict is an enable naming a node other than the enabled one;
+	// the store and worker are pinned to the node that holds their volumes.
+	ErrNodeConflict = errors.New("SQS is enabled on another node")
+	// ErrConflict is a concurrent change that kept winning the write.
+	ErrConflict = errors.New("SQS service state changed concurrently")
+
+	regionPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,62}$`)
+	nodePattern   = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`)
+)
+
+// Store reads and writes the records.
+type Store struct{ KV jetstream.KeyValue }
+
+// Open opens the bucket, creating it when absent; the operator path uses it.
+func Open(ctx context.Context, js jetstream.JetStream) (Store, error) {
+	kv, err := sqsjs.OpenDurableKV(ctx, js, Bucket, "SQS service desired state per Region")
+	if err != nil {
+		return Store{}, err
+	}
+	return Store{KV: kv}, nil
+}
+
+// Bind binds the bucket without creating it.
+func Bind(ctx context.Context, js jetstream.JetStream) (Store, error) {
+	kv, err := sqsjs.BindDurableKV(ctx, js, Bucket)
+	if err != nil {
+		return Store{}, err
+	}
+	return Store{KV: kv}, nil
+}
+
+func key(region string) string { return "region." + region }
+
+// Get returns region's record, or a disabled one when none is stored.
+func (s Store) Get(ctx context.Context, region string) (Record, error) {
+	rec, _, err := s.get(ctx, region)
+	return rec, err
+}
+
+func (s Store) get(ctx context.Context, region string) (Record, uint64, error) {
+	if !regionPattern.MatchString(region) {
+		return Record{}, 0, errors.New("invalid region")
+	}
+	e, err := s.KV.Get(ctx, key(region))
+	if errors.Is(err, jetstream.ErrKeyNotFound) {
+		return Record{Region: region, Desired: Disabled}, 0, nil
+	}
+	if err != nil {
+		return Record{}, 0, err
+	}
+	var rec Record
+	if err := json.Unmarshal(e.Value(), &rec); err != nil || rec.Region != region ||
+		(rec.Desired != Enabled && rec.Desired != Disabled) {
+		return Record{}, 0, fmt.Errorf("SQS service record for %s is unreadable", region)
+	}
+	return rec, e.Revision(), nil
+}
+
+// Set records desired for region. Repeating the current state writes nothing.
+// Enabling names the node; disabling keeps it, since the volumes stay there.
+func (s Store) Set(ctx context.Context, region string, desired Desired, node string, now time.Time) (Record, error) {
+	if desired != Enabled && desired != Disabled {
+		return Record{}, errors.New("invalid desired state")
+	}
+	if desired == Enabled && !nodePattern.MatchString(node) {
+		return Record{}, errors.New("invalid node")
+	}
+	for range 5 {
+		cur, rev, err := s.get(ctx, region)
+		if err != nil {
+			return Record{}, err
+		}
+		if desired == Enabled && cur.Node != "" && cur.Node != node {
+			return cur, ErrNodeConflict
+		}
+		if cur.Desired == desired && (desired == Disabled || cur.Node == node) {
+			return cur, nil
+		}
+		next := Record{Region: region, Desired: desired, Node: cur.Node, Generation: cur.Generation + 1, UpdatedAt: now.UTC()}
+		if desired == Enabled {
+			next.Node = node
+		}
+		b, err := json.Marshal(next)
+		if err != nil {
+			return Record{}, err
+		}
+		if rev == 0 {
+			_, err = s.KV.Create(ctx, key(region), b)
+		} else {
+			_, err = s.KV.Update(ctx, key(region), b, rev)
+		}
+		if err == nil {
+			return next, nil
+		}
+		if !errors.Is(err, jetstream.ErrKeyExists) && !errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
+			return Record{}, err
+		}
+	}
+	return Record{}, ErrConflict
+}
+
+// Gate answers whether region's SQS should take tenant requests, rereading
+// the record at most every TTL. Any read failure answers no.
+type Gate struct {
+	JS     jetstream.JetStream
+	Region string
+	TTL    time.Duration
+	Now    func() time.Time
+
+	mu      sync.Mutex
+	checked time.Time
+	open    bool
+}
+
+// Open reports whether the record says enabled.
+func (g *Gate) Open(ctx context.Context) bool {
+	now := time.Now
+	if g.Now != nil {
+		now = g.Now
+	}
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	if !g.checked.IsZero() && now().Sub(g.checked) < g.TTL {
+		return g.open
+	}
+	g.open = false
+	if st, err := Bind(ctx, g.JS); err == nil {
+		rec, err := st.Get(ctx, g.Region)
+		g.open = err == nil && rec.Desired == Enabled
+	}
+	g.checked = now()
+	return g.open
+}

spinifex/domains/sqs/servicestate/servicestate_test.go

@@ -0,0 +1,78 @@
+package servicestate
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+)
+
+func TestSet_DefaultDisabledEnableIsIdempotentAndPinnedToItsNode(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	ctx := context.Background()
+	st, err := Open(ctx, js)
+	if err != nil {
+		t.Fatal(err)
+	}
+	now := time.Unix(1000, 0)
+
+	rec, err := st.Get(ctx, "ap-southeast-2")
+	if err != nil || rec.Desired != Disabled || rec.Generation != 0 {
+		t.Fatalf("absent record = %+v, %v; want disabled", rec, err)
+	}
+	on, err := st.Set(ctx, "ap-southeast-2", Enabled, "node1", now)
+	if err != nil || on.Desired != Enabled || on.Node != "node1" || on.Generation != 1 {
+		t.Fatalf("enable = %+v, %v", on, err)
+	}
+	again, err := st.Set(ctx, "ap-southeast-2", Enabled, "node1", now.Add(time.Hour))
+	if err != nil || again != on {
+		t.Fatalf("repeated enable = %+v, %v; want unchanged %+v", again, err, on)
+	}
+	if _, err := st.Set(ctx, "ap-southeast-2", Enabled, "node2", now); !errors.Is(err, ErrNodeConflict) {
+		t.Fatalf("enable on another node = %v; want ErrNodeConflict", err)
+	}
+
+	off, err := st.Set(ctx, "ap-southeast-2", Disabled, "", now)
+	if err != nil || off.Desired != Disabled || off.Node != "node1" || off.Generation != 2 {
+		t.Fatalf("disable = %+v, %v; want node kept", off, err)
+	}
+	if _, err := st.Set(ctx, "ap-southeast-2", Enabled, "node2", now); !errors.Is(err, ErrNodeConflict) {
+		t.Fatalf("re-enable elsewhere = %v; the volumes stay on node1", err)
+	}
+	if rec, _ := st.Get(ctx, "us-east-1"); rec.Desired != Disabled {
+		t.Fatalf("another Region = %+v; want disabled", rec)
+	}
+}
+
+func TestGate_FollowsTheRecordAndFailsClosed(t *testing.T) {
+	_, _, js := testutil.StartTestJetStream(t)
+	ctx := context.Background()
+	now := time.Unix(1000, 0)
+	g := &Gate{JS: js, Region: "ap-southeast-2", TTL: time.Second, Now: func() time.Time { return now }}
+	if g.Open(ctx) {
+		t.Fatal("gate open with no bucket")
+	}
+	st, err := Open(ctx, js)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := st.Set(ctx, "ap-southeast-2", Enabled, "node1", now); err != nil {
+		t.Fatal(err)
+	}
+	if g.Open(ctx) {
+		t.Fatal("gate reread before its TTL")
+	}
+	now = now.Add(time.Second)
+	if !g.Open(ctx) {
+		t.Fatal("gate closed after enable")
+	}
+	if _, err := st.Set(ctx, "ap-southeast-2", Disabled, "", now); err != nil {
+		t.Fatal(err)
+	}
+	now = now.Add(time.Second)
+	if g.Open(ctx) {
+		t.Fatal("gate open after disable")
+	}
+}

spinifex/domains/sqs/storelaunch/launcher.go

@@ -1,301 +0,0 @@
-package storelaunch
-
-import (
-	"context"
-	"crypto/rand"
-	"encoding/hex"
-	"errors"
-	"fmt"
-
-	"github.com/mulgadc/spinifex/spinifex/domains/sqs/firststore"
-	"github.com/mulgadc/spinifex/spinifex/domains/sqs/storevolume"
-	"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerident"
-	"github.com/mulgadc/spinifex/spinifex/domains/sqs/workerlaunch"
-	"github.com/nats-io/nats.go/jetstream"
-)
-
-// Volumes prepares the store's data volume and reads its own metadata.
-type Volumes interface {
-	// Prepare creates a new, empty, formatted volume and returns its ID. On
-	// failure it returns any volume it created that it could not discard.
-	Prepare(ctx context.Context, sizeGiB int64) (string, error)
-	// Encrypted reports the volume record's own encryption flag.
-	Encrypted(ctx context.Context, volumeID string) (bool, error)
-}
-
-// NetworkAllocator manages the store's one ENI in the worker's subnet. Find
-// only looks and returns nil when none is visible.
-type NetworkAllocator interface {
-	FindStoreNetwork(ctx context.Context, region, claimID string) (*workerlaunch.Network, error)
-	CreateStoreNetwork(ctx context.Context, region, claimID string) (*workerlaunch.Network, error)
-}
-
-// Spec is one store VM launch. ClientNKey is the worker's recorded public
-// key; nothing here is private.
-type Spec struct {
-	InstanceType string
-	Network      workerlaunch.Network
-	DataVolumeID string
-	ClientNKey   string
-}
-
-// Host reports whether this node can boot the store now. It allocates and
-// records nothing.
-type Host interface {
-	Ready(ctx context.Context) error
-}
-
-// VMLauncher starts the store VM on this node. Any error leaves the outcome
-// unknown to the caller.
-type VMLauncher interface {
-	LaunchStore(ctx context.Context, spec Spec) (string, error)
-}
-
-var (
-	ErrIncomplete     = errors.New("SQS store launcher is missing a capability")
-	ErrHostNotReady   = errors.New("this node cannot boot the SQS store yet")
-	ErrWorkerNotBound = errors.New("the Region's SQS worker is not bound on this node")
-	ErrKeyNotRecorded = errors.New("the SQS worker has not recorded its public key")
-	ErrWrongNode      = errors.New("the Region's SQS store is pinned to another node")
-	ErrUnencrypted    = errors.New("SQS store data volume is not encrypted")
-	// ErrVolumeNotCreated is a volume Prepare that sent no create; a retry may
-	// try again once its cause is fixed.
-	ErrVolumeNotCreated = errors.New("SQS store volume was not created")
-	// ErrOutcomeUnknown is returned for a dispatched launch whose result was
-	// not seen; it is never sent again until reconciled.
-	ErrOutcomeUnknown = errors.New("SQS store launch outcome is unknown; reconcile before relaunching")
-	// ErrVolumeOutcomeUnknown is returned for a volume create that was sent
-	// and not recorded; it is never sent again until reconciled.
-	ErrVolumeOutcomeUnknown = errors.New("SQS store volume create outcome is unknown; reconcile before relaunching")
-	// ErrNetworkOutcomeUnknown is returned for an ENI create that was sent
-	// with no ENI visible; it is never sent again until reconciled.
-	ErrNetworkOutcomeUnknown = errors.New("SQS store network create outcome is unknown; reconcile before relaunching")
-)
-
-// VolumeError reports a store volume this call knows of but could not use:
-// one a failed prepare left behind, or one that is not encrypted. It is kept.
-type VolumeError struct {
-	VolumeID string
-	Err      error
-}
-
-func (e *VolumeError) Error() string {
-	return fmt.Sprintf("SQS store data volume %s is kept: %v", e.VolumeID, e.Err)
-}
-
-func (e *VolumeError) Unwrap() error { return e.Err }
-
-// Launcher realizes the store on NodeID, the node the Region's worker is
-// bound to. The daemon never holds the worker's private key.
-type Launcher struct {
-	Records    Records
-	Workers    workerident.Store
-	FirstStore firststore.Store
-	Volumes    Volumes
-	Network    NetworkAllocator
-	Host       Host
-	VMs        VMLauncher
-	NodeID     string
-
-	// RetainedVolume extracts a volume ID from a Prepare error, if any.
-	RetainedVolume func(error) string
-	newClaimID     func() (string, error)
-}
-
-// Result is the Region's bound store. Bound is not ready.
-type Result struct {
-	InstanceID   string
-	Address      string
-	InstanceType string
-}
-
-// Ensure returns the Region's bound store or launches it. Every step whose
-// outcome can be lost is marked first, so a retry after a lost reply resumes
-// or fails closed and never creates a second volume, ENI or VM.
-func (l *Launcher) Ensure(ctx context.Context, region, instanceType string, sizeGiB int64) (Result, error) {
-	if l.Records.KV == nil || l.Workers.KV == nil || l.FirstStore.KV == nil || l.Volumes == nil ||
-		l.Network == nil || l.Host == nil || l.VMs == nil || l.NodeID == "" || instanceType == "" ||
-		sizeGiB < 1 || sizeGiB > MaxSizeGiB {
-		return Result{}, ErrIncomplete
-	}
-	rec, err := l.Records.Current(ctx, region)
-	switch {
-	case errors.Is(err, jetstream.ErrKeyNotFound):
-	case err != nil:
-		return Result{}, err
-	case rec.NodeID != l.NodeID:
-		return Result{}, ErrWrongNode
-	case rec.InstanceType != instanceType:
-		return Result{}, fmt.Errorf("%w: %s", ErrTypeChanged, rec.InstanceType)
-	case rec.SizeGiB != sizeGiB:
-		return Result{}, fmt.Errorf("%w: %d GiB", ErrSizeChanged, rec.SizeGiB)
-	case rec.State == StateBound:
-		return Result{InstanceID: rec.InstanceID, Address: rec.Address, InstanceType: rec.InstanceType}, nil
-	case rec.Dispatched:
-		return Result{}, ErrOutcomeUnknown
-	}
-	if err := l.Host.Ready(ctx); err != nil {
-		return Result{}, fmt.Errorf("%w: %w", ErrHostNotReady, err)
-	}
-	pub, err := l.recordedKey(ctx, region)
-	if err != nil {
-		return Result{}, err
-	}
-	claim := rec.ClaimID
-	if claim == "" {
-		if claim, err = l.claimID(); err != nil {
-			return Result{}, err
-		}
-	}
-	if err := l.Records.Begin(ctx, region, claim, l.NodeID, pub, instanceType, sizeGiB); err != nil {
-		return Result{}, err
-	}
-	if rec, err = l.Records.Current(ctx, region); err != nil {
-		return Result{}, err
-	}
-
-	volumeID, err := l.volume(ctx, region, rec)
-	if err != nil {
-		return Result{}, err
-	}
-	network, err := l.network(ctx, region, claim)
-	if err != nil {
-		return Result{}, err
-	}
-	if err := l.Records.MarkDispatched(ctx, region, claim); err != nil {
-		if errors.Is(err, ErrMarked) {
-			return Result{}, ErrOutcomeUnknown
-		}
-		return Result{}, err
-	}
-	instanceID, err := l.VMs.LaunchStore(ctx, Spec{InstanceType: instanceType, Network: *network,
-		DataVolumeID: volumeID, ClientNKey: pub})
-	if err != nil {
-		return Result{}, fmt.Errorf("%w: %w", ErrOutcomeUnknown, err)
-	}
-	if err := l.bind(ctx, region, claim, instanceID, network.ENIIP); err != nil {
-		return Result{}, fmt.Errorf("store %s launched but not bound: %w", instanceID, err)
-	}
-	return Result{InstanceID: instanceID, Address: network.ENIIP, InstanceType: instanceType}, nil
-}
-
-// recordedKey requires the worker bound on this node and returns the public
-// key it recorded, after recording the launch intent that confirms it.
-func (l *Launcher) recordedKey(ctx context.Context, region string) (string, error) {
-	w, err := l.Workers.Current(ctx, region)
-	if errors.Is(err, jetstream.ErrKeyNotFound) {
-		return "", ErrWorkerNotBound
-	}
-	if err != nil {
-		return "", err
-	}
-	if w.State != workerident.StateBound || w.NodeID != l.NodeID {
-		return "", ErrWorkerNotBound
-	}
-	id := firststore.Identity{InstanceID: w.InstanceID, ClaimID: w.ClaimID, Generation: w.Generation}
-	holder, pub, err := l.FirstStore.RecordedKey(ctx, region)
-	if errors.Is(err, firststore.ErrNotClaimed) || errors.Is(err, firststore.ErrKeyNotRecorded) {
-		return "", ErrKeyNotRecorded
-	}
-	if err != nil {
-		return "", err
-	}
-	if holder != id {
-		return "", fmt.Errorf("%w: the key was recorded by another worker identity", ErrKeyNotRecorded)
-	}
-	if err := l.FirstStore.RecordLaunchIntent(ctx, region, id, pub); err != nil {
-		return "", fmt.Errorf("record store launch intent: %w", err)
-	}
-	return pub, nil
-}
-
-// volume returns the recorded volume, or marks the record and prepares one.
-// Its encryption is read from the volume's own record before any use.
-func (l *Launcher) volume(ctx context.Context, region string, rec Record) (string, error) {
-	volumeID := rec.DataVolumeID
-	if volumeID == "" {
-		if err := l.Records.MarkVolumeAttempted(ctx, region, rec.ClaimID); err != nil {
-			if errors.Is(err, ErrMarked) {
-				return "", ErrVolumeOutcomeUnknown
-			}
-			return "", err
-		}
-		id, err := l.Volumes.Prepare(ctx, rec.SizeGiB)
-		if errors.Is(err, storevolume.ErrNotCreated) {
-			if cerr := l.Records.ClearVolumeAttempt(ctx, region, rec.ClaimID); cerr != nil {
-				return "", fmt.Errorf("%w: no volume was created, but its mark was not cleared: %w", ErrVolumeOutcomeUnknown, cerr)
-			}
-			return "", fmt.Errorf("%w: %w", ErrVolumeNotCreated, err)
-		}
-		if err != nil {
-			err = fmt.Errorf("%w: %w", ErrVolumeOutcomeUnknown, err)
-			if l.RetainedVolume != nil {
-				if kept := l.RetainedVolume(err); kept != "" {
-					return "", &VolumeError{VolumeID: kept, Err: err}
-				}
-			}
-			return "", err
-		}
-		if err := l.Records.RecordVolume(ctx, region, rec.ClaimID, id); err != nil {
-			return "", &VolumeError{VolumeID: id, Err: fmt.Errorf("%w: %w", ErrVolumeOutcomeUnknown, err)}
-		}
-		volumeID = id
-	}
-	encrypted, err := l.Volumes.Encrypted(ctx, volumeID)
-	if err != nil {
-		return "", fmt.Errorf("read store data volume %s: %w", volumeID, err)
-	}
-	if !encrypted {
-		return "", &VolumeError{VolumeID: volumeID, Err: ErrUnencrypted}
-	}
-	return volumeID, nil
-}
-
-// network reuses the claim's visible ENI, or marks the record and sends one
-// create. Once marked, an ENI that is not visible fails closed.
-func (l *Launcher) network(ctx context.Context, region, claim string) (*workerlaunch.Network, error) {
-	found, err := l.Network.FindStoreNetwork(ctx, region, claim)
-	if err != nil {
-		return nil, fmt.Errorf("store network: %w", err)
-	}
-	if found != nil {
-		return found, nil
-	}
-	if err := l.Records.MarkNetworkAttempted(ctx, region, claim); err != nil {
-		if errors.Is(err, ErrMarked) {
-			return nil, ErrNetworkOutcomeUnknown
-		}
-		return nil, fmt.Errorf("store network: %w", err)
-	}
-	created, err := l.Network.CreateStoreNetwork(ctx, region, claim)
-	if err != nil {
-		return nil, fmt.Errorf("%w: %w", ErrNetworkOutcomeUnknown, err)
-	}
-	return created, nil
-}
-
-// bind retries a lost reply with the same arguments, which Bind accepts as
-// idempotent.
-func (l *Launcher) bind(ctx context.Context, region, claim, instanceID, address string) error {
-	var err error
-	for range 3 {
-		if err = l.Records.Bind(ctx, region, claim, instanceID, address); err == nil {
-			return nil
-		}
-		if errors.Is(err, ErrSuperseded) || errors.Is(err, ErrConflict) {
-			return err
-		}
-	}
-	return err
-}
-
-func (l *Launcher) claimID() (string, error) {
-	if l.newClaimID != nil {
-		return l.newClaimID()
-	}
-	b := make([]byte, 12)
-	if _, err := rand.Read(b); err != nil {
-		return "", fmt.Errorf("mint claim ID: %w", err)
-	}
-	return "store-" + hex.EncodeToString(b), nil
-}