fix: say RemoveSystemImage deletes the UEFI store before config.json

The instance prune, the reaper after a terminate of an imaged instance, and
the reaper after a failed launch seeded from an image each leave the
image's store object in place.

.github/scripts/e2e-select.sh

@@ -1,148 +0,0 @@
-#!/usr/bin/env bash
-# e2e-select.sh push <before> <sha> | suites "<e2e-x ...>"
-# Prints the e2e gate's single_legs, multi_suites and multi_timeout outputs.
-# Exits non-zero on anything it cannot read, which fails the e2e status.
-set -euo pipefail
-
-SCRIPTS_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
-# shellcheck source=suite-sets.sh
-. "${SCRIPTS_DIR}/suite-sets.sh"
-
-# Provision, install and teardown around the suites; the floor is what the
-# jobs have always had, so a budget can only be raised from it.
-SETUP_ALLOWANCE_MIN=15
-FLOOR_MIN=60
-
-BASE_RUNNER=ci-single
-SERVICE_RUNNER=ci-pseudo
-
-log() { printf '[e2e-select] %s\n' "$*" >&2; }
-die() { log "error: $*"; exit 1; }
-
-# in_list <word> <list>
-in_list() {
-  local w
-  for w in $2; do [ "$w" = "$1" ] && return 0; done
-  return 1
-}
-
-# keep <list> <eligible>: the words of <list> that are in <eligible>, deduplicated.
-keep() {
-  local out="" s
-  for s in $1; do
-    in_list "$s" "$2" || continue
-    in_list "$s" "$out" || out="${out:+$out }$s"
-  done
-  printf '%s' "$out"
-}
-
-# budget <list>: max over the suites' own timeouts plus the setup allowance.
-# A sum would demand hours for a set that shares one job and never takes them.
-budget() {
-  local want="${FLOOR_MIN}" s t
-  for s in $1; do
-    t="$(e2e_suite_timeout "$s")"
-    t=$((${t%m} + SETUP_ALLOWANCE_MIN))
-    [ "$t" -gt "$want" ] && want="$t"
-  done
-  printf '%s' "$want"
-}
-
-prefixed() {
-  local out="" s
-  for s in $1; do out="${out:+$out }e2e-$s"; done
-  printf '%s' "$out"
-}
-
-# leg <name> <runner> <suites>: one matrix entry, or nothing for no suites.
-leg() {
-  [ -n "$3" ] || return 0
-  jq -nc --arg leg "$1" --arg runner "$2" --arg suites "$(prefixed "$3")" \
-    --argjson timeout "$(budget "$3")" \
-    '{leg: $leg, runner: $runner, suites: $suites, timeout: $timeout}'
-}
-
-# emit <base-single> <service-single> <multi>
-emit() {
-  local legs
-  legs=$( { leg base "${BASE_RUNNER}" "$1"; leg service "${SERVICE_RUNNER}" "$2"; } | jq -sc .)
-  log "base leg: ${1:-none}"
-  log "service leg: ${2:-none}"
-  log "multi: ${3:-none}"
-  printf 'single_legs=%s\n' "$legs"
-  printf 'multi_suites=%s\n' "$(prefixed "$3")"
-  printf 'multi_timeout=%s\n' "$(budget "$3")"
-}
-
-# select_for_paths <newline-separated paths> [all]: base set plus matches.
-# With "all", every mapped suite is selected, for a push with no baseline.
-select_for_paths() {
-  local matched="" p s
-  if [ "${2:-}" = all ]; then
-    matched=$(e2e_path_suites_all | tr '\n' ' ')
-  else
-    while IFS= read -r p; do
-      [ -n "$p" ] || continue
-      for s in $(e2e_suites_for_path "$p"); do
-        in_list "$s" "$matched" || { matched="${matched:+$matched }$s"; log "${p} -> ${s}"; }
-      done
-    done <<<"$1"
-  fi
-  # A matched suite runs where it is eligible, preferring one node, so route53
-  # goes to the service leg and lb to the multi job.
-  local extra_single="" extra_multi=""
-  for s in $matched; do
-    if in_list "$s" "${E2E_SUITES_SINGLE}"; then
-      extra_single="${extra_single:+$extra_single }$s"
-    elif in_list "$s" "${E2E_SUITES_MULTI}"; then
-      extra_multi="${extra_multi:+$extra_multi }$s"
-    else
-      die "E2E_PATH_SUITES names '${s}', which no topology can run"
-    fi
-  done
-  local base_single base_multi
-  base_single=$(keep "${E2E_SUITES_PUSH_BASE}" "${E2E_SUITES_SINGLE}")
-  base_multi=$(keep "${E2E_SUITES_PUSH_BASE}" "${E2E_SUITES_MULTI}")
-  # A matched suite already in the base leg runs there, not twice.
-  extra_single=$(for s in $extra_single; do in_list "$s" "$base_single" || printf '%s ' "$s"; done)
-  emit "$base_single" "$(keep "$extra_single" "${E2E_SUITES_SINGLE}")" \
-    "$(keep "$base_multi $extra_multi" "${E2E_SUITES_MULTI}")"
-}
-
-mode="${1:-}"
-case "$mode" in
-  push)
-    [ $# -eq 3 ] || die "usage: $0 push <before> <sha>"
-    before="$2" sha="$3"
-    git cat-file -e "${sha}^{commit}" 2>/dev/null || die "commit ${sha} is not in this checkout"
-    if [ -z "${before//0/}" ]; then
-      log "no previous commit for this push; selecting every mapped suite"
-      select_for_paths "" all
-      exit 0
-    fi
-    if ! git cat-file -e "${before}^{commit}" 2>/dev/null; then
-      git fetch -q --filter=blob:none origin "${before}" 2>/dev/null || true
-    fi
-    if ! git cat-file -e "${before}^{commit}" 2>/dev/null; then
-      log "previous commit ${before} is gone; selecting every mapped suite"
-      select_for_paths "" all
-      exit 0
-    fi
-    # A rename counts as both paths, so moving code out of a service selects it.
-    changed=$(git diff --no-renames --name-only "${before}" "${sha}") || die "git diff ${before} ${sha} failed"
-    log "$(printf '%s\n' "$changed" | grep -c . || true) paths changed ${before:0:9}..${sha:0:9}"
-    select_for_paths "$changed"
-    ;;
-  suites)
-    [ $# -eq 2 ] || die "usage: $0 suites \"<e2e-x ...>\""
-    if [ -z "${2// /}" ]; then
-      select_for_paths ""
-      exit 0
-    fi
-    requested=$(printf '%s' "$2" | sed 's/e2e-//g')
-    emit "$(keep "$requested" "${E2E_SUITES_SINGLE}")" "" "$(keep "$requested" "${E2E_SUITES_MULTI}")"
-    ;;
-  *)
-    die "usage: $0 push <before> <sha> | suites \"<list>\""
-    ;;
-esac

.github/scripts/mulga-design-check.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Warn when a PR body has no filled "Mulga design" line. Reads PR_BODY and
+# never fails the job: the check is a prompt, not a gate.
+set -euo pipefail
+shopt -s nocasematch
+
+section_re='^#{1,6}[[:space:]]*Mulga design[[:space:]]*$'
+line_re='^[[:space:]]*(Updated|Not needed)[[:space:]]*:[[:space:]]*(.*)$'
+placeholder_re='<[^<>]*>'
+
+filled() {
+  local body=$1 line value kept inside=0 pre rest
+  body=${body//$'\r'/}
+  while [[ $body == *'<!--'*'-->'* ]]; do
+    pre=${body%%'<!--'*}
+    rest=${body#*'<!--'}
+    body=$pre${rest#*'-->'}
+  done
+  while IFS= read -r line; do
+    if [[ $line =~ $section_re ]]; then
+      inside=1
+    elif ((inside)) && [[ ${line#"${line%%[![:space:]]*}"} == '#'* ]]; then
+      return 1
+    elif ((inside)) && [[ $line =~ $line_re ]]; then
+      value=${BASH_REMATCH[2]} kept=''
+      while [[ $value =~ $placeholder_re ]]; do
+        kept+=${value%%"${BASH_REMATCH[0]}"*}
+        value=${value#*"${BASH_REMATCH[0]}"}
+      done
+      [[ $kept$value == *[![:space:]]* ]] && return 0
+    fi
+  done <<<"$body"
+  return 1
+}
+
+if filled "${PR_BODY:-}"; then
+  echo "The Mulga design line is filled in."
+else
+  echo '::warning title=Mulga design::Add a "## Mulga design" section with one line: "Updated: <mulga PR>" or "Not needed: <reason>".'
+fi

.github/scripts/suite-sets.sh

@@ -66,67 +66,6 @@ E2E_SUITES_MULTI="multinode lb cert quota route53 kvquorum lbrecovery instancere
 E2E_SUITES_SINGLE_RE="$(printf '%s' "${E2E_SUITES_SINGLE}" | tr ' ' '|')"
 E2E_SUITES_MULTI_RE="$(printf '%s' "${E2E_SUITES_MULTI}" | tr ' ' '|')"
 
-# The push gate always runs this set. A push adds to it the suites whose code
-# it changed, by E2E_PATH_SUITES below; a dispatch names its own set instead.
-E2E_SUITES_PUSH_BASE="single multinode cert iam partialblock"
-
-# "suite prefix" pairs: a push changing a path under prefix adds suite. The
-# recovery and fault suites stay nightly, and shared code adds nothing because
-# the base set covers it. Image directories carry each guest's own scripts.
-E2E_PATH_SUITES="
-rds cmd/rds-agent/
-rds internal/rdsgw/
-rds spinifex/agents/rds/
-rds spinifex/domains/rds/
-rds tests/e2e/rds/
-rds scripts/images/rds-mariadb/
-rds scripts/images/rds-postgres/
-ecs cmd/ecs-agent/
-ecs contracts/ecs/
-ecs internal/ecsgw/
-ecs spinifex/agents/ecs/
-ecs spinifex/domains/ecs/
-ecs tests/e2e/ecs/
-ecs scripts/images/ecs-agent/
-eks cmd/eks-
-eks contracts/eks/
-eks internal/eksgw/
-eks spinifex/agents/eks/
-eks spinifex/domains/eks/
-eks tests/e2e/eks/
-eks scripts/images/eks-node/
-rds scripts/images/common/
-ecs scripts/images/common/
-eks scripts/images/common/
-rds scripts/build-system-image.sh
-ecs scripts/build-system-image.sh
-eks scripts/build-system-image.sh
-route53 spinifex/domains/route53/
-route53 tests/e2e/route53/
-lb cmd/lb-agent/
-lb spinifex/agents/elbv2/
-lb spinifex/domains/elbv2/
-lb tests/e2e/lb/
-"
-
-# Every suite E2E_PATH_SUITES can name, once each, in table order.
-#   e2e_path_suites_all
-e2e_path_suites_all() {
-  printf '%s\n' "${E2E_PATH_SUITES}" | awk 'NF == 2 && !seen[$1]++ { print $1 }'
-}
-
-# The suites a change to <path> triggers, one per line.
-#   e2e_suites_for_path <path>
-e2e_suites_for_path() {
-  local path="$1" suite prefix
-  while read -r suite prefix; do
-    [ -n "${prefix}" ] || continue
-    case "${path}" in
-      "${prefix}"*) printf '%s\n' "${suite}" ;;
-    esac
-  done <<<"${E2E_PATH_SUITES}"
-}
-
 # What every e2e-nightly permutation cell must prove, deliberately narrower
 # than the two sets above: those cells have a 35-minute budget and exist to show
 # that every install / network / host-OS combination boots and serves. The

.github/workflows/actions.lock

@@ -50,6 +50,8 @@ workflows:
     '.github/workflows/image-scripts.yml':
         - 'actions/checkout@v7.0.1'
         - 'actions/setup-go@v7.0.0'
+    '.github/workflows/mulga-design.yml':
+        - 'actions/checkout@v7.0.1'
     '.github/workflows/perf-3node.yml':
         - 'actions/checkout@v7.0.1'
         - 'actions/create-github-app-token@v3.3.0'

.github/workflows/e2e.yml

@@ -12,8 +12,6 @@ on:
       - go.sum
       - Makefile
       - tests/e2e/**
-      - scripts/images/**
-      - scripts/build-system-image.sh
       - .github/workflows/e2e.yml
   workflow_dispatch:
     inputs:
@@ -33,75 +31,88 @@ permissions:
 
 env:
   GIT_BRANCH: ${{ github.ref_name }}
+  # lb is out of the push default: it runs in the nightly multi cells and on
+  # dispatch (suites=e2e-lb), which keeps the multi job to multinode and cert.
+  SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-iam e2e-partialblock' }}
   TEST_RUN: ${{ inputs.test_run || '' }}
   SPINIFEX_OTLP_ENDPOINT: ${{ vars.SPINIFEX_OTLP_ENDPOINT || 'http://192.168.1.13:8200' }}
   SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }}
 
 jobs:
-  # A push runs the base set plus the suites its changed paths map to in
-  # suite-sets.sh; a dispatch runs its suites input, or the base set if empty.
-  # Budgets come from the suites' own timeouts there.
+  # The inner timeout must fire before the outer one. `go test` reports where it
+  # stopped and the suite's own failure; a job cancelled on timeout reports only
+  # "The operation was canceled" and takes the artifacts with it.
+  #
+  # A job budget cannot be a literal here because the suite set is a dispatch
+  # input: storagefault asks for 90m and diskperf for 180m, both eligible on one
+  # node, so any fixed number is either wrong for them or wasteful for everyone
+  # else. Resolving it from suite-sets.sh keeps the one source of truth and
+  # leaves the default push set at exactly the 60m it has always had.
   plan:
-    name: Select Suites
+    name: Resolve Job Budgets
     runs-on: ubuntu-26.04
     timeout-minutes: 5
     outputs:
-      single_legs: ${{ steps.select.outputs.single_legs }}
-      multi_suites: ${{ steps.select.outputs.multi_suites }}
-      multi_timeout: ${{ steps.select.outputs.multi_timeout }}
+      single_timeout: ${{ steps.budgets.outputs.single_timeout }}
+      multi_timeout: ${{ steps.budgets.outputs.multi_timeout }}
     steps:
-      # Full history without file contents: the diff needs both commits' trees.
       - uses: actions/checkout@v7.0.1
         with:
           path: spinifex
-          fetch-depth: 0
-          filter: blob:none
           sparse-checkout: .github/scripts
 
-      - name: Select suites
-        id: select
-        working-directory: spinifex
+      - name: Resolve Budgets
+        id: budgets
         env:
-          EVENT: ${{ github.event_name }}
-          BEFORE: ${{ github.event.before }}
-          IN_SUITES: ${{ inputs.suites }}
+          # Provision, install and teardown around the suites, plus the headroom
+          # a multi-suite job needs for whatever ran before the one that hangs.
+          SETUP_ALLOWANCE_MIN: 15
+          # What the job has always had. Kept as a floor so resolving the budget
+          # can only ever raise it, never quietly shorten an existing run.
+          FLOOR_MIN: 60
         run: |
-          set -euo pipefail
-          SELECT=.github/scripts/e2e-select.sh
-          if [ "${EVENT}" = push ]; then
-            bash "${SELECT}" push "${BEFORE}" "${GITHUB_SHA}" | tee -a "$GITHUB_OUTPUT"
-          else
-            bash "${SELECT}" suites "${IN_SUITES}" | tee -a "$GITHUB_OUTPUT"
-          fi
+          . "${GITHUB_WORKSPACE}/spinifex/.github/scripts/suite-sets.sh"
+          # Only max() is a sound bound: the suites share one job, so a sum would
+          # demand hours for a set that has never taken them. Max is what makes
+          # the hanging suite panic on its own timeout first, which is the point.
+          budget() {
+            local want="${FLOOR_MIN}" s t
+            for s in $(echo "${SUITES}" | sed 's/e2e-//g' | tr ' ' '\n' | grep -xE "$1"); do
+              t="$(e2e_suite_timeout "${s}")"
+              t=$(( ${t%m} + SETUP_ALLOWANCE_MIN ))
+              [ "${t}" -gt "${want}" ] && want="${t}"
+            done
+            echo "${want}"
+          }
+          single=$(budget "${E2E_SUITES_SINGLE_RE}")
+          multi=$(budget "${E2E_SUITES_MULTI_RE}")
+          echo "single_timeout=${single}" >> "$GITHUB_OUTPUT"
+          echo "multi_timeout=${multi}" >> "$GITHUB_OUTPUT"
+          echo "suites='${SUITES}' -> single ${single}m, multi ${multi}m"
 
   e2e_single:
-    # The base leg keeps the job and artifact names it always had: mulga's
-    # registry intake finds a run's results by them.
-    name: ${{ matrix.leg == 'base' && 'Single-Node E2E' || format('Single-Node E2E ({0})', matrix.leg) }}
-    # `base` runs on ci-single; `service`, on ci-pseudo, carries the matched
-    # single-node suites so a long one stays off the base leg's critical path.
-    if: needs.plan.outputs.single_legs != '[]'
+    name: Single-Node E2E
+    # The dispatch check gates on exactly the suites this job's Run step
+    # allow-lists, i.e. E2E_SUITES_SINGLE in .github/scripts/suite-sets.sh —
+    # an `if:` expression cannot source that file, so this one list is
+    # hand-mirrored and must be updated with it. Gating on a suite the job
+    # won't run — lb, which only executes in the multinode job — would
+    # provision a tofu VM and pay the Spinifex install only to run nothing and
+    # tear down. Push triggers keep running via the null-check on
+    # inputs.suites.
+    if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-single') || contains(github.event.inputs.suites, 'e2e-iam') || contains(github.event.inputs.suites, 'e2e-cert') || contains(github.event.inputs.suites, 'e2e-eks') || contains(github.event.inputs.suites, 'e2e-ecs') || contains(github.event.inputs.suites, 'e2e-storagegrowth') || contains(github.event.inputs.suites, 'e2e-partialblock') || contains(github.event.inputs.suites, 'e2e-rds')
     needs: plan
-    runs-on: [self-hosted, "${{ matrix.runner }}"]
-    timeout-minutes: ${{ matrix.timeout }}
-    strategy:
-      fail-fast: false
-      matrix:
-        include: ${{ fromJSON(needs.plan.outputs.single_legs) }}
+    runs-on: [self-hosted, ci-single]
+    timeout-minutes: ${{ fromJSON(needs.plan.outputs.single_timeout) }}
     env:
-      SUITES: ${{ matrix.suites }}
-      # Both legs can run on banksia at once, so every host path carries the leg.
-      TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}-${{ matrix.leg }}.log
-      ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}-${{ matrix.leg }}
-      # The base leg keeps the cell name it always had, so its history in
-      # Elastic and telemetry continues; any other leg appends its name.
-      E2E_CELL: ${{ matrix.leg == 'base' && 'single' || format('single-{0}', matrix.leg) }}
+      TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
+      ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}
       # Job-level env wins over the workflow-level default. This job and
       # e2e_multi share one run id and both name their hosts node1/node2/node3,
       # so without ci.cell their telemetry collapses into one bucket. This job
       # is where "Bootstrap via Binary Install" bakes SPINIFEX_OTEL_ATTRS into
       # the VM's telemetry.env, so the label has to be set here.
-      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=${{ matrix.leg == 'base' && 'single' || format('single-{0}', matrix.leg) }}
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=single
     steps:
       - name: Reap Orphan Procs
         run: |
@@ -171,7 +182,7 @@ jobs:
       - name: Resolve CI job id
         uses: $/.github/actions/otel-job-id
         with:
-          job-name: ${{ matrix.leg == 'base' && 'Single-Node E2E' || format('Single-Node E2E ({0})', matrix.leg) }}
+          job-name: Single-Node E2E
 
       - name: Build Install Artifacts
         working-directory: mulga/scripts/tofu-cluster
@@ -215,12 +226,6 @@ jobs:
           echo "wan_ip=${WAN_IP}" >> "$GITHUB_OUTPUT"
           echo "ssh_key=${SSH_KEY}" >> "$GITHUB_OUTPUT"
 
-      # build-system-image.sh reuses a raw image under ten minutes old from
-      # its build directory, and every runner on banksia shares the default
-      # one in /tmp, so a job could test another commit's guest agent.
-      - name: Use a per-job image build directory
-        run: echo "SYSTEM_IMAGE_BUILD_DIR=${RUNNER_TEMP}/system-image-build" >> "$GITHUB_ENV"
-
       # The eks suite's CreateCluster boots a control-plane VM from the
       # eks-node AMI, so that AMI must exist in the VM's image store before the
       # suite runs. build-system-image.sh customizes the image with libguestfs
@@ -244,7 +249,7 @@ jobs:
           group_open "make build-eks-node-image"
           make build-eks-node-image
           group_close
-          ok "raw: $(du -h "${SYSTEM_IMAGE_BUILD_DIR}/eks-node-image-build/eks-node-alpine.raw" | cut -f1)"
+          ok "raw: $(du -h /tmp/eks-node-image-build/eks-node-alpine.raw | cut -f1)"
 
       - name: Import eks-node AMI on VM
         if: contains(env.SUITES, 'e2e-eks')
@@ -256,16 +261,13 @@ jobs:
           SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
           VM=tf-user@${{ steps.vm.outputs.wan_ip }}
           banner "import eks-node AMI to VM image store"
-          # The VM's /tmp is a tmpfs and scp writes each raw at its full size,
-          # so every import removes its file before the next image arrives.
           scp "${SSH_OPTS[@]}" \
-            "${SYSTEM_IMAGE_BUILD_DIR}/eks-node-image-build/eks-node-alpine.raw" \
+            /tmp/eks-node-image-build/eks-node-alpine.raw \
             "$VM":/tmp/eks-node-alpine.raw
           ssh "${SSH_OPTS[@]}" "$VM" \
             "spx admin images import --file /tmp/eks-node-alpine.raw \
                --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
-               --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}; \
-               rc=\$?; rm -f /tmp/eks-node-alpine.raw; exit \$rc" >/dev/null 2>&1
+               --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}" >/dev/null 2>&1
           ok "eks-node AMI imported"
 
       # The eks data-plane subtest (TestEKS/KubectlGetNodes) drives the full
@@ -299,40 +301,6 @@ jobs:
           EOF
           ok "kubectl installed on VM"
 
-      # The ecs suite boots its container instance from the ecs-node AMI, found
-      # by its managed-by tag, so the tag matters. The build directory is named
-      # for the manifest's IMAGE_NAME (ecs-agent), not for the make target.
-      - name: Build ecs-node AMI (libguestfs, runner)
-        if: contains(env.SUITES, 'e2e-ecs')
-        working-directory: spinifex
-        run: |
-          set -euo pipefail
-          # direct backend = no libvirt. Set here rather than in the script:
-          # it has to be in the environment the build itself runs in.
-          export LIBGUESTFS_BACKEND=direct
-          "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ensure-libguestfs.sh"
-          make build-ecs-node-image
-
-      - name: Import ecs-node AMI on VM
-        if: contains(env.SUITES, 'e2e-ecs')
-        run: |
-          set -euo pipefail
-          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
-          eval "$(grep -E '^(ALPINE_VERSION|AMI_NAME|SYSTEM_TAG)=' \
-            spinifex/scripts/images/ecs-agent/manifest.conf)"
-          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
-          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
-          banner "import ecs-node AMI to VM image store"
-          scp "${SSH_OPTS[@]}" \
-            "${SYSTEM_IMAGE_BUILD_DIR}/ecs-agent-image-build/ecs-agent-alpine.raw" \
-            "$VM":/tmp/ecs-agent-alpine.raw
-          ssh "${SSH_OPTS[@]}" "$VM" \
-            "spx admin images import --file /tmp/ecs-agent-alpine.raw \
-               --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
-               --boot-mode bios --ami-name ${AMI_NAME} --tag ${SYSTEM_TAG}; \
-               rc=\$?; rm -f /tmp/ecs-agent-alpine.raw; exit \$rc" >/dev/null 2>&1
-          ok "ecs-node AMI imported"
-
       # CreateDBInstance boots the DB VM from an RDS AMI, so both engines' images
       # must be in the VM's image store before the suite runs — same split as the
       # eks-node pair above: build on the runner (needs source + network), import
@@ -355,7 +323,7 @@ jobs:
             group_open "make build-${IMAGE}-image"
             make "build-${IMAGE}-image"
             group_close
-            ok "raw: $(du -h "${SYSTEM_IMAGE_BUILD_DIR}/${IMAGE}-image-build/${IMAGE}-alpine.raw" | cut -f1)"
+            ok "raw: $(du -h "/tmp/${IMAGE}-image-build/${IMAGE}-alpine.raw" | cut -f1)"
           done
 
       - name: Import RDS AMIs on VM
@@ -377,13 +345,12 @@ jobs:
             for tag in ${SYSTEM_TAG}; do TAG_ARGS="${TAG_ARGS} --tag ${tag}"; done
             banner "import ${IMAGE} AMI to VM image store"
             scp "${SSH_OPTS[@]}" \
-              "${SYSTEM_IMAGE_BUILD_DIR}/${IMAGE}-image-build/${IMAGE}-alpine.raw" \
+              "/tmp/${IMAGE}-image-build/${IMAGE}-alpine.raw" \
               "$VM":"/tmp/${IMAGE}-alpine.raw"
             ssh "${SSH_OPTS[@]}" "$VM" \
               "spx admin images import --file /tmp/${IMAGE}-alpine.raw \
                  --distro alpine --version ${ALPINE_VERSION} --arch x86_64 \
-                 --boot-mode bios --ami-name ${AMI_NAME}${TAG_ARGS}; \
-                 rc=\$?; rm -f /tmp/${IMAGE}-alpine.raw; exit \$rc" >/dev/null 2>&1
+                 --boot-mode bios --ami-name ${AMI_NAME}${TAG_ARGS}" >/dev/null 2>&1
             ok "${IMAGE} AMI imported"
           done
 
@@ -561,7 +528,7 @@ jobs:
         uses: $/.github/actions/junit-to-es
         with:
           junit-glob: ${{ env.ARTIFACT_DIR }}/junit-*.xml
-          cell: ${{ env.E2E_CELL }}
+          cell: single
 
       - name: Analyze e2e failures
         if: always()
@@ -581,7 +548,7 @@ jobs:
       - uses: actions/upload-artifact@v7.0.2
         if: always()
         with:
-          name: ${{ matrix.leg == 'base' && format('e2e-artifacts-{0}', github.run_id) || format('e2e-artifacts-{0}-{1}', github.run_id, matrix.leg) }}
+          name: e2e-artifacts-${{ github.run_id }}
           path: ${{ env.ARTIFACT_DIR }}
           if-no-files-found: warn
           retention-days: 14
@@ -624,7 +591,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v7.0.2
         with:
-          name: ${{ matrix.leg == 'base' && format('tofu-log-single-{0}', github.run_id) || format('tofu-log-single-{0}-{1}', github.run_id, matrix.leg) }}
+          name: tofu-log-single-${{ github.run_id }}
           path: ${{ env.TOFU_LOG }}
           if-no-files-found: ignore
           retention-days: 14
@@ -640,13 +607,15 @@ jobs:
 
   e2e_multi:
     name: Multi-Node E2E
-    # Runs when the plan job selected any multi-node suite, and only then.
-    if: needs.plan.outputs.multi_suites != ''
+    # The dispatch check gates on exactly the suites this job's Run step
+    # allow-lists (multinode|lb|cert) so lb — which only runs here — still
+    # provisions the 3-node env when requested on its own, and the env stays
+    # unprovisioned when none of them are asked for.
+    if: github.event_name != 'workflow_dispatch' || contains(github.event.inputs.suites, 'e2e-multinode') || contains(github.event.inputs.suites, 'e2e-lb') || contains(github.event.inputs.suites, 'e2e-cert')
     needs: plan
     runs-on: [self-hosted, ci-multi]
     timeout-minutes: ${{ fromJSON(needs.plan.outputs.multi_timeout) }}
     env:
-      SUITES: ${{ needs.plan.outputs.multi_suites }}
       TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-multi-artifacts-${{ github.run_id }}
       # See the matching note on e2e_single: ci.cell is what separates this
@@ -1103,10 +1072,7 @@ jobs:
   e2e_status:
     name: Report E2E Status
     if: always()
-    # plan is a dependency because a failed plan skips both e2e jobs, and a
-    # skipped job counts as passed below: without it a broken selection would
-    # report green having run nothing.
-    needs: [plan, e2e_single, e2e_multi]
+    needs: [e2e_single, e2e_multi]
     runs-on: ubuntu-26.04
     timeout-minutes: 5
     steps:
@@ -1115,16 +1081,15 @@ jobs:
           GH_TOKEN: ${{ github.token }}
           REPORT_SHA: ${{ github.sha }}
         run: |
-          plan_ok=true; single_ok=true; multi_ok=true
-          [ "${{ needs.plan.result }}" = success ] || plan_ok=false
+          single_ok=true; multi_ok=true
           case "${{ needs.e2e_single.result }}" in success|skipped) ;; *) single_ok=false ;; esac
           case "${{ needs.e2e_multi.result }}" in success|skipped) ;; *) multi_ok=false ;; esac
-          if $plan_ok && $single_ok && $multi_ok; then
+          if $single_ok && $multi_ok; then
             STATE="success"
             DESC="E2E suites passed"
           else
             STATE="failure"
-            DESC="E2E failed: plan=${{ needs.plan.result }} single=${{ needs.e2e_single.result }} multi=${{ needs.e2e_multi.result }}"
+            DESC="E2E failed: single=${{ needs.e2e_single.result }} multi=${{ needs.e2e_multi.result }}"
           fi
           gh api "repos/${{ github.repository }}/statuses/${REPORT_SHA}" \
             -f state="$STATE" \
@@ -1141,7 +1106,7 @@ jobs:
   # status-report check.
   e2e_es_summary:
     name: ES CI Run Summary
-    needs: [plan, e2e_single, e2e_multi]
+    needs: [e2e_single, e2e_multi]
     if: always()
     runs-on: [self-hosted, ci-single]
     timeout-minutes: 5
@@ -1153,11 +1118,6 @@ jobs:
       # this host, so no credentials are needed for the POST.
       MULGA_ES_SINK: ${{ vars.MULGA_ES_SINK || '192.168.1.13:9200' }}
     steps:
-      # The workspace outlives the run on a self-hosted runner, and the status
-      # files are collected recursively, so an earlier run's would count here.
-      - name: Clear previous summary downloads
-        run: rm -rf "${GITHUB_WORKSPACE}/es-summary-single" "${GITHUB_WORKSPACE}/es-summary-multi"
-
       # Either leg may not have run (e.g. a workflow_dispatch that only
       # requested single-node suites), so a missing artifact is tolerated —
       # the aggregation script below treats an absent suite-status.txt as
@@ -1166,7 +1126,7 @@ jobs:
         uses: actions/download-artifact@v8.0.2
         continue-on-error: true
         with:
-          pattern: e2e-artifacts-${{ github.run_id }}*
+          name: e2e-artifacts-${{ github.run_id }}
           path: ${{ github.workspace }}/es-summary-single
 
       - name: Download multi-node artifacts
@@ -1186,7 +1146,6 @@ jobs:
           BRANCH: ${{ github.head_ref || github.ref_name }}
           SHA: ${{ github.sha }}
           SERVER_URL: ${{ github.server_url }}
-          PLAN_RESULT: ${{ needs.plan.result }}
           SINGLE_RESULT: ${{ needs.e2e_single.result }}
           MULTI_RESULT: ${{ needs.e2e_multi.result }}
           SINGLE_DIR: ${{ github.workspace }}/es-summary-single
@@ -1194,8 +1153,7 @@ jobs:
         run: |
           set -euo pipefail
 
-          # One subdirectory per single-node leg's artifact.
-          SINGLE_STATUS=$(find "${SINGLE_DIR}" -name suite-status.txt 2>/dev/null | sort || true)
+          SINGLE_STATUS="${SINGLE_DIR}/suite-status.txt"
           MULTI_STATUS="${MULTI_DIR}/suite-status.txt"
 
           # Collect "<suite>: FAILED" lines from both legs' suite-status.txt
@@ -1204,9 +1162,7 @@ jobs:
           # rather than a bash array so this stays correct under `set -u`
           # even when nothing failed.
           FAILED_SUITES=""
-          # SINGLE_STATUS is a newline-separated list, so it is split here.
-          # shellcheck disable=SC2086
-          for f in ${SINGLE_STATUS} "$MULTI_STATUS"; do
+          for f in "$SINGLE_STATUS" "$MULTI_STATUS"; do
             [ -f "$f" ] || continue
             while IFS= read -r line; do
               case "$line" in
@@ -1226,7 +1182,7 @@ jobs:
           case "$MULTI_RESULT" in success | skipped) ;; *) multi_ok=false ;; esac
 
           STATE=success
-          if [ "$PLAN_RESULT" != success ] || [ "$single_ok" != true ] || [ "$multi_ok" != true ] || [ "$FAILED_COUNT" -gt 0 ]; then
+          if [ "$single_ok" != true ] || [ "$multi_ok" != true ] || [ "$FAILED_COUNT" -gt 0 ]; then
             STATE=failure
           fi
 

.github/workflows/mulga-design.yml

@@ -0,0 +1,32 @@
+name: Mulga design
+
+# Reminds the author to say whether the Mulga design records need to change.
+# It reads only the pull request body, uses no secrets, and warns without ever
+# failing: the check is a prompt, not a gate. Fork PRs are skipped because their
+# authors cannot see the design records.
+
+on:
+  pull_request:
+    branches:
+      - dev
+    types: [opened, edited, reopened]
+
+permissions:
+  contents: read
+
+jobs:
+  mulga_design:
+    name: Mulga design line
+    if: ${{ !endsWith(github.actor, '[bot]') && github.event.pull_request.head.repo.full_name == github.repository }}
+    runs-on: ubuntu-26.04
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v7.0.1
+        with:
+          sparse-checkout: .github/scripts
+
+      - name: Check the Mulga design line
+        env:
+          PR_BODY: ${{ github.event.pull_request.body }}
+        working-directory: .github/scripts
+        run: bash mulga-design-check.sh

.github/workflows/spinifex.yml

@@ -160,7 +160,6 @@ jobs:
 
   race_detection:
     name: Race Detection
-    if: ${{ github.event_name != 'pull_request' }}
     runs-on: ubuntu-26.04
 
     steps:
@@ -364,10 +363,6 @@ jobs:
         if: ${{ !cancelled() }}
         run: make -C spinifex test-integration
 
-      - name: Run Gated Suites
-        if: ${{ !cancelled() }}
-        run: make -C spinifex test-gated GOTESTFLAGS=-count=1
-
       # Keeps what this run read or wrote and drops the rest, so the saved entry
       # is one run's working set however often dev is pushed. A red run may have
       # stopped early and used only part of the set, so it saves untrimmed.

Makefile

@@ -309,8 +309,8 @@ test-build-scripts:
 	done
 	$(_Q)shellcheck -S warning .github/scripts/hypervisor-disk-preflight.sh \
 		.github/scripts/wait-for-memory.sh \
-		.github/scripts/go-build-cache.sh \
-		.github/scripts/e2e-select.sh
+		.github/scripts/mulga-design-check.sh \
+		.github/scripts/go-build-cache.sh
 	@$(_HDR) "\n....Running shellcheck over build/scripts/**/*.sh + the node scripts...."
 	$(_Q)shellcheck -S warning $$(find build/scripts -name '*.sh' | sort) $(NODE_SCRIPTS)
 	$(_Q)shellcheck -S warning -x scripts/build-oracle-appliance.sh $$(find scripts/appliance -name '*.sh' | sort)
@@ -345,16 +345,6 @@ test-integration:
 	@$(MAKE) --no-print-directory generate-aws-model-coverage
 	@echo "AWS operation coverage: $(AWS_MODEL_OPERATION_COVERAGE_DIR)"
 
-# Suites the unit tiers skip: the three-node SQS R3 scenarios behind
-# SPINIFEX_SQS_R3 and the lambdatestlink-tagged tests. CI runs these on every PR.
-test-gated:
-	@echo -e "\n....Running env- and tag-gated suites...."
-	$(_Q)LOG_IGNORE=1 SPINIFEX_SQS_R3=1 go test $(GOTESTFLAGS) -timeout 180s -run '^TestR3$$' ./spinifex/domains/sqs/message/jetstream/ $(_RACEQ)
-	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -tags=lambdatestlink -timeout 120s ./cmd/lambda-agent/ ./spinifex/agents/lambda/hostlink/ ./spinifex/runtime/compute/lambda/guestconn/ ./spinifex/runtime/compute/lambda/testlink/ ./spinifex/domains/network/host/ $(_RACEQ)
-	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -tags=lambdatestlink -timeout 120s -run '^TestLambdaTestLink(Executor|Reply)_' ./spinifex/runtime/roles/awsgw/ $(_RACEQ)
-	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -tags=lambdatestlink,lambdatestlinkseam -timeout 120s -run '^TestLambdaTestLinkSeam_' ./spinifex/runtime/roles/awsgw/ $(_RACEQ)
-	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -tags=lambdatestlink -timeout 120s -run '^TestLambdaTestLink' ./spinifex/runtime/daemon/ $(_RACEQ)
-
 generate-aws-model-coverage:
 	@go run ./cmd/aws-model-coverage -out $(AWS_MODEL_OPERATION_COVERAGE_DIR) -json $(AWS_MODEL_OPERATION_COVERAGE_DIR)/coverage.json
 
@@ -390,7 +380,11 @@ test-cover:
 # Run unit tests with race detector
 test-race:
 	@echo -e "\n....Running tests with race detector for $(GO_PROJECT_NAME)...."
-	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... ./contracts/... ./architecture/... $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 SPINIFEX_SQS_R3=1 go test $(GOTESTFLAGS) -race -timeout 300s ./spinifex/... ./cmd/... ./internal/... ./contracts/... ./architecture/... $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s ./cmd/lambda-agent/ ./spinifex/agents/lambda/hostlink/ ./spinifex/runtime/compute/lambda/testlink/ ./spinifex/domains/network/host/ $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s -run '^TestLambdaTestLink(Executor|Reply)_' ./spinifex/runtime/roles/awsgw/ $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink,lambdatestlinkseam -timeout 120s -run '^TestLambdaTestLinkSeam_' ./spinifex/runtime/roles/awsgw/ $(_RACEQ)
+	$(_Q)LOG_IGNORE=1 go test $(GOTESTFLAGS) -race -tags=lambdatestlink -timeout 120s -run '^TestLambdaTestLink' ./spinifex/runtime/daemon/ $(_RACEQ)
 
 # Unit tests for in-repo GitHub Actions (e.g. .github/actions/e2e-analyze).
 # Kept out of `test-cover` so coverage % isn't diluted by CI-only tooling.
@@ -601,7 +595,7 @@ distro-arm64:
 distro-clean:
 	rm -rf dist/
 
-.PHONY: test-package-check build build-lambda-testlink build-lambda-testlink-image build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-worker-microvm-image build-lambda-runtime-microvm-image install-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration test-gated generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
+.PHONY: test-package-check build build-lambda-testlink build-lambda-testlink-image build-ui test-ui-e2e ui-e2e-container ui-baselines build-installer build-lb-agent build-ecs-agent build-system-image build-eks-node-image import-eks-node-image publish-eks-node-image build-ecs-node-image import-ecs-node-image build-rds-postgres-image import-rds-postgres-image build-rds-mariadb-image import-rds-mariadb-image build-microvm-image build-sqs-worker-microvm-image build-lambda-runtime-microvm-image install-microvm install-sqs-worker-microvm go_build preflight test test-cover test-race diff-coverage bench fuzz test-actions test-images test-build-scripts test-harness test-integration generate-aws-model-coverage aws-model-coverage test-segscan-oracle \
 	deploy reinstall clean \
 	install-system install-go install-aws quickinstall \
 	lint fix govulncheck nilaway \

cmd/lambda-agent/start_testlink.go

@@ -79,12 +79,6 @@ func (t testlink) run(parent context.Context) (int, error) {
 	if err := checkBinding(scope, grant, first, time.Now()); err != nil {
 		return exitRefused, err
 	}
-	// A host that sent no credential is refused before anything is installed.
-	if first.Credential == nil || !first.Credential.Complete() {
-		return exitRefused, errors.New("first frame carries no execution credential")
-	}
-	cred := *first.Credential
-	first.Credential = nil
 	if err := t.pkg.install(spec); err != nil {
 		return exitRefused, fmt.Errorf("install package: %w", err)
 	}
@@ -100,7 +94,12 @@ func (t testlink) run(parent context.Context) (int, error) {
 
 	ctx, cancel := context.WithCancelCause(parent)
 	defer cancel(nil)
-	cmd, err := t.startBootstrap(ctx, bootstrapEnv(ln.Addr().String(), spec.handler, cred))
+	cmd, err := t.startBootstrap(ctx, []string{
+		"AWS_LAMBDA_RUNTIME_API=" + ln.Addr().String(),
+		"_HANDLER=" + spec.handler,
+		"LAMBDA_TASK_ROOT=" + taskRoot,
+		"PATH=/bin:/sbin:/usr/bin",
+	})
 	if err != nil {
 		return exitLink, fmt.Errorf("start bootstrap: %w", err)
 	}
@@ -124,20 +123,6 @@ func (t testlink) run(parent context.Context) (int, error) {
 	return exitClosed, nil
 }
 
-// bootstrapEnv is the bootstrap's whole environment: the runtime variables and
-// the execution role's three credential variables.
-func bootstrapEnv(runtimeAPI, handler string, cred hostlink.Credential) []string {
-	return []string{
-		"AWS_LAMBDA_RUNTIME_API=" + runtimeAPI,
-		"_HANDLER=" + handler,
-		"LAMBDA_TASK_ROOT=" + taskRoot,
-		"PATH=/bin:/sbin:/usr/bin",
-		"AWS_ACCESS_KEY_ID=" + cred.AccessKeyID,
-		"AWS_SECRET_ACCESS_KEY=" + cred.SecretAccessKey,
-		"AWS_SESSION_TOKEN=" + cred.SessionToken,
-	}
-}
-
 // shutdownGrace is how long past the invocation deadline the agent waits for
 // the host's shutdown frame once its result is written.
 const shutdownGrace = 2 * time.Second

cmd/lambda-agent/start_testlink_test.go

@@ -13,7 +13,6 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strconv"
-	"strings"
 	"testing"
 	"time"
 
@@ -70,8 +69,6 @@ type bootstrapStarts struct {
 	sealed  bool
 	// handlerOut receives the _HANDLER the bootstrap process itself saw.
 	handlerOut string
-	// env is the environment the agent gave the bootstrap.
-	env []string
 }
 
 // testlink builds a guest whose fw_cfg also carries a controlled package,
@@ -105,7 +102,6 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 		runtimeAddr: "127.0.0.1:0",
 		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
 			b.n++
-			b.env = env
 			b.started <- struct{}{}
 			cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestHelperBootstrap$")
 			cmd.Env = append(env, helperEnv+"=1", helperHandlerOut+"="+b.handlerOut)
@@ -144,7 +140,7 @@ func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
 	case <-time.After(100 * time.Millisecond):
 	}
 	hs := hostlink.NewTestStream(conn)
-	req := hostlink.Request{Credential: &testCredential, GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: []byte(`{"a":40,"b":2}`)}
+	req := hostlink.Request{GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: []byte(`{"a":40,"b":2}`)}
 	require.NoError(t, hs.HostSend(req))
 	got, err := hs.HostResult(ctx)
 	require.NoError(t, err)
@@ -168,55 +164,6 @@ func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
 	assert.Equal(t, testHandler, string(seen), "bootstrap process not given the configured handler")
 	assert.True(t, b.sealed, "task dir not made read-only")
 	assert.FileExists(t, filepath.Join(b.task, "lib", "data.txt"))
-	assert.ElementsMatch(t, []string{
-		"AWS_LAMBDA_RUNTIME_API=" + envValue(b.env, "AWS_LAMBDA_RUNTIME_API"),
-		"_HANDLER=" + testHandler,
-		"LAMBDA_TASK_ROOT=" + taskRoot,
-		"PATH=/bin:/sbin:/usr/bin",
-		"AWS_ACCESS_KEY_ID=" + testCredential.AccessKeyID,
-		"AWS_SECRET_ACCESS_KEY=" + testCredential.SecretAccessKey,
-		"AWS_SESSION_TOKEN=" + testCredential.SessionToken,
-	}, b.env, "the bootstrap gets exactly the runtime and credential variables")
-	assert.NotEmpty(t, envValue(b.env, "AWS_LAMBDA_RUNTIME_API"))
-}
-
-var testCredential = hostlink.Credential{AccessKeyID: "ASIAGUESTKEY", SecretAccessKey: "guest-secret", SessionToken: "guest-token"}
-
-func envValue(env []string, name string) string {
-	for _, kv := range env {
-		if v, ok := strings.CutPrefix(kv, name+"="); ok {
-			return v
-		}
-	}
-	return ""
-}
-
-// A first frame without a complete credential installs and runs nothing.
-func TestTestlinkFrameWithoutCredentialNeverStartsBootstrap(t *testing.T) {
-	for name, cred := range map[string]*hostlink.Credential{
-		"none":       nil,
-		"incomplete": {AccessKeyID: "ASIAGUESTKEY"},
-	} {
-		t.Run(name, func(t *testing.T) {
-			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-			defer cancel()
-			h := newHost(t)
-			h.accept(ctx)
-			var b bootstrapStarts
-			done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
-			conn := h.guest(t)
-			defer conn.Close()
-			require.NoError(t, hostlink.NewTestStream(conn).HostSend(hostlink.Request{Credential: cred, GrantID: "g-1", AttemptID: "a-1",
-				RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: []byte(`{}`)}))
-
-			res := wait(t, done, "agent to stop")
-			assert.Equal(t, exitRefused, res.code)
-			assert.ErrorContains(t, res.err, "no execution credential")
-			assert.Zero(t, b.n)
-			assert.False(t, b.sealed)
-			assert.NoFileExists(t, filepath.Join(b.task, "bootstrap"), "package installed")
-		})
-	}
 }
 
 // A bootstrap that posts its response and exits at once does not cost the
@@ -238,7 +185,7 @@ func TestTestlinkBootstrapExitingAfterResponseStillDelivers(t *testing.T) {
 	defer conn.Close()
 	hs := hostlink.NewTestStream(conn)
 	payload := bytes.Repeat([]byte("0123456789"), 300_000)
-	req := hostlink.Request{Credential: &testCredential, GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: payload}
+	req := hostlink.Request{GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: payload}
 	require.NoError(t, hs.HostSend(req))
 	got, err := hs.HostResult(ctx)
 	require.NoError(t, err)
@@ -393,7 +340,7 @@ func TestTestlinkBadPackageNeverStartsBootstrap(t *testing.T) {
 
 	conn := h.guest(t)
 	defer conn.Close()
-	require.NoError(t, hostlink.NewTestStream(conn).HostSend(hostlink.Request{Credential: &testCredential, GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1",
+	require.NoError(t, hostlink.NewTestStream(conn).HostSend(hostlink.Request{GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1",
 		DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli()}))
 	res := wait(t, done, "agent to stop")
 	assert.Equal(t, exitRefused, res.code)
@@ -458,7 +405,7 @@ func TestTestlinkUnboundRequestNeverStartsBootstrap(t *testing.T) {
 			case tc.deadlineMs > 0:
 				deadline = tc.deadlineMs
 			}
-			req := hostlink.Request{Credential: &testCredential, GrantID: tc.grantID, AttemptID: tc.attempt, RequestID: "r-1",
+			req := hostlink.Request{GrantID: tc.grantID, AttemptID: tc.attempt, RequestID: "r-1",
 				DeadlineMs: deadline, Payload: []byte(`{}`)}
 			require.NoError(t, hostlink.NewTestStream(conn).HostSend(req))
 

contracts/cluster/v1/heartbeat.go

@@ -1,6 +1,9 @@
 package clusterv1
 
-import "time"
+import (
+	"encoding/json"
+	"time"
+)
 
 // ClusterStateBucket is the KV bucket holding each daemon's heartbeat alongside
 // other cluster-wide state.
@@ -11,6 +14,27 @@ func HeartbeatKey(node string) string {
 	return "heartbeat." + node
 }
 
+// LivenessWindow is how recently a node must have heartbeated for a reader
+// deciding on a takeover to count it live. It is six heartbeats, compared
+// across node clocks, where NTP skew is small against it.
+const LivenessWindow = 60 * time.Second
+
+// LiveAt reports whether a heartbeat written at written still vouches for its
+// node at now: it does while it is under LivenessWindow old.
+func LiveAt(written, now time.Time) bool {
+	return now.Sub(written) < LivenessWindow
+}
+
+// HeartbeatWritten dates a stored heartbeat value. One that cannot be decoded
+// or dated vouches for nothing, which is an answer rather than a fault.
+func HeartbeatWritten(data []byte) (time.Time, bool) {
+	var beat HeartbeatTimestamp
+	if json.Unmarshal(data, &beat) != nil {
+		return time.Time{}, false
+	}
+	return beat.Time()
+}
+
 // HeartbeatTimestamp is the read side of a heartbeat for node liveness. It
 // decodes only the timestamp, so a reader does not depend on the capacity
 // figures the daemon writes beside it.

contracts/cluster/v1/heartbeat_test.go

@@ -33,3 +33,23 @@ func TestHeartbeatTimestamp_UndatedBeatVouchesForNothing(t *testing.T) {
 		assert.False(t, ok, "timestamp %q", ts)
 	}
 }
+
+// A beat is live strictly under the window, so one exactly the window old is
+// the first a takeover may treat as stopped.
+func TestLiveAt_IsStrictlyUnderTheWindow(t *testing.T) {
+	now := time.Date(2026, 11, 5, 12, 0, 0, 0, time.UTC)
+	assert.True(t, clusterv1.LiveAt(now.Add(-clusterv1.LivenessWindow+time.Nanosecond), now))
+	assert.False(t, clusterv1.LiveAt(now.Add(-clusterv1.LivenessWindow), now))
+	assert.True(t, clusterv1.LiveAt(now.Add(time.Second), now), "a beat from a clock running ahead is live")
+}
+
+func TestHeartbeatWritten_DatesAStoredBeatAndRefusesAnUndecodableOne(t *testing.T) {
+	written, ok := clusterv1.HeartbeatWritten([]byte(`{"node":"node-a","timestamp":"2026-10-30T01:02:03Z"}`))
+	require.True(t, ok)
+	assert.True(t, written.Equal(time.Date(2026, 10, 30, 1, 2, 3, 0, time.UTC)))
+
+	for _, raw := range []string{"not json", `{"node":"node-a"}`} {
+		_, ok := clusterv1.HeartbeatWritten([]byte(raw))
+		assert.False(t, ok, raw)
+	}
+}

contracts/lambda/v1/README.md

@@ -21,7 +21,7 @@ Tenant code may be able to reach the listener, but it cannot open the agent's co
 **Authentication:** mutual TLS 1.3, bound to the environment ID and realization generation, using material the user runtime cannot read (ADR-0009:S17).
 For each generation the executor creates ephemeral self-signed host and guest certificates whose single URI SAN is `spinifex-lambda://<Region>/<account>/<function ID>/<environment ID>/<generation>/<role>`, with role `host` or `guest`.
 Each end validates the peer certificate against the one pinned peer certificate and separately requires that URI SAN with the opposite role; hostname verification is not relied on.
-ALPN carries the L-GUEST major version, `spinifex-lambda-guest/1`, and an unsupported version fails the handshake. Builds with the `lambdatestlink` tag offer only `spinifex-lambda-testlink/2` on both host and guest, whose first frame carries the execution-role credential, so a test-link end and a release end fail the handshake; release builds are unchanged.
+ALPN carries the L-GUEST major version, `spinifex-lambda-guest/1`, and an unsupported version fails the handshake.
 Under TLS 1.3 the guest's handshake can complete before the host refuses its certificate, so the guest treats the connection as accepted only after the host's first message.
 
 **Bootstrap material:** the executor passes these fw_cfg entries under `opt/spinifex/lambda/`, and the agent reads them as root and unloads `qemu_fw_cfg` before user code starts.

docs/COMMANDS.md

@@ -19,7 +19,7 @@ Every service is `PartOf=spinifex.target` and ordered by its own `After=`/`Wants
 | `spinifex-predastore.service` | `spx service predastore start` | S3-compatible object storage, serving that host's nodes and the S3 gate among them |
 | `spinifex-northstar.service` | `spx service northstar start` | Authoritative DNS for `*.spx3.net`, recursive via upstream forwarders, syncing zones from its S3 bucket |
 | `spinifex-viperblock.service` | `spx service viperblock start` | Block storage with the NBD plugin. `KillMode=process`: nbdkit serving a guest survives a restart of this unit |
-| `spinifex-daemon.service` | `spx service spinifex start` | VM orchestration, NATS subscriptions, health endpoint. Two `ExecStartPre` steps run first: `spx admin hostnet check`, and `spx admin node-facts collect`, which runs as root and whose failure does not stop the daemon. `KillMode=process`: guest QEMU survives a restart and the new daemon reattaches. `SIGHUP` reloads config |
+| `spinifex-daemon.service` | `spx service spinifex start` | VM orchestration, NATS subscriptions, health endpoint. `KillMode=process`: guest QEMU survives a restart and the new daemon reattaches. `SIGHUP` reloads config |
 | `spinifex-awsgw.service` | `spx service awsgw start` | AWS-compatible gateway: SigV4 auth, IAM policy enforcement, TLS. Also serves the Private Admin API |
 | `spinifex-vpcd.service` | `spx service vpcd start` | VPC daemon: translates `vpc.*` NATS events to OVN logical switches, ports and routers |
 | `spinifex-ui.service` | `spx service spinifex-ui start` | Web UI serving the React frontend |
@@ -129,7 +129,6 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 |---------|-------|-------------|
 | `spx admin preflight` | `--quiet` (print only non-OK rows) | Reports whether this host has the privileged helper scripts and sudoers grants the installed build depends on. `spx version` knows the build but not the host: `setup.sh` and `setup-ovn.sh` install those assets, and a binary-swap deploy (`make deploy`, or a manual copy) skips them. It also reports any running service still executing a replaced binary — overwriting `/usr/local/bin/spx` while `spinifex.target` is up does not move services onto it, so each keeps executing the replaced inode until it restarts and a node can serve a mixture of two builds indefinitely. Checks compare content, not existence, so a present-but-stale asset is caught. On x86_64 it also checks that the QEMU binary the daemon will run lists the `microvm` machine type and that the microvm firmware `/usr/share/spinifex/microvm/qboot.rom` is installed, since load balancers boot on both. It runs the host network check `spinifex-daemon` and `spinifex-vpcd` run before they start, and fails while firewalld or ufw is active, since the Spinifex nftables policy is the only supported node firewall. It reports a Warning, without failing, when systemd-resolved is active but `/etc/resolv.conf` is not its stub, since lookups then bypass the route that sends Spinifex names to northstar. Exits non-zero if any asset is Missing, Stale, Ungranted, Unsupported or Failed. |
 | `spx admin node js-probe` | `--timeout` (default: 10s) | Round-trips a canary key through the cluster-state KV bucket and exits non-zero if the write or the read-back fails. JetStream can latch a write error per message block and keep serving reads — nats-server's filestore records the failure on the block and returns it on every later write, and nothing clears it short of a restart. The HTTP health endpoint reports the server-wide JetStream state, which that latch never touches, so only an actual write observes it. |
-| `spx admin node-facts collect` | — | Runs `dmidecode`, `ethtool`, `udevadm`, `networkctl lldp` and `systemd-detect-virt`, and writes their raw output to `/etc/spinifex/node-facts/`, readable by the spinifex group. `spinifex-daemon.service` runs it as a root `ExecStartPre` because the daemon itself is unprivileged; the daemon parses the files when the operator console asks. A probe that fails writes no new file, so a file from an earlier successful run can remain and the console reads it as available; a source with no file is shown as unavailable. |
 | `spx admin banner` | `--boot-check` | Writes the node information banner to `/etc/issue` (shown before login on the physical/serial console) and `/etc/motd` (shown after SSH login). With `--boot-check` it also detects a management IP that has changed since last boot and updates `/etc/spinifex/node.conf` accordingly. |
 
 ### Storage Inspection
@@ -137,7 +136,6 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | Command | Flags | Description |
 |---------|-------|-------------|
 | `spx admin volumes orphans` | — | Lists volumes the storage provider holds that the control plane has no record of. Their blocks consume space but they have no API handle, so they can be neither described nor deleted through the EC2 API. **Reports only, never deletes**: the evidence that an orphan is still wanted is the record that went missing, so removal is an operator decision taken per volume. It also lists UEFI variable stores nothing owns: a `uefi/i-*/` prefix with no instance record and no store record, a `uefi/ami-*/` prefix with no AMI document, and a store record that is not `deleting` and has no instance record. These are reported and never deleted, because an absent instance record is not proof of a terminate. |
-| `spx admin volumes obligations` | — | Lists the open unpublish obligations, oldest first: a volume export that a force detach, or a terminate whose unmount did not complete, left on the node that held the attachment. Each row names the volume, the node, the instance and the age, plus attempts and the last error once the node has tried. The node settles each one once it answers; an obligation stays open while its node is down and closes without the node only once the node has been removed from the cluster. **Reports only.** |
 | `spx admin volumes unsealed` | — | Lists volumes a node has open, or had open, without a confirmed seal to the backend — the node named holds writes predastore may not have. Starting such a volume elsewhere is refused while that node is alive (its daemon heartbeat is under 60 seconds old, or cannot be read) and is otherwise allowed, in which case it opens from the last checkpoint that did reach the backend. This is the list of volumes where that would cost something. |
 | `spx admin storage status` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Sends the signed `GetStorageStatus` action to the AWS gateway with credentials from the standard AWS chain (e.g. `AWS_PROFILE=spinifex`) and prints what the console's Service Metrics page shows: predastore encoding, meta and blob node counts, cluster health, a meta node table (health, raft state, leader, term, commit and applied index) and the blob nodes. Cluster health is `Degraded` when any meta node is unreachable or the nodes reporting raft state do not agree on one leader. A meta node read only through its host's `/readyz` shows healthy with blank raft fields. |
 
@@ -157,9 +155,8 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | `spx admin account list --remote` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Same listing read through `POST /admin/ListAccounts` over SigV4 instead of NATS, so an off-cluster operator or the load-test harness sees its tenants without SSH access to a node. Credentials come from the standard AWS credential chain: on a cluster node that means `AWS_PROFILE=spinifex` (there is no default profile, so without it the command fails to find credentials), or equivalent credentials; the caller must be an IAM user in the super-admin account holding `spinifex:ListAccounts`. |
 | `spx admin account quota get <account-id>` | `--endpoint` (default: this node's AWS gateway), `--region` (default: this node's region), `--ca-bundle` (default: this node's CA) | Reads the limits in force for one account through `POST /admin/GetAccountQuota` over SigV4, and reports for each dimension whether it came from the `[quota]` block in `awsgw.toml` (`config`) or from an override stored against the account (`override`). Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:GetAccountQuota`. |
 | `spx admin account quota set <account-id>` | `--vcpus`, `--vpcs`, `--subnets`, `--eips`, `--volumes`, `--volumes-gib`, `--rds-instances`, `--load-balancers`, `--clear`, plus `--endpoint`, `--region`, `--ca-bundle` | Overrides individual limits for one account through `POST /admin/PutAccountQuota`, leaving every dimension not named inheriting the configured baseline. `-1` uncaps a dimension; `0` is a real limit that denies every request on it. `--clear` removes the whole override record and cannot be combined with a limit flag. The account must already exist. Prints the resulting limits, so a caller never has to guess what it applied. The caller must hold `spinifex:PutAccountQuota`. |
-| `spx admin account delete <account-id>` | `--dry-run`, `--force`, `--non-finalizing` (hidden), `--yes` (requires `--name`), `--name` (account name, must match the stored record), `--endpoint`, `--region`, `--ca-bundle`, `--client-token` | Always runs through `POST /admin/DeleteAccount` over SigV4 and follows `DescribeAccountDeletion` to completion; the CLI never tears an account down in its own process, so a CLI delete and an API delete cannot race. Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:DeleteAccount` and `spinifex:DescribeAccountDeletion`. Prints the inventory it is about to remove → confirms the account name (prompted unless `--name` is given) → marks the account `TERMINATING`, which blocks its credentials cluster-wide → deletes its resources in dependency order (compute, attachments, storage, network, platform, identity, account), waiting for each stage to actually drain before starting the next → seals the account's quota override record so no later write can recreate it, which needs every gateway node the cluster configures to answer a read-only probe first (otherwise nothing is sealed and the account stays `TERMINATING`) → releases the name reservation and quota counter → deletes the account record. Anything that will not delete is reported as **stuck** with the reason, and the account stays `TERMINATING` so the residue keeps an owner. `000000000000` and `000000000001` are refused. `--dry-run` lists and changes nothing. The compute stage reaps ECS clusters, EKS nodegroups and clusters, and RDS instances before it reaps plain instances, because their backing VMs belong to the system account and a tenant-scoped instance listing cannot see them. The storage stage reaps the account's S3 buckets: uploads in flight are aborted, then the objects are removed a page at a time, then the bucket. Only buckets the account owns are ever listed, so the shared platform buckets are out of reach rather than filtered out. Platform-managed serving endpoints are reaped ahead of instances, because only their own delete releases the ENI and attached data volume alongside the VM, and spot instance requests are cancelled first so a live request cannot launch a replacement for something already terminated. The network stage reaps load balancers, then target groups, then NAT gateways and egress-only internet gateways before the subnets, security groups and addresses they hold. The platform stage reaps ACM certificates. |
+| `spx admin account delete <account-id>` | `--dry-run`, `--force`, `--yes` (requires `--name`), `--name` (account name, must match the stored record), `--endpoint`, `--region`, `--ca-bundle`, `--client-token` | Always runs through `POST /admin/DeleteAccount` over SigV4 and follows `DescribeAccountDeletion` to completion; the CLI never tears an account down in its own process, so a CLI delete and an API delete cannot race. Credentials come from the standard AWS chain; the caller must be an IAM user in the super-admin account holding `spinifex:DeleteAccount` and `spinifex:DescribeAccountDeletion`. Prints the inventory it is about to remove → confirms the account name (prompted unless `--name` is given) → marks the account `TERMINATING`, which blocks its credentials cluster-wide → deletes its resources in dependency order (compute, attachments, storage, network, platform, identity, account), waiting for each stage to actually drain before starting the next → seals the account's quota override record so no later write can recreate it, which needs every gateway node the cluster configures to answer a read-only probe first (otherwise nothing is sealed and the account stays `TERMINATING`) → releases the name reservation and quota counter → deletes the account record. Anything that will not delete is reported as **stuck** with the reason, and the account stays `TERMINATING` so the residue keeps an owner. `000000000000` and `000000000001` are refused. `--dry-run` lists and changes nothing. The compute stage reaps ECS clusters, EKS nodegroups and clusters, and RDS instances before it reaps plain instances, because their backing VMs belong to the system account and a tenant-scoped instance listing cannot see them. The storage stage reaps the account's S3 buckets: uploads in flight are aborted, then the objects are removed a page at a time, then the bucket. Only buckets the account owns are ever listed, so the shared platform buckets are out of reach rather than filtered out. Platform-managed serving endpoints are reaped ahead of instances, because only their own delete releases the ENI and attached data volume alongside the VM, and spot instance requests are cancelled first so a live request cannot launch a replacement for something already terminated. The network stage reaps load balancers, then target groups, then NAT gateways and egress-only internet gateways before the subnets, security groups and addresses they hold. The platform stage reaps ACM certificates. |
 | `spx admin account delete --force` | as above | Escalates only for resources the ordinary API refuses: it clears a volume attachment in the control plane without the guest's cooperation (`ec2.ForceDetachVolume`), hard-destroys after a graceful attempt times out, and treats an already-missing resource as deleted. It never reorders the stages and never widens scope beyond the account. Use it once the guest is being destroyed — this is the escape from a volume stuck attached to an instance that will not terminate, where neither can otherwise be deleted. |
-| `spx admin account delete --non-finalizing` | as above | An exercise run, not a finalising delete, but the cleanup is real. Marks the account `TERMINATING`, deletes the resources of each EC2 platform kind the evaluation found quiesced (for example key pairs, empty placement groups and orphaned tags), re-reads the owner inventory against that cleanup, and ends `EXERCISED` with the account still `TERMINATING`. No other stage runs and the account record, name reservation and quota are left in place. It refuses `--force` and `--dry-run`. |
 | `spx admin account delete --remote` | as above | Accepted for existing scripts and ignored: every delete already goes through the admin API. |
 | `spx admin principal create <name>` | `--grant` (repeatable or comma-separated; default: every admin method — `CreateAccount`, `DeleteAccount`, `DescribeAccountDeletion`, `GetAccountQuota`, `ListAccounts`, `PutAccountQuota`) | Creates an IAM user in the super-admin account with an inline `spx-admin-methods` policy allowing the requested methods, revokes any access key left from a previous run, and mints one new key. Each method is named rather than granted as `spinifex:*`, so a later addition to the admin surface is not authorised by an existing key. The principal's grants are exactly that one policy: re-running with fewer `--grant` values narrows it, and any other inline policy on the user is removed, so a rename cannot leave a wider grant behind. The secret is printed once; store it in a secret manager or an AWS profile, never in a config file. `admin` is refused as a name — it is the account's own administrator and rotating its key would revoke the bootstrap credential. |
 | `spx admin principal list` | — | Lists every principal in the super-admin account with its access-key count and what it may call. An attached policy is reported by name (`AdministratorAccess (attached)`) rather than expanded, which is how an unscoped principal is spotted. |

docs/DESIGN.md

@@ -111,9 +111,7 @@ Daemons (`spinifex/runtime/daemon/daemon.go`) subscribe to NATS topics and handl
 When a daemon handles `RunInstances`:
 
 1. **Resource Check**: Validates CPU/memory availability via `ResourceManager`
-2. **Volume Generation**: Creates boot and EFI volumes via Viperblock
-   A UEFI instance whose record carries a variable store pointer has no EFI volume: its `vars.fd` is fetched from the object under `uefi/` in the configured Predastore bucket that the pointer names, recorded in the `spinifex-uefi-stores` JetStream bucket (or reused when `seeded.json` on this node already names the same object) after a firmware architecture and VARS size check, and a record with both or neither is refused.
-   RunInstances still creates the EFI volume, nothing writes `vars.fd` back to the object store yet, and a terminated instance's store is removed by a cluster-wide reaper.
+2. **Volume Generation**: Creates the boot volume through the EBS provider and, for a UEFI guest, seeds its variable store as an object under `uefi/<instance-id>/` with a store record in `spinifex-uefi-stores` that points at it
 3. **Volume Mount**: Publishes each volume through the EBS provider contract (`PublishVolume` on `ebs.provider.v1.{node}.mount`), receives its export
 4. **QEMU Launch**: Builds and executes QEMU command with NBD-backed disks
 5. **QMP Monitoring**: Establishes QEMU Machine Protocol connection for VM management
@@ -227,9 +225,6 @@ It may keep writing for 40 seconds after the last write the store acknowledged,
 A slow acknowledgement therefore shortens the remaining validity rather than extending it, and an acquisition acknowledged after its validity has already run out is refused as retryable.
 
 For a holder that cannot renew, a timer is armed for the validity deadline, leaving a nominal 5 seconds before the entry can expire; when it fires, the export's nbdkit is fenced with SIGKILL (`volume_fence.go`).
-The kill comes first: the read that names the winner (bounded at 2 seconds) and the notice to the daemon follow it, so a hung lease store cannot hold the writer alive.
-The notice is a request that viperblockd repeats every 2 seconds until the local daemon replies with `VolumeFencedAck`, because the fence often runs while the node's NATS server is restarting.
-On the notice the daemon moves the guest using the volume to `error` with state reason `Server.VolumeFenced` and SIGKILLs its QEMU immediately rather than shutting it down gracefully, because there is no disk left to flush to; it only stops the guest process and removes its taps, and never terminates the instance or deletes a volume.
 The fence runs on its own goroutine, so Go scheduling, that goroutine and process termination all delay the kill, and enough delay can use up the whole margin; a kill held in uninterruptible sleep, which the fence waits up to 30 seconds for, is the extreme case.
 Once a lease is lost or has lapsed it stays lost: a renewal that succeeds afterwards is ignored, and another open on the same node cannot share it while the fence runs.
 

docs/compute/vpc-networking/README.md

@@ -315,7 +315,7 @@ prefix_len = 24
 
 ### Host access to instances (jumpbox pattern)
 
-The spinifex host automatically reaches every instance's **private IP**: each node converges a host route into OVN (`<vpc-cidr> via <gateway-transit-ip> dev spx-nat-host`) for every attached VPC, and the IGW attach exempts the transit net from SNAT, so replies to host-initiated connections come back un-NATted. No per-instance setup. This only exists in routed NAT mode; other modes install no host routes. Every node runs this loop itself, on each intent change, on a nudge from the local IGW hooks and every 30 seconds, so a node that missed the attach event still gets the route. A route is removed on every node when no attached VPC holds its CIDR any more, such as after a detach. When several attached VPCs share one CIDR, the bootstrap VPC wins, then the lowest VPC ID, so detaching one gateway repoints the route to another attached VPC instead of removing it.
+The spinifex host automatically reaches every instance's **private IP**: IGW attach installs a host route into OVN (`<vpc-cidr> via <gateway-transit-ip> dev spx-nat-host`) and exempts the transit net from SNAT, so replies to host-initiated connections come back un-NATted. No per-instance setup.
 
 Security groups still apply and the default SG is closed to the host, same as AWS — open SSH/ICMP from the transit net first:
 

docs/coverage/ec2/README.md

@@ -43,21 +43,6 @@ A missing `Name`, `BlockDeviceMappings` or root snapshot answers `MissingParamet
 Registering with `--ena-support`, `--no-ena-support` and neither, copying the image, and three refusals were run on a three-node cluster with the AWS CLI.
 Launching an instance from an `EnaSupport` image was not run, so the launch path is unchanged by construction and not by test.
 
-### CreateSnapshot Error Mapping
-
-`CreateSnapshot` drains and snapshots the volume inside the EBS provider.
-A provider that cannot snapshot a volume because it is in use, as the `qemunbd` provider cannot for an attached volume, answers `ErrVolumeInUse`, which becomes `IncorrectState` with the message `The volume 'vol-...' is in use. Detach it before creating a snapshot.`
-The `viperblockd` provider drains a mounted export and snapshots it, so an attached volume succeeds there.
-A provider that cannot answer now, for example because another node holds the volume, answers `ServiceUnavailable` so the caller retries.
-Any other provider failure is `ServerInternal`.
-The snapshot taken for `CreateImage` maps these errors the same way.
-
-### EBS Encryption by Default Follows the Platform
-
-Whether new volumes are encrypted is decided by the platform's encryption key, not by the account.
-`GetEbsEncryptionByDefault` and `EnableEbsEncryptionByDefault` report `true` when the platform has an encryption key configured and `false` otherwise, and `EnableEbsEncryptionByDefault` does not change that.
-`DisableEbsEncryptionByDefault` is refused with `OperationNotPermitted` and the message `EBS encryption by default cannot be disabled: this platform encrypts every volume` while the platform encrypts, and answers `false` where it does not.
-
 ### Operations
 
 | Operation |

docs/security/network-connections/README.md

@@ -99,7 +99,7 @@ behavior in other words.
 | 4432 | Formation server | HTTPS | Cluster (bootstrap only) | Cluster join coordination; active only while a join token is valid. Binds the node's `--bind` (lan) address. See *Formation port lifecycle* below. | Short-lived bearer token + TLS¹ |
 | 4222 | spinifex-nats (client) | NATS + TLS | Cluster | Internal service bus for EC2/EBS/VPC/S3 handlers | Token + mutual TLS (cluster CA) |
 | 4248 | spinifex-nats (cluster) | NATS + TLS | Cluster | Inter-node NATS federation | Token + mutual TLS (cluster CA) |
-| 5001 | iperf2 (admin network test) | TCP | Cluster, any plane | Throughput test between two nodes, started only by the super-admin account. Listens on one plane address for the length of one test, then exits. Runs with no capabilities. | None — peer-scoped by firewall, and the client is refused unless the target address is the one the named target node reports on that plane |
+| 5001 | iperf2 (admin network test) | TCP | Cluster, any plane | Throughput test between two nodes, started only by the super-admin account. Listens on one plane address for the length of one test, then exits. Runs with no capabilities. | None — peer-scoped by firewall, and the client may only target its own plane subnet |
 | 5300 | northstar | DNS (UDP + TCP) | Cluster | Forward target for every node's per-instance DNS shim, dialled cross-node. Binds the wildcard by design. | None — restrict by firewall |
 | 6660 | predastore (blob node) | QUIC / UDP | Cluster | Erasure-coded object shard transport between hosts. Multi-node clusters only — see *Predastore ports* below. | TLS 1.3, server certificate verified against the cluster CA |
 | 7660 | predastore (meta node) | QUIC / UDP | Cluster | Raft consensus over global state — buckets and the object index — between hosts. Multi-node clusters only. | TLS 1.3, server certificate verified against the cluster CA |

internal/awsmodel/ec2error_test.go

@@ -77,8 +77,8 @@ func TestEC2ErrorCatalogMetadataAndClasses(t *testing.T) {
 	if catalog.metadata.VerifiedOn == "" {
 		t.Fatal("verifiedOn is empty")
 	}
-	if len(catalog.codes) != 116 {
-		t.Fatalf("catalog code count = %d, want 116", len(catalog.codes))
+	if len(catalog.codes) != 115 {
+		t.Fatalf("catalog code count = %d, want 115", len(catalog.codes))
 	}
 	if catalog.codes["InvalidParameterValue"] != ec2ErrorClassClient {
 		t.Fatal("InvalidParameterValue is not classified as client")

internal/awsmodel/models/ec2/error-codes.json

@@ -1,6 +1,6 @@
 {
   "source": "https://docs.aws.amazon.com/ec2/latest/devguide/errors-overview.html",
-  "verifiedOn": "2026-11-04",
+  "verifiedOn": "2026-08-05",
   "commonClient": [
     "AuthFailure",
     "Blocked",
@@ -95,7 +95,6 @@
     "InvalidVolumeID.Malformed",
     "InvalidVpcID.NotFound",
     "InvalidVpcRange",
-    "NatGatewayNotFound",
     "OperationNotPermitted",
     "ReservationCapacityExceeded",
     "Resource.AlreadyAssociated",

internal/gwsign/signer.go

@@ -15,7 +15,6 @@ import (
 	v4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
 	awsconfig "github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
-	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
 	"github.com/mulgadc/spinifex/internal/imdscreds"
 )
 
@@ -58,9 +57,8 @@ func NewInstanceRole() *Signer {
 }
 
 // newInstanceRole reads from the IMDS root base; "" means the default endpoint.
-// optFns adjust the SDK IMDS client, e.g. its retryer; none keeps the defaults.
-func newInstanceRole(base string, optFns ...func(*imds.Options)) *Signer {
-	return &Signer{provider: aws.NewCredentialsCache(imdscreds.NewProvider(nil, base, optFns...))}
+func newInstanceRole(base string) *Signer {
+	return &Signer{provider: aws.NewCredentialsCache(imdscreds.NewProvider(nil, base))}
 }
 
 // EnsureCredentials retrieves once from the provider, surfacing the error so a

internal/gwsign/signer_test.go

@@ -12,7 +12,6 @@ import (
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/credentials"
-	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
 	"github.com/mulgadc/bluebottle/pkg/sigv4"
 )
 
@@ -125,41 +124,27 @@ func TestNewIMDS_BuildsProvider(t *testing.T) {
 	}
 }
 
-// imdsStub serves one instance-role credential over IMDSv2. It echoes the token
-// TTL header as real IMDS does, so the SDK client caches the token rather than
-// re-fetching it, through the retry backoff, on every later call.
-func imdsStub(t *testing.T, accessKey, expiration string) *httptest.Server {
-	t.Helper()
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		switch r.URL.Path {
-		case "/latest/api/token":
-			w.Header().Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "21600")
-			_, _ = w.Write([]byte("token"))
-		case "/latest/meta-data/iam/security-credentials/":
-			_, _ = w.Write([]byte("sqsWorkerRole"))
-		case "/latest/meta-data/iam/security-credentials/sqsWorkerRole":
-			_, _ = w.Write([]byte(`{"Code":"Success","AccessKeyId":"` + accessKey + `","SecretAccessKey":"s","Token":"t","Expiration":"` + expiration + `"}`))
-		default:
-			http.NotFound(w, r)
-		}
-	}))
-	t.Cleanup(srv.Close)
-	return srv
-}
-
-// singleAttempt disables IMDS client retries so an unreachable endpoint fails
-// on the first attempt instead of after the SDK's 1s-per-retry backoff.
-func singleAttempt(o *imds.Options) { o.Retryer = aws.NopRetryer{} }
-
 // The instance-role signer ignores environment keys and signs only with the
 // credentials the metadata service issues.
 func TestNewInstanceRole_UsesOnlyIMDS(t *testing.T) {
 	t.Setenv("AWS_ACCESS_KEY_ID", testAccessKey)
 	t.Setenv("AWS_SECRET_ACCESS_KEY", testSecretKey)
 	const imdsKey = "ASIAIMDSONLYEXAMPLE1"
-	srv := imdsStub(t, imdsKey, "2099-01-01T00:00:00Z")
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/latest/api/token":
+			_, _ = w.Write([]byte("token"))
+		case "/latest/meta-data/iam/security-credentials/":
+			_, _ = w.Write([]byte("sqsWorkerRole"))
+		case "/latest/meta-data/iam/security-credentials/sqsWorkerRole":
+			_, _ = w.Write([]byte(`{"Code":"Success","AccessKeyId":"` + imdsKey + `","SecretAccessKey":"s","Token":"t","Expiration":"2099-01-01T00:00:00Z"}`))
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	t.Cleanup(srv.Close)
 
-	s := newInstanceRole(srv.URL, singleAttempt)
+	s := newInstanceRole(srv.URL)
 	req := httptest.NewRequest(http.MethodPost, "https://gw.example/", nil)
 	if err := s.Sign(req, emptySHA, testService, testRegion); err != nil {
 		t.Fatal(err)
@@ -169,7 +154,7 @@ func TestNewInstanceRole_UsesOnlyIMDS(t *testing.T) {
 	}
 
 	srv.Close()
-	cold := newInstanceRole(srv.URL, singleAttempt)
+	cold := newInstanceRole(srv.URL)
 	if err := cold.EnsureCredentials(context.Background()); err == nil {
 		t.Fatal("environment keys were used when IMDS was unreachable")
 	}
@@ -179,8 +164,19 @@ func TestNewInstanceRole_UsesOnlyIMDS(t *testing.T) {
 // IMDS must fail the sign rather than reuse the expired key with a new expiry.
 func TestNewInstanceRole_FailedRefreshErrorsInsteadOfStaleKey(t *testing.T) {
 	const staleKey = "ASIASTALEEXAMPLE0001"
-	srv := imdsStub(t, staleKey, "2000-01-01T00:00:00Z")
-	s := newInstanceRole(srv.URL, singleAttempt)
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/latest/api/token":
+			_, _ = w.Write([]byte("token"))
+		case "/latest/meta-data/iam/security-credentials/":
+			_, _ = w.Write([]byte("sqsWorkerRole"))
+		case "/latest/meta-data/iam/security-credentials/sqsWorkerRole":
+			_, _ = w.Write([]byte(`{"Code":"Success","AccessKeyId":"` + staleKey + `","SecretAccessKey":"s","Token":"t","Expiration":"2000-01-01T00:00:00Z"}`))
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	s := newInstanceRole(srv.URL)
 	if err := s.EnsureCredentials(t.Context()); err != nil {
 		t.Fatalf("first retrieve: %v", err)
 	}

internal/imdscreds/imdscreds.go

@@ -36,10 +36,10 @@ func Fetch(client *http.Client, base string) (aws.Credentials, error) {
 }
 
 // NewProvider builds an ec2rolecreds provider against base, an IMDS root URL
-// such as http://169.254.169.254/latest. client and optFns are passed to
-// NewClient.
-func NewProvider(client *http.Client, base string, optFns ...func(*imds.Options)) aws.CredentialsProvider {
-	imdsClient := NewClient(client, base, optFns...)
+// such as http://169.254.169.254/latest. A nil client uses the SDK IMDS
+// client's default HTTP client.
+func NewProvider(client *http.Client, base string) aws.CredentialsProvider {
+	imdsClient := NewClient(client, base)
 	return noExtendOnFailure{provider: ec2rolecreds.New(func(o *ec2rolecreds.Options) {
 		o.Client = imdsClient
 	})}
@@ -71,15 +71,16 @@ func (p noExtendOnFailure) AdjustExpiresBy(creds aws.Credentials, dur time.Durat
 	return p.provider.AdjustExpiresBy(creds, dur)
 }
 
-// NewClient builds an SDK IMDS client against base, an IMDS root URL with or
-// without a trailing /latest. A nil client uses the SDK's default HTTP client;
-// optFns adjust the options after the defaults here, as imds.New applies them.
-func NewClient(client *http.Client, base string, optFns ...func(*imds.Options)) *imds.Client {
+// NewClient builds an SDK IMDS client against base, an IMDS root URL such as
+// http://169.254.169.254/latest. A trailing /latest is stripped since the
+// client appends its own; both forms of base keep working. A nil client uses
+// the SDK IMDS client's default HTTP client.
+func NewClient(client *http.Client, base string) *imds.Client {
 	opts := imds.Options{Endpoint: rootEndpoint(base)}
 	if client != nil {
 		opts.HTTPClient = client
 	}
-	return imds.New(opts, optFns...)
+	return imds.New(opts)
 }
 
 // rootEndpoint strips a trailing /latest from base so it can be used as the

internal/testkit/nats.go

@@ -27,7 +27,8 @@ func StartTestNATS(t testing.TB) (*server.Server, *nats.Conn) {
 	}
 	ns, err := server.NewServer(opts)
 	require.NoError(t, err)
-	startServer(t, ns)
+	go ns.Start()
+	waitReadyForConnections(t, ns)
 	t.Cleanup(func() { ns.Shutdown() })
 
 	nc, err := nats.Connect(ns.ClientURL())
@@ -36,14 +37,17 @@ func StartTestNATS(t testing.TB) (*server.Server, *nats.Conn) {
 	return ns, nc
 }
 
-// startServer runs Start in this goroutine: it returns once JetStream is
-// enabled and the client listener is bound, so the readiness check passes on
-// its first look instead of spinning or sleeping while the server comes up.
-func startServer(t testing.TB, ns *server.Server) {
+// waitReadyForConnections spins until the server accepts connections. Calling
+// ReadyForConnections with a longer timeout makes it sleep 25ms per attempt,
+// which every embedded server in the suite would otherwise pay.
+func waitReadyForConnections(t testing.TB, ns *server.Server) {
 	t.Helper()
-	ns.Start()
-	if !ns.ReadyForConnections(5 * time.Second) {
-		t.Fatal("embedded NATS server not ready for connections within 5s")
+	deadline := time.Now().Add(5 * time.Second)
+	for !ns.ReadyForConnections(time.Millisecond) {
+		if time.Now().After(deadline) {
+			t.Fatal("embedded NATS server not ready for connections within 5s")
+		}
+		time.Sleep(200 * time.Microsecond)
 	}
 }
 
@@ -62,7 +66,8 @@ func StartTestJetStream(t *testing.T) (*server.Server, *nats.Conn, jetstream.Jet
 	}
 	ns, err := server.NewServer(opts)
 	require.NoError(t, err)
-	startServer(t, ns)
+	go ns.Start()
+	waitReadyForConnections(t, ns)
 	t.Cleanup(func() { ns.Shutdown() })
 
 	nc, err := nats.Connect(ns.ClientURL())

scripts/README.md

@@ -98,9 +98,6 @@ Both ship to `/usr/local/share/spinifex/` on every node, so they are available o
 | `build-system-image.sh` | Builds a minimal system AMI (Alpine or Ubuntu) from a manifest. All customisation happens inside the libguestfs appliance, so the build touches no host block device, no host mount, and needs no sudo. `--import` also registers it as an AMI. |
 | `publish-system-image.sh` | Converts a built raw image to compressed qcow2, writes the `.sha256` sidecar the catalog verifier expects, and uploads both to the R2 bucket behind `iso.mulgadc.com/system-ami/`. |
 | `build-microvm-image.sh` | Builds the minimal Alpine kernel + initramfs used by the QEMU microvm machine type and the lb-agent, carrying haproxy (ALB) and nginx-stream (NLB) data planes. |
-| `build-lambda-runtime-microvm-image.sh` | Builds the Lambda runtime microVM kernel and initramfs: a guest running the Lambda guest agent (`bin/lambda-agent` by default) with virtio-net for its host control connection. Takes an optional agent binary and output directory (default `build/microvm/lambda-runtime`). Not shipped. |
-| `build-sqs-store-microvm-image.sh` | Builds the SQS microVM kernel and initramfs: one guest running the dedicated SQS `nats-server` and the SQS worker as separate processes, with ext4 support and a clean power-down. The guest never formats a disk. Output goes to `build/microvm/sqs-worker`. |
-| `lib/microvm-image.sh` | Shared steps for the two builders above, sourced rather than run: tool checks, the Alpine rootfs, and module stripping that keeps `virtio*`, `*fw_cfg*` and the modules the caller names with their dependencies. |
 | `build-qboot.sh` | Builds qboot, the firmware the daemon passes to every microvm with `-bios`, from a pinned upstream commit in a digest-pinned Debian container. Fails unless the output matches the recorded sha256, which is also Debian 13's `qboot.rom`. A host whose `/usr/share/qemu/qboot.rom` (`qemu-system-data`) has that hash installs it and skips the container; any other host compiles. Writes `qboot.rom` and its GPL-2.0 licence beside the microvm kernel, and the matching source tarball, which releases publish, under `build/qboot/`. `make build-microvm-image` runs it. |
 | `mkosi-build.sh` | Runs an mkosi system-image build inside the pinned builder container, so a runner needs only Docker and the toolchain cannot drift. `--image` names an output and expands to the right profile list. |
 | `mkosi-publish.sh` | Publishes an mkosi-built image to R2 from inside the same pinned container — qcow2 conversion, `.sha256` sidecar, and upload. |

spec/EBSProvider/Capabilities.tla

@@ -1,27 +0,0 @@
-------------------------------------------------- MODULE Capabilities -------------------------------------------------
-(*********************************************************************************************************************)
-(* The optional behaviour that an implementation advertises.                                                         *)
-(*********************************************************************************************************************)
-EXTENDS EBSProvider
-
------------------------------------------------------------------------------------------------------------------------
-\* Interface methods
-
-GetCapabilitiesRequests == {<<>>}
-
-GetCapabilities(req, val, err) ==
-  (*******************************************************************************************************************)
-  (* The implementation reports which optional behaviour it supports.                                                *)
-  (*******************************************************************************************************************)
-     \* The provider failed to answer.
-  \/ /\ val = None
-     /\ err \in ProviderErrors
-     /\ UNCHANGED vars
-
-     \* The request was accepted.
-     \* Every answer is the same, so an earlier one may be cached.
-  \/ /\ val = Capabilities
-     /\ err = None
-     /\ UNCHANGED vars
-
-=======================================================================================================================

spec/EBSProvider/EBSProvider.tla

@@ -1,163 +0,0 @@
-------------------------------------------------- MODULE EBSProvider --------------------------------------------------
-(*********************************************************************************************************************)
-(* The state that an implementation exposes, and the types and constants that every other module shares.             *)
-(*********************************************************************************************************************)
-EXTENDS
-  Naturals,
-  Sequences,
-  FiniteSets,
-  Functions,
-  TLC
-
------------------------------------------------------------------------------------------------------------------------
-\* Constants
-
-CONSTANTS
-  MaxBytes,
-  VolumeIDs, VolumeHandles,
-  NodeIDs, NBDURIs, DevicePaths,
-  SnapshotIDs, SnapshotHandles,
-  AvailabilityZones, Timestamps,
-  Parameters, ProviderData,
-  ListTokens, MaxListResults,
-  \* The optional behaviour that the implementation advertises, which never changes.
-  Capabilities,
-  \* Whether the provider may fail to answer at any point in a behaviour.
-  ProviderMayFail,
-  \* Stands in for an optional field that a request or response omitted.
-  None
-
-ASSUME MaxBytes \in Nat \ {0}
-ASSUME VolumeIDs /= {} /\ VolumeHandles /= {}
-ASSUME NodeIDs /= {} => NBDURIs /= {} \/ DevicePaths /= {}
-ASSUME SnapshotIDs /= {} => SnapshotHandles /= {} /\ Timestamps /= {}
-ASSUME ListTokens /= {}
-ASSUME MaxListResults \in Nat \ {0}
-ASSUME None \notin UNION {VolumeIDs, VolumeHandles, NodeIDs, SnapshotIDs, SnapshotHandles,
-                          AvailabilityZones, Timestamps, Parameters, ProviderData, ListTokens}
-ASSUME Capabilities \in [online_expansion          : BOOLEAN,
-                         sparse_extent_reporting   : BOOLEAN,
-                         crash_consistent_snapshot : BOOLEAN,
-                         volume_seeding            : BOOLEAN,
-                         read_only_publish         : BOOLEAN,
-                         volume_enumeration        : BOOLEAN,
-                         snapshot_enumeration      : BOOLEAN,
-                         owner_routing             : BOOLEAN,
-                         exclusion                 : [scope             : {"none", "node", "cluster"},
-                                                      claim_ttl_seconds : Nat,
-                                                      fences_lost_claim : BOOLEAN]]
-ASSUME ProviderMayFail \in BOOLEAN
-
------------------------------------------------------------------------------------------------------------------------
-\* Variables
-
-VARIABLES
-  \* Every volume that the implementation exposes, keyed by volume ID.
-  volumes,
-  \* Every publication that the implementation exposes, keyed by the volume and the node it is published to.
-  publications,
-  \* Every snapshot that the implementation exposes, keyed by snapshot ID.
-  snapshots
-
-vars == <<volumes, publications, snapshots>>
-
------------------------------------------------------------------------------------------------------------------------
-\* Miscellaneous
-
-IsPartialFn(f, D, R) == DOMAIN f \subseteq D /\ \A x \in DOMAIN f : f[x] \in R
-
-Optional(S) == S \union {None}
-
-InsertKey(f, x, y) == f @@ (x :> y)
-
-RemoveKey(f, x) == RestrictDomain(f, LAMBDA x_2 : x_2 /= x)
-
-KeyExists(f, x) == x \in DOMAIN f
-
------------------------------------------------------------------------------------------------------------------------
-\* Types
-
-Bytes == 0..MaxBytes
-
-\* The errors that an implementation may return when its provider fails to answer.
-ProviderErrors == IF ProviderMayFail
-                  THEN {"internal", "unavailable"}
-                  \* Otherwise an implementation could conform by refusing to answer.
-                  ELSE {}
-
-CapacityRanges == [required_bytes : Bytes, limit_bytes : Bytes]
-
-VolumeStates == {"available", "in-use"}
-
-Volumes == [capacity_bytes     : 1..MaxBytes,
-            state              : VolumeStates,
-            handle             : VolumeHandles,
-            availability_zone  : Optional(AvailabilityZones),
-
-            \* Never returned, but kept to tell a retry from a conflicting request.
-            source_snapshot_id : Optional(SnapshotIDs),
-            parameters         : Optional(Parameters),
-            provider_data      : Optional(ProviderData)]
-
-\* How a published volume is reached on its node, either over NBD or as a host block device.
-Exports == [kind    : {"nbd"},
-            nbd_uri : NBDURIs]
-           \union
-           [kind        : {"block"},
-            device_path : DevicePaths,
-            size_bytes  : 1..MaxBytes]
-
-\* A volume published to a node, and whether it was published read-only.
-Publications == [export    : Exports,
-                 read_only : BOOLEAN]
-
-SnapshotStates == {"pending", "completed", "error"}
-
-Snapshots == [size_bytes       : 1..MaxBytes,
-              created_at       : Timestamps,
-              state            : SnapshotStates,
-              handle           : SnapshotHandles,
-              source_volume_id : VolumeIDs]
-
------------------------------------------------------------------------------------------------------------------------
-\* State helpers
-
-\* The part of a volume that the implementation returns.
-VolumeView(volID) ==
-  ("id" :> volID) @@ Restrict(volumes[volID], {"capacity_bytes",
-                                               "state",
-                                               "handle",
-                                               "availability_zone",
-                                               "provider_data"})
-
-\* The nodes that a volume is published to.
-PublishedNodes(volID) == {n \in NodeIDs : KeyExists(publications, <<volID, n>>)}
-
-\* The part of a publication that the implementation returns.
-PublicationView(volID, nodeID) == [volume_id |-> volID,
-                                   node_id   |-> nodeID,
-                                   export    |-> publications[volID, nodeID].export]
-
-\* The part of a snapshot that the implementation returns.
-SnapshotView(snapID) == ("id" :> snapID) @@ snapshots[snapID]
-
-\* The number of refs that a page may hold, with an omitted or oversized request clamped to the maximum.
-PageSize(m) == IF m <= 0 \/ m > MaxListResults THEN MaxListResults ELSE m
-
------------------------------------------------------------------------------------------------------------------------
-\* Invariants
-
-TypeOK ==
-  /\ IsPartialFn(volumes, VolumeIDs, Volumes)
-  /\ IsPartialFn(publications, VolumeIDs \X NodeIDs, Publications)
-  /\ IsPartialFn(snapshots, SnapshotIDs, Snapshots)
-
------------------------------------------------------------------------------------------------------------------------
-\* Initial state
-
-Init ==
-  /\ volumes      = <<>>
-  /\ publications = <<>>
-  /\ snapshots    = <<>>
-
-=======================================================================================================================

spec/EBSProvider/README.md

@@ -1,140 +0,0 @@
-# EBSProvider specification
-
-The contract that every implementation of the `EBSProvider` interface must meet.
-
-## Terminology
-
-- **Provider:** the service that backs an implementation's operations, such as Viperblock.
-  The memory implementation has none.
-- **Implementation:** a Go implementation of the `EBSProvider` interface, which adapts between the shapes of the contract and the shapes that its provider expects.
-- **Contract:** the behaviour that this specification requires of every implementation, as observed through the interface.
-
-## Open questions
-
-### EBSProvider
-
-- What may a failed call change?
-  A failed call leaves the state unchanged, but a timeout can arrive after the provider acted, and a crash can leave work half done.
-- Is the schema version part of the contract?
-  Every method refuses an unsupported version with `unsupported_version`, and the conformance suite tests it, but the specification does not model versions.
-- Are `internal` and `unavailable` the only errors that a failed provider may cause?
-- Must a present but malformed ID be refused with `invalid_argument`?
-  IDs are opaque here, so only an omitted one is refused.
-
-### Capabilities
-
-#### GetCapabilities
-
-- Is it a lasting promise that capabilities never change, even when an implementation or its provider is upgraded or reconfigured after an earlier answer was cached?
-
-### Volumes
-
-- Who decides whether a volume is `in-use`?
-  An implementation may track attachments itself, or derive the state from whether a publication claim is held.
-
-#### CreateVolume
-
-- May the capacity be anywhere in the requested range, as CSI allows?
-  An implementation that rounds up would then fail the comparison that tells a retry from a conflicting request.
-- May seed data be combined with a source snapshot?
-  The Viperblock implementation refuses, the memory implementation accepts.
-- Must the source snapshot be completed?
-  AWS refuses one that is not.
-- Must an implementation store the availability zone, parameters and source snapshot, so that it can detect a conflicting retry?
-  The Viperblock implementation does not.
-- Should a retry be compared against the original request rather than the current volume?
-  A retry after ExpandVolume conflicts on capacity.
-- Should a retry with a different source snapshot volume ID conflict?
-  It is not compared.
-- Is byte equality the meaning of equal parameters, or may an implementation compare them semantically?
-- Should `MaxSeedBytes` be modelled?
-  Seed data larger than it is refused with `invalid_argument`.
-
-#### GetVolume
-
-- Must an implementation compare the handle?
-  The Viperblock implementation ignores it.
-- Is `not_found` the right error for a volume that was deleted and recreated under the same ID?
-
-#### ListVolumes
-
-- Must following next tokens list every volume that exists throughout the walk exactly once?
-  The conformance suite tests it, but the specification describes each call alone, so it does not say how one page relates to the next.
-- What should a starting token that the implementation never issued do?
-  Every implementation treats any token as a position to resume after, while CSI refuses an invalid one with `ABORTED`.
-- May a page hold fewer refs than requested yet carry a next token?
-  AWS and CSI allow it, but here a next token comes only with a full page, so that a walk cannot stall on empty pages.
-
-#### ExpandVolume
-
-- Should a request below the current capacity succeed without change, as CSI recommends, rather than be refused with `invalid_argument`?
-  It is refused here, as AWS refuses `ModifyVolume` to a smaller size and the conformance suite tests, while CSI says the plugin should reply OK when the volume is already at least the requested size.
-- Must a block export's size follow an online expansion?
-  A published volume keeps its export's `size_bytes`, so after it grows the export reports the old size.
-
-#### PublishVolume
-
-- Should `volume_in_use` depend on the exclusion scope?
-  An implementation whose scope is `none` lets two writers in, and one whose scope is `node` cannot see another node, yet both are refused here.
-- Must a republish to the same node with a different `read_only` be refused?
-  The Viperblock implementation refuses it with `volume_in_use`, while the memory implementation ignores the flag.
-- May two volumes on one node share an NBD URI or a device path?
-- Must a block export's size equal the volume's capacity?
-
-#### UnpublishVolume
-
-- Must an implementation compare the handle?
-  Both the Viperblock and memory implementations ignore it.
-- Is a provider that fails to seal the volume a failed provider?
-  The Viperblock implementation answers `internal` and leaves the volume published, which the specification allows only when the provider may fail.
-- Should a claim that lapses through its TTL be modelled, as a step that happens without a call?
-- Should a volume that does not exist be refused with `not_found`?
-  It succeeds here, as CSI requires and the conformance suite tests, while AWS refuses `DetachVolume` on a missing volume with `InvalidVolume.NotFound`.
-
-#### DeleteVolume
-
-- Must an implementation compare the handle before refusing a published volume?
-  The memory implementation does, and succeeds without deleting it, while the Viperblock implementation ignores the handle and refuses with `volume_in_use`.
-- Should a volume that does not exist be refused with `not_found`?
-  It succeeds here, as CSI requires and the conformance suite tests, while AWS refuses `DeleteVolume` on a missing volume with `InvalidVolume.NotFound`.
-
-### Snapshots
-
-- Should a pending snapshot's move to `completed` or `error` be modelled, as a step that happens without a call?
-  Without it a pending snapshot never completes, and no snapshot is ever in `error`.
-
-#### CreateSnapshot
-
-- May success return a `pending` snapshot?
-  AWS starts a snapshot as `pending`, while the memory implementation returns `completed` and the NATS implementation waits for completion and refuses anything else.
-- Must a retry compare the volume handle?
-  It is ignored, as in the memory implementation, so a volume deleted and recreated under the same ID returns the old snapshot.
-- Should a copy's ID count as a retry?
-  A copy keeps its source's volume ID, so creating a snapshot of that volume under the copy's ID returns the copy.
-- Should `crash_consistent_snapshot` be modelled?
-  The specification does not model volume data, so it cannot say what a snapshot of a published volume holds.
-
-#### CopySnapshot
-
-- Should a retried copy return the existing copy, as a retried CreateVolume does, rather than `already_exists`?
-- May a source that is not completed be copied?
-  AWS refuses until it is.
-- What state does a copy start in?
-  The memory implementation always returns `completed`, while AWS starts a copy as `pending`.
-
-#### ListSnapshots
-
-- The questions for ListVolumes apply here too.
-- Must a ref carry the source volume ID?
-  The Viperblock implementation leaves it empty, since reading it costs a fetch per snapshot, yet reconciliation needs it to act on a snapshot whose metadata is lost.
-
-#### DeleteSnapshot
-
-- Is the handle check part of the contract?
-  The Viperblock implementation ignores the handle and deletes anyway.
-- May a pending snapshot be deleted?
-  AWS allows it.
-- What does success promise?
-  The Go docs say the snapshot is gone, but a snapshot whose handle did not match is left in place.
-- Should a snapshot that does not exist be refused with `not_found`?
-  It succeeds here, as CSI requires and the conformance suite tests, while AWS refuses `DeleteSnapshot` on a missing snapshot with `InvalidSnapshot.NotFound`.

spec/EBSProvider/Snapshots.tla

@@ -1,159 +0,0 @@
--------------------------------------------------- MODULE Snapshots ---------------------------------------------------
-(*********************************************************************************************************************)
-(* The lifecycle of a snapshot, from its creation to its deletion.                                                   *)
-(*********************************************************************************************************************)
-EXTENDS EBSProvider
-
------------------------------------------------------------------------------------------------------------------------
-\* Interface methods
-
-CreateSnapshotRequests == [snapshot_id   : Optional(SnapshotIDs),
-                           volume_id     : Optional(VolumeIDs),
-                           volume_handle : Optional(VolumeHandles)]
-
-CreateSnapshot(req, val, err) ==
-  (*******************************************************************************************************************)
-  (* A volume's data is frozen as a snapshot, from which new volumes can be restored. A retry of a call that         *)
-  (* already succeeded returns the same snapshot.                                                                    *)
-  (*******************************************************************************************************************)
-  LET snapID == req.snapshot_id
-      volID  == req.volume_id
-
-      Errors ==
-        LET checks == [invalid_argument |-> None \in {snapID, volID},
-                       already_exists   |-> /\ KeyExists(snapshots, snapID)
-                                            /\ snapshots[snapID].source_volume_id /= volID,
-                       \* Only checked for a new snapshot, so a retry outlives the volume.
-                       not_found        |-> /\ ~KeyExists(snapshots, snapID)
-                                            /\ \/ ~KeyExists(volumes, volID)
-                                               \/ req.volume_handle \notin {None, volumes[volID].handle}]
-        IN {e \in DOMAIN checks : checks[e]}
-
-     \* The request was rejected, or the provider failed to answer.
-  IN \/ /\ val = None
-        /\ err \in Errors \union ProviderErrors
-        /\ UNCHANGED vars
-
-     \* The request was accepted.
-     \/ /\ Errors = {}
-           \* A retry returns the existing snapshot.
-        /\ \/ /\ KeyExists(snapshots, snapID)
-              /\ UNCHANGED snapshots
-
-           \/ /\ ~KeyExists(snapshots, snapID)
-              \* The handle, creation time and state are the implementation's choice.
-              /\ \E h \in SnapshotHandles \ {snapshots[x].handle : x \in DOMAIN snapshots},
-                    t \in Timestamps,
-                    s \in {"pending", "completed"} :
-                   snapshots' = InsertKey(snapshots, snapID, [size_bytes       |-> volumes[volID].capacity_bytes,
-                                                              created_at       |-> t,
-                                                              state            |-> s,
-                                                              handle           |-> h,
-                                                              source_volume_id |-> volID])
-        /\ val = SnapshotView(snapID)'
-        /\ err = None
-        /\ UNCHANGED <<volumes, publications>>
-
-
-CopySnapshotRequests == [source_snapshot_id      : Optional(SnapshotIDs),
-                         destination_snapshot_id : Optional(SnapshotIDs),
-                         volume_id               : Optional(VolumeIDs)]
-
-CopySnapshot(req, val, err) ==
-  (*******************************************************************************************************************)
-  (* A snapshot is copied under a new ID, sharing its frozen data, and the copy can be deleted independently         *)
-  (* of it.                                                                                                          *)
-  (*******************************************************************************************************************)
-  LET srcSnapID == req.source_snapshot_id
-      dstSnapID == req.destination_snapshot_id
-      volID     == req.volume_id
-
-      Errors ==
-        LET checks == [invalid_argument |-> \/ None \in {srcSnapID, dstSnapID, volID}
-                                            \/ srcSnapID = dstSnapID
-                                            \* Only compared, not looked up, since a snapshot outlives its volume.
-                                            \/ /\ KeyExists(snapshots, srcSnapID)
-                                               /\ snapshots[srcSnapID].source_volume_id /= volID,
-                       already_exists   |-> KeyExists(snapshots, dstSnapID),
-                       not_found        |-> srcSnapID \notin Optional(DOMAIN snapshots)]
-        IN {e \in DOMAIN checks : checks[e]}
-
-     \* The request was rejected, or the provider failed to answer.
-  IN \/ /\ val = None
-        /\ err \in Errors \union ProviderErrors
-        /\ UNCHANGED vars
-
-     \* The request was accepted.
-     \/ /\ Errors = {}
-        \* The handle and creation time are the implementation's choice.
-        /\ \E h \in SnapshotHandles \ {snapshots[x].handle : x \in DOMAIN snapshots},
-              t \in Timestamps :
-             snapshots' = InsertKey(snapshots, dstSnapID, [created_at |-> t,
-                                                           handle     |-> h]
-                                                           @@ snapshots[srcSnapID])
-        /\ val = SnapshotView(dstSnapID)'
-        /\ err = None
-        /\ UNCHANGED <<volumes, publications>>
-
-
-ListSnapshotsRequests == [max_results    : -1..(MaxListResults + 1),
-                          starting_token : Optional(ListTokens)]
-
-ListSnapshots(req, val, err) ==
-  (*******************************************************************************************************************)
-  (* Some of the snapshots are listed, a page at a time, and nothing changes. A call with no starting token that     *)
-  (* returns no next token lists every snapshot.                                                                     *)
-  (*******************************************************************************************************************)
-  LET Errors ==
-    LET checks == [unsupported_capability |-> ~Capabilities.snapshot_enumeration]
-    IN {e \in DOMAIN checks : checks[e]}
-
-     \* The request was rejected, or the provider failed to answer.
-  IN \/ /\ val = None
-        /\ err \in Errors \union ProviderErrors
-        /\ UNCHANGED vars
-
-     \* The request was accepted.
-     \/ /\ Errors = {}
-        /\ LET refs == {[id               |-> id,
-                         source_volume_id |-> snapshots[id].source_volume_id,
-                         handle           |-> snapshots[id].handle] : id \in DOMAIN snapshots}
-           \* The page and the next token are the implementation's choice.
-           IN \E page \in SUBSET refs, next \in Optional(ListTokens) :
-                /\ Cardinality(page) <= PageSize(req.max_results)
-                /\ next /= None => Cardinality(page) = PageSize(req.max_results)
-                /\ req.starting_token = None /\ next = None => page = refs
-                /\ val = [snapshots |-> page, next_token |-> next]
-        /\ err = None
-        /\ UNCHANGED vars
-
-
-DeleteSnapshotRequests == [snapshot_id : Optional(SnapshotIDs),
-                           handle      : Optional(SnapshotHandles)]
-
-DeleteSnapshot(req, err) ==
-  (*******************************************************************************************************************)
-  (* A snapshot is deleted, so that no new volume can be restored from it. Volumes already restored from it,         *)
-  (* and copies of it, are unaffected.                                                                               *)
-  (*******************************************************************************************************************)
-  LET snapID == req.snapshot_id
-
-      Errors ==
-        LET checks == [invalid_argument |-> snapID = None]
-        IN {e \in DOMAIN checks : checks[e]}
-
-     \* The request was rejected, or the provider failed to answer.
-  IN \/ /\ err \in Errors \union ProviderErrors
-        /\ UNCHANGED vars
-
-     \* The request was accepted.
-     \/ /\ Errors = {}
-        /\ snapshots' = IF /\ KeyExists(snapshots, snapID)
-                           /\ req.handle \notin {None, snapshots[snapID].handle}
-                        THEN snapshots
-                        \* Record removal is idempotent so that retries succeed.
-                        ELSE RemoveKey(snapshots, snapID)
-        /\ err = None
-        /\ UNCHANGED <<volumes, publications>>
-
-=======================================================================================================================

spec/EBSProvider/Volumes.tla

@@ -1,286 +0,0 @@
---------------------------------------------------- MODULE Volumes ----------------------------------------------------
-(*********************************************************************************************************************)
-(* The lifecycle of a volume, from its creation to its deletion.                                                     *)
-(*********************************************************************************************************************)
-EXTENDS EBSProvider
-
------------------------------------------------------------------------------------------------------------------------
-\* Interface methods
-
-CreateVolumeRequests == [volume_id                 : Optional(VolumeIDs),
-                         capacity_range            : CapacityRanges,
-                         availability_zone         : Optional(AvailabilityZones),
-                         source_snapshot_id        : Optional(SnapshotIDs),
-                         source_snapshot_volume_id : Optional(VolumeIDs),
-                         parameters                : Optional(Parameters),
-                         seed_data                 : Bytes]
-
-CreateVolume(req, val, err) ==
-  (*******************************************************************************************************************)
-  (* A volume of the requested capacity is created, either blank or holding a snapshot's data, optionally            *)
-  (* with seed data written at its start. A retry of a call that already succeeded returns the same volume.          *)
-  (*******************************************************************************************************************)
-  LET volID     == req.volume_id
-      snapID    == req.source_snapshot_id
-      snapVolID == req.source_snapshot_volume_id
-      capacity  == req.capacity_range
-
-      \* Any warranted error may be returned, since the order of checks is not part of the contract.
-      Errors ==
-        LET checks == [invalid_argument       |-> \/ volID = None
-                                                  \/ capacity.required_bytes <= 0
-                                                  \/ /\ capacity.limit_bytes > 0
-                                                     /\ capacity.required_bytes > capacity.limit_bytes
-                                                  \/ req.seed_data > capacity.required_bytes
-                                                  \/ snapID /= None /\ snapVolID = None
-                                                  \* Only checked for a new volume, so a retry outlives the snapshot.
-                                                  \/ /\ ~KeyExists(volumes, volID)
-                                                     /\ KeyExists(snapshots, snapID)
-                                                     /\ \/ capacity.required_bytes < snapshots[snapID].size_bytes
-                                                        \/ snapshots[snapID].source_volume_id /= snapVolID,
-                       \* Seed data is not compared, since it only applies on creation.
-                       already_exists         |-> /\ KeyExists(volumes, volID)
-                                                  /\ \/ volumes[volID].capacity_bytes     /= capacity.required_bytes
-                                                     \/ volumes[volID].availability_zone  /= req.availability_zone
-                                                     \/ volumes[volID].source_snapshot_id /= snapID
-                                                     \/ volumes[volID].parameters         /= req.parameters,
-                       not_found              |-> /\ ~KeyExists(volumes, volID)
-                                                  /\ snapID \notin Optional(DOMAIN snapshots),
-                       unsupported_capability |-> /\ req.seed_data > 0
-                                                  /\ ~Capabilities.volume_seeding]
-        IN {e \in DOMAIN checks : checks[e]}
-
-     \* The request was rejected, or the provider failed to answer.
-  IN \/ /\ val = None
-        /\ err \in Errors \union ProviderErrors
-        /\ UNCHANGED vars
-
-     \* The request was accepted.
-     \/ /\ Errors = {}
-           \* A retry returns the existing volume.
-        /\ \/ /\ KeyExists(volumes, volID)
-              /\ UNCHANGED volumes
-
-           \/ /\ ~KeyExists(volumes, volID)
-              \* The handle and provider data are the implementation's choice.
-              /\ \E h \in VolumeHandles \ {volumes[x].handle : x \in DOMAIN volumes},
-                    pd \in Optional(ProviderData) :
-                   volumes' = InsertKey(volumes, volID, [capacity_bytes     |-> capacity.required_bytes,
-                                                         state              |-> "available",
-                                                         handle             |-> h,
-                                                         availability_zone  |-> req.availability_zone,
-                                                         source_snapshot_id |-> snapID,
-                                                         parameters         |-> req.parameters,
-                                                         provider_data      |-> pd])
-        /\ val = VolumeView(volID)'
-        /\ err = None
-        /\ UNCHANGED <<publications, snapshots>>
-
-
-GetVolumeRequests == [volume_id : Optional(VolumeIDs),
-                      handle    : Optional(VolumeHandles)]
-
-GetVolume(req, val, err) ==
-  (*******************************************************************************************************************)
-  (* A volume's current capacity, state and handle are read, and nothing changes.                                    *)
-  (*******************************************************************************************************************)
-  LET volID == req.volume_id
-
-      Errors ==
-        LET checks == [invalid_argument |-> volID = None,
-                       not_found        |-> \/ ~KeyExists(volumes, volID)
-                                            \/ req.handle \notin {None, volumes[volID].handle}]
-        IN {e \in DOMAIN checks : checks[e]}
-
-     \* The request was rejected, or the provider failed to answer.
-  IN \/ /\ val = None
-        /\ err \in Errors \union ProviderErrors
-        /\ UNCHANGED vars
-
-     \* The request was accepted.
-     \/ /\ Errors = {}
-        /\ val = VolumeView(volID)
-        /\ err = None
-        /\ UNCHANGED vars
-
-
-ListVolumesRequests == [max_results    : -1..(MaxListResults + 1),
-                        starting_token : Optional(ListTokens)]
-
-ListVolumes(req, val, err) ==
-  (*******************************************************************************************************************)
-  (* Some of the volumes are listed, a page at a time, and nothing changes. A call with no starting token that       *)
-  (* returns no next token lists every volume.                                                                       *)
-  (*******************************************************************************************************************)
-  LET Errors ==
-    LET checks == [unsupported_capability |-> ~Capabilities.volume_enumeration]
-    IN {e \in DOMAIN checks : checks[e]}
-
-     \* The request was rejected, or the provider failed to answer.
-  IN \/ /\ val = None
-        /\ err \in Errors \union ProviderErrors
-        /\ UNCHANGED vars
-
-     \* The request was accepted.
-     \/ /\ Errors = {}
-        /\ LET refs == {[id |-> id, handle |-> volumes[id].handle] : id \in DOMAIN volumes}
-           \* The page and the next token are the implementation's choice.
-           IN \E page \in SUBSET refs, next \in Optional(ListTokens) :
-                /\ Cardinality(page) <= PageSize(req.max_results)
-                /\ next /= None => Cardinality(page) = PageSize(req.max_results)
-                /\ req.starting_token = None /\ next = None => page = refs
-                /\ val = [volumes |-> page, next_token |-> next]
-        /\ err = None
-        /\ UNCHANGED vars
-
-
-ExpandVolumeRequests == [volume_id      : Optional(VolumeIDs),
-                         handle         : Optional(VolumeHandles),
-                         capacity_range : CapacityRanges]
-
-ExpandVolume(req, val, err) ==
-  (*******************************************************************************************************************)
-  (* A volume grows to the requested capacity, keeping its data. A published volume grows only if the                *)
-  (* implementation supports online expansion, and a request for the current capacity succeeds without change.       *)
-  (*******************************************************************************************************************)
-  LET volID    == req.volume_id
-      capacity == req.capacity_range
-
-      Errors ==
-        LET checks == [invalid_argument |-> \/ volID = None
-                                            \/ capacity.required_bytes = 0
-                                            \/ /\ capacity.limit_bytes > 0
-                                               /\ capacity.required_bytes > capacity.limit_bytes
-                                            \/ /\ KeyExists(volumes, volID)
-                                               /\ capacity.required_bytes < volumes[volID].capacity_bytes,
-                       not_found        |-> \/ ~KeyExists(volumes, volID)
-                                            \/ req.handle \notin {None, volumes[volID].handle},
-                       volume_in_use    |-> /\ PublishedNodes(volID) /= {}
-                                            /\ ~Capabilities.online_expansion
-                                            /\ capacity.required_bytes > volumes[volID].capacity_bytes]
-        IN {e \in DOMAIN checks : checks[e]}
-
-     \* The request was rejected, or the provider failed to answer.
-  IN \/ /\ val = None
-        /\ err \in Errors \union ProviderErrors
-        /\ UNCHANGED vars
-
-     \* The request was accepted.
-     \/ /\ Errors = {}
-        /\ volumes' = [volumes EXCEPT ![volID].capacity_bytes = capacity.required_bytes]
-        /\ val = VolumeView(volID)'
-        /\ err = None
-        /\ UNCHANGED <<publications, snapshots>>
-
-
-PublishVolumeRequests == [volume_id : Optional(VolumeIDs),
-                          handle    : Optional(VolumeHandles),
-                          node_id   : Optional(NodeIDs),
-                          read_only : BOOLEAN]
-
-PublishVolume(req, val, err) ==
-  (*******************************************************************************************************************)
-  (* A volume's data becomes reachable on a node, through an export that a VM there can attach, either an NBD        *)
-  (* URI or a host block device. Reads and writes through the export reach the volume, and a read-only export        *)
-  (* refuses writes. Only one node at a time may reach a volume.                                                     *)
-  (*******************************************************************************************************************)
-  LET volID  == req.volume_id
-      nodeID == req.node_id
-
-      Errors ==
-        LET checks == [invalid_argument       |-> None \in {volID, nodeID},
-                       not_found              |-> \/ ~KeyExists(volumes, volID)
-                                                  \/ req.handle \notin {None, volumes[volID].handle},
-                       unsupported_capability |-> /\ req.read_only
-                                                  /\ ~Capabilities.read_only_publish,
-                       volume_in_use          |-> PublishedNodes(volID) \ {nodeID} /= {}]
-        IN {e \in DOMAIN checks : checks[e]}
-
-     \* The request was rejected, or the provider failed to answer.
-  IN \/ /\ val = None
-        /\ err \in Errors \union ProviderErrors
-        /\ UNCHANGED vars
-
-     \* The request was accepted.
-     \/ /\ Errors = {}
-           \* A retry returns the existing publication.
-        /\ \/ /\ KeyExists(publications, <<volID, nodeID>>)
-              /\ UNCHANGED publications
-
-           \/ /\ ~KeyExists(publications, <<volID, nodeID>>)
-              \* The type of export is the implementation's choice.
-              /\ \E x \in Exports :
-                   publications' = InsertKey(publications, <<volID, nodeID>>, [export    |-> x,
-                                                                               read_only |-> req.read_only])
-        /\ volumes' = [volumes EXCEPT ![volID].state = "in-use"]
-        /\ val = PublicationView(volID, nodeID)'
-        /\ err = None
-        /\ UNCHANGED snapshots
-
-
-UnpublishVolumeRequests == [volume_id : Optional(VolumeIDs),
-                            handle    : Optional(VolumeHandles),
-                            node_id   : Optional(NodeIDs)]
-
-UnpublishVolume(req, err) ==
-  (*******************************************************************************************************************)
-  (* A volume's export on a node is torn down, so that its data is no longer reachable there. Once no node           *)
-  (* can reach the volume, it is available again.                                                                    *)
-  (*******************************************************************************************************************)
-  LET volID  == req.volume_id
-      nodeID == req.node_id
-
-      Errors ==
-        LET checks == [invalid_argument |-> None \in {volID, nodeID}]
-        IN {e \in DOMAIN checks : checks[e]}
-
-     \* The request was rejected, or the provider failed to answer.
-  IN \/ /\ err \in Errors \union ProviderErrors
-        /\ UNCHANGED vars
-
-     \* The request was accepted.
-     \/ /\ Errors = {}
-        \* Record removal is idempotent so that retries succeed.
-        /\ publications' = RemoveKey(publications, <<volID, nodeID>>)
-        \* The volume is available again once no node holds it.
-        /\ volumes' = IF \/ ~KeyExists(volumes, volID)
-                         \/ PublishedNodes(volID)' /= {}
-                      THEN volumes
-                      ELSE [volumes EXCEPT ![volID].state = "available"]
-        /\ err = None
-        /\ UNCHANGED snapshots
-
-
-DeleteVolumeRequests == [volume_id : Optional(VolumeIDs),
-                         handle    : Optional(VolumeHandles)]
-
-DeleteVolume(req, err) ==
-  (*******************************************************************************************************************)
-  (* A volume is deleted, so that its data is gone. Snapshots taken from it are unaffected.                          *)
-  (*******************************************************************************************************************)
-  LET volID == req.volume_id
-
-      Errors ==
-        LET checks == [invalid_argument |-> volID = None,
-                       volume_in_use    |-> /\ PublishedNodes(volID) /= {}
-                                            /\ req.handle \in {None, volumes[volID].handle}]
-        IN {e \in DOMAIN checks : checks[e]}
-
-     \* The request was rejected, or the provider failed to answer.
-  IN \/ /\ err \in Errors \union ProviderErrors
-        /\ UNCHANGED vars
-
-     \* The request was accepted.
-     \/ /\ Errors = {}
-        \* Record removal is idempotent so that retries succeed.
-        /\ volumes' = IF /\ KeyExists(volumes, volID)
-                         /\ req.handle \notin {None, volumes[volID].handle}
-                      THEN volumes
-                      ELSE RemoveKey(volumes, volID)
-        /\ err = None
-        /\ UNCHANGED <<publications, snapshots>>
-
------------------------------------------------------------------------------------------------------------------------
-\* Specification
-
-=======================================================================================================================

spinifex/agents/lambda/agent/agent_test.go

@@ -46,30 +46,12 @@ func (l *fakeLink) Complete(ctx context.Context, r hostlink.Response) error {
 	return nil
 }
 
-// runtime serves srv and polls next, answering using reply. Cleanup cancels the
-// poll before closing the server: an in-flight next long-poll otherwise holds
-// its connection active and blocks Close forever.
-func runtime(t *testing.T, srv *runtimeapi.Server, reply func(id string, body []byte) (path string, out string)) {
+// runtime polls next and answers using reply.
+func runtime(t *testing.T, base string, reply func(id string, body []byte) (path string, out string)) {
 	t.Helper()
-	ts := httptest.NewServer(srv.Handler())
-	ctx, cancel := context.WithCancel(context.Background())
-	done := make(chan struct{})
-	t.Cleanup(func() {
-		cancel()
-		<-done
-		ts.Close()
-	})
-	do := func(method, url string, body io.Reader) (*http.Response, error) {
-		req, err := http.NewRequestWithContext(ctx, method, url, body)
-		if err != nil {
-			return nil, err
-		}
-		return ts.Client().Do(req)
-	}
 	go func() {
-		defer close(done)
 		for {
-			resp, err := do(http.MethodGet, ts.URL+api+"invocation/next", nil)
+			resp, err := http.Get(base + api + "invocation/next")
 			if err != nil {
 				return
 			}
@@ -77,7 +59,7 @@ func runtime(t *testing.T, srv *runtimeapi.Server, reply func(id string, body []
 			resp.Body.Close()
 			id := resp.Header.Get("Lambda-Runtime-Aws-Request-Id")
 			path, out := reply(id, body)
-			r, err := do(http.MethodPost, ts.URL+api+"invocation/"+id+path, strings.NewReader(out))
+			r, err := http.Post(base+api+"invocation/"+id+path, "application/json", strings.NewReader(out))
 			if err != nil {
 				return
 			}
@@ -90,7 +72,9 @@ func deadline(d time.Duration) int64 { return time.Now().Add(d).UnixMilli() }
 
 func TestRunServesRequestsAndEchoesIDs(t *testing.T) {
 	srv := runtimeapi.New()
-	runtime(t, srv, func(id string, body []byte) (string, string) {
+	ts := httptest.NewServer(srv.Handler())
+	defer ts.Close()
+	runtime(t, ts.URL, func(id string, body []byte) (string, string) {
 		if string(body) == "bad" {
 			return "/error", `{"errorMessage":"no"}`
 		}
@@ -165,7 +149,9 @@ func TestRunReturnsLinkError(t *testing.T) {
 func TestRunBoundsTheResultWriteByTheDeadlinePlusGrace(t *testing.T) {
 	t.Run("on time", func(t *testing.T) {
 		srv := runtimeapi.New()
-		runtime(t, srv, func(string, []byte) (string, string) { return "/response", "ok" })
+		ts := httptest.NewServer(srv.Handler())
+		defer ts.Close()
+		runtime(t, ts.URL, func(string, []byte) (string, string) { return "/response", "ok" })
 		dl := deadline(time.Minute)
 		link := &fakeLink{got: make(chan hostlink.Response, 1), ctxs: make(chan context.Context, 1),
 			reqs: []hostlink.Request{{RequestID: "r", DeadlineMs: dl}}}

spinifex/agents/lambda/hostlink/link.go

@@ -7,7 +7,6 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
-	"log/slog"
 )
 
 // ErrShutdown is returned by Link.Next when the host asks the agent to stop.
@@ -23,24 +22,6 @@ type Request struct {
 	TraceID     string `json:"trace_id,omitempty"`
 	DeadlineMs  int64  `json:"deadline_ms"`
 	Payload     []byte `json:"payload"`
-	// Credential is the environment's execution-role session. Only test-link
-	// hosts send it; release hosts leave it nil.
-	Credential *Credential `json:"credential,omitempty"`
-}
-
-// Credential is one execution-role session. It is never logged or stored.
-type Credential struct {
-	AccessKeyID     string `json:"access_key_id"`
-	SecretAccessKey string `json:"secret_access_key"`
-	SessionToken    string `json:"session_token"`
-}
-
-// LogValue keeps every value out of logs.
-func (Credential) LogValue() slog.Value { return slog.StringValue("[redacted]") }
-
-// Complete reports whether all three values are present.
-func (c Credential) Complete() bool {
-	return c.AccessKeyID != "" && c.SecretAccessKey != "" && c.SessionToken != ""
 }
 
 // Response is the outcome of a Request. FunctionError is empty on success. A

spinifex/agents/lambda/hostlink/protocol_release.go

@@ -1,8 +0,0 @@
-//go:build !lambdatestlink
-
-package hostlink
-
-import lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
-
-// Protocol is the ALPN protocol release hosts and guests offer.
-const Protocol = lambdav1.ALPNv1

spinifex/agents/lambda/hostlink/protocol_testlink.go

@@ -1,8 +0,0 @@
-//go:build lambdatestlink
-
-package hostlink
-
-// Protocol is the only ALPN protocol a test-link host and guest offer. It
-// differs from the release protocol because the first frame carries the
-// execution credential, which an older guest would ignore.
-const Protocol = "spinifex-lambda-testlink/2"

spinifex/agents/lambda/hostlink/tls.go

@@ -86,11 +86,11 @@ func (id Identity) ClientConfig() *tls.Config {
 		MinVersion:         tls.VersionTLS13,
 		MaxVersion:         tls.VersionTLS13,
 		Certificates:       []tls.Certificate{id.Cert},
-		NextProtos:         []string{Protocol},
+		NextProtos:         []string{lambdav1.ALPNv1},
 		InsecureSkipVerify: true, //nolint:gosec // VerifyConnection performs full validation against the pinned host certificate.
 		VerifyConnection: func(cs tls.ConnectionState) error {
-			if cs.NegotiatedProtocol != Protocol {
-				return fmt.Errorf("hostlink: application protocol %q, want %q", cs.NegotiatedProtocol, Protocol)
+			if cs.NegotiatedProtocol != lambdav1.ALPNv1 {
+				return fmt.Errorf("hostlink: application protocol %q, want %q", cs.NegotiatedProtocol, lambdav1.ALPNv1)
 			}
 			return pki.VerifyPinned(cs.PeerCertificates, id.HostCert, x509.ExtKeyUsageServerAuth, id.Scope.URI(lambdav1.RoleHost))
 		},

spinifex/agents/lambda/hostlink/tls_test.go

@@ -69,7 +69,7 @@ func newTestIdent(t *testing.T, scope lambdav1.Scope, role string) testIdent {
 func hostConfig(host testIdent, pinGuest *x509.Certificate, scope lambdav1.Scope) *tls.Config {
 	return &tls.Config{
 		MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{host.cert},
-		ClientAuth: tls.RequireAnyClientCert, NextProtos: []string{Protocol},
+		ClientAuth: tls.RequireAnyClientCert, NextProtos: []string{lambdav1.ALPNv1},
 		VerifyConnection: func(cs tls.ConnectionState) error {
 			return pki.VerifyPinned(cs.PeerCertificates, pinGuest, x509.ExtKeyUsageClientAuth, scope.URI(lambdav1.RoleGuest))
 		},

spinifex/agents/lambda/runtimeapi/server_admission_test.go

@@ -8,7 +8,6 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
-	"runtime"
 	"sync/atomic"
 	"testing"
 	"time"
@@ -30,20 +29,6 @@ func await[T any](t *testing.T, ch <-chan T) T {
 	}
 }
 
-// allocBytesPerOp is the mean heap allocated per call of f, measured the same
-// way testing.Benchmark does but over a fixed number of runs.
-func allocBytesPerOp(runs int, f func()) int64 {
-	f()
-	runtime.GC()
-	var before, after runtime.MemStats
-	runtime.ReadMemStats(&before)
-	for range runs {
-		f()
-	}
-	runtime.ReadMemStats(&after)
-	return int64(after.TotalAlloc-before.TotalAlloc) / int64(runs)
-}
-
 // arm puts a delivered, unclaimed flight in place without any HTTP traffic.
 func arm(s *Server, id string) *flight {
 	f := &flight{inv: inv(id, time.Minute), state: stRunning, done: make(chan outcome, 1)}
@@ -151,13 +136,16 @@ func TestDeclaredOversizeCompletesWithoutPayloadBuffer(t *testing.T) {
 	assert.Equal(t, ErrTypeUnhandled, res.FunctionError)
 	assert.Equal(t, ErrTypeResponseTooLarge, errType(t, res.Payload))
 
-	perOp := allocBytesPerOp(8, func() {
-		s := New()
-		f := arm(s, "req-1")
-		postTo(s, "response", "req-1", &patterned{n: n}, int64(n))
-		await(t, f.done)
+	br := testing.Benchmark(func(b *testing.B) {
+		b.ReportAllocs()
+		for range b.N {
+			s := New()
+			f := arm(s, "req-1")
+			postTo(s, "response", "req-1", &patterned{n: n}, int64(n))
+			await(t, f.done)
+		}
 	})
-	assert.Less(t, perOp, int64(256<<10), "an oversize body must not be buffered")
+	assert.Less(t, br.AllocedBytesPerOp(), int64(256<<10), "an oversize body must not be buffered")
 }
 
 func TestDeclaredOversizeCutShortIsUnknown(t *testing.T) {
@@ -221,13 +209,16 @@ func TestChunkedBodyCap(t *testing.T) {
 	})
 	t.Run("peak allocation stays within the cap plus one segment", func(t *testing.T) {
 		for _, n := range []int{MaxResponseBytes, MaxResponseBytes + 1} {
-			perOp := allocBytesPerOp(8, func() {
-				s := New()
-				f := arm(s, "req-1")
-				postTo(s, "response", "req-1", &patterned{n: n}, -1)
-				await(t, f.done)
+			br := testing.Benchmark(func(b *testing.B) {
+				b.ReportAllocs()
+				for range b.N {
+					s := New()
+					f := arm(s, "req-1")
+					postTo(s, "response", "req-1", &patterned{n: n}, -1)
+					await(t, f.done)
+				}
 			})
-			assert.Less(t, perOp, int64(MaxResponseBytes+segmentBytes+64<<10), "body of %d bytes", n)
+			assert.Less(t, br.AllocedBytesPerOp(), int64(MaxResponseBytes+segmentBytes+64<<10), "body of %d bytes", n)
 		}
 	})
 }

spinifex/agents/rds/agent/agent_test.go

@@ -459,7 +459,6 @@ func TestAcknowledgeBootstrap_WaitsForTheReceiptThenRetiresIt(t *testing.T) {
 // instance's volume carries the source instance's receipt. Acknowledging on one
 // would destroy the ciphertext this instance still needs.
 func TestAcknowledgeBootstrap_IgnoresAForeignReceipt(t *testing.T) {
-	t.Parallel()
 	tests := map[string]struct{ payloadID, dbID string }{
 		"another payload":  {payloadID: "bp-somebody-else", dbID: "db-resolved"},
 		"another instance": {payloadID: testPayloadID, dbID: "db-the-snapshot-source"},
@@ -467,7 +466,6 @@ func TestAcknowledgeBootstrap_IgnoresAForeignReceipt(t *testing.T) {
 
 	for name, tc := range tests {
 		t.Run(name, func(t *testing.T) {
-			t.Parallel()
 			cp := newFakeControlPlane()
 			cfg := testConfig(t)
 			cfg.DataMount = t.TempDir()
@@ -570,7 +568,6 @@ func TestRun_SendsConfiguredIdentifier(t *testing.T) {
 }
 
 func TestRun_WaitsForDataMountBeforePollingCommands(t *testing.T) {
-	t.Parallel()
 	cp := newFakeControlPlane()
 	cp.bootstrapOut = bootstrapOutput(nil)
 	cfg := testConfig(t)
@@ -857,7 +854,6 @@ func TestAcknowledgeBootstrap_StopsOnATerminalDenial(t *testing.T) {
 // window. Reporting that would put a failure on every healthy create's beat, and
 // then into the reason a create that timed out for some other cause records.
 func TestAcknowledgeBootstrap_DoesNotReportAnAbsentReceiptAsAFailure(t *testing.T) {
-	t.Parallel()
 	cp := newFakeControlPlane()
 	cfg := testConfig(t)
 	cfg.DataMount = t.TempDir()

spinifex/agents/rds/agent/command.go

@@ -126,8 +126,6 @@ type commander struct {
 	id       identity
 	registry commandRegistry
 	wait     time.Duration
-	// The wait after a failed poll, pollErrorBackoff unless a test shortens it.
-	pollBackoff time.Duration
 	// The control plane's budget per type, capped by the remaining budget the
 	// issuer stamps on each copy and measured from receipt. IssuedAt is not
 	// compared: the guest clock is not trusted against the host's.
@@ -154,13 +152,12 @@ func newCommander(cp controlPlane, registry commandRegistry, wait time.Duration)
 		wait = defaultPollWait
 	}
 	return &commander{
-		cp:          cp,
-		registry:    registry,
-		wait:        wait,
-		pollBackoff: pollErrorBackoff,
-		budget:      controlPlaneBudget,
-		serial:      make(chan queuedCommand, serialQueueDepth),
-		inFlight:    make(map[string]struct{}),
+		cp:       cp,
+		registry: registry,
+		wait:     wait,
+		budget:   controlPlaneBudget,
+		serial:   make(chan queuedCommand, serialQueueDepth),
+		inFlight: make(map[string]struct{}),
 	}
 }
 
@@ -190,7 +187,7 @@ func (c *commander) Run(ctx context.Context) {
 			select {
 			case <-ctx.Done():
 				return
-			case <-time.After(c.pollBackoff):
+			case <-time.After(pollErrorBackoff):
 			}
 			continue
 		}

spinifex/agents/rds/agent/command_test.go

@@ -158,30 +158,19 @@ func TestCommander_RetainsAndRetriesTheCompletePendingBatch(t *testing.T) {
 // A poll error must not spin the loop: without a backoff a gateway that is down
 // would be re-polled at line rate for as long as it stayed down.
 func TestCommander_BacksOffAfterAFailedPoll(t *testing.T) {
-	t.Parallel()
 	cp := newFakeControlPlane()
 	cp.pollErr = errors.New("gateway unreachable")
 
 	c := newCommander(cp, newCommandRegistry(&fakeEngine{}, &fakeStorage{}), 10*time.Millisecond)
-	// Outlasts the test, so any second poll inside the window is a missing backoff.
-	c.pollBackoff = time.Hour
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
 	defer cancel()
 	done := make(chan struct{})
 	go func() { c.Run(ctx); close(done) }()
-
-	select {
-	case <-cp.polled:
-	case <-ctx.Done():
-		t.Fatal("the commander never polled")
-	}
-	// Without the backoff the loop re-polls within microseconds of the failure.
-	time.Sleep(50 * time.Millisecond)
-	cancel()
 	<-done
 
-	if polls := len(cp.snapshotReplies()); polls != 1 {
-		t.Errorf("polled %d times against a broken channel inside the backoff, want 1", polls)
+	// pollErrorBackoff is 5s, so the window admits the first poll and no more.
+	if polls := len(cp.snapshotReplies()); polls > 1 {
+		t.Errorf("polled %d times in 200ms against a broken channel, want 1", polls)
 	}
 }
 
@@ -317,7 +306,6 @@ func TestCommander_ACopyArrivingBeforeTheReplyIsCarriedDoesNotRunAgain(t *testin
 // A quiesce that waited out its issuer's budget behind another command would
 // put the engine in backup mode for a snapshot nobody is taking.
 func TestCommander_AStaleQuiesceNeverTakesAHold(t *testing.T) {
-	t.Parallel()
 	cp := newFakeControlPlane()
 	cp.pollQueue = [][]rdsinstance.Command{
 		{{CommandID: "cmd-apply", Type: rdsinstance.CommandApplyParams}},
@@ -363,7 +351,6 @@ func TestCommander_AStaleQuiesceNeverTakesAHold(t *testing.T) {
 // Engine commands still run one at a time: a parameter apply rewrites the
 // option file a graceful stop would otherwise race.
 func TestCommander_AParameterApplyNeverRunsAlongsideAnEngineStop(t *testing.T) {
-	t.Parallel()
 	cp := newFakeControlPlane()
 	cp.pollQueue = [][]rdsinstance.Command{
 		{{CommandID: "cmd-apply", Type: rdsinstance.CommandApplyParams}},
@@ -427,7 +414,6 @@ func (c *commander) pendingReplies() []rdsinstance.CommandReply {
 // its issuer about to give up. A quiesce granted a fresh budget then would take
 // a hold nobody is left to release.
 func TestCommander_AQuiesceStampedNearlyOutOfBudgetNeverTakesAHold(t *testing.T) {
-	t.Parallel()
 	cp := newFakeControlPlane()
 	cp.pollQueue = [][]rdsinstance.Command{
 		{{CommandID: "cmd-apply", Type: rdsinstance.CommandApplyParams}},

spinifex/agents/rds/agent/config_test.go

@@ -16,16 +16,8 @@ func testLoadConfig(t *testing.T, engine string) Config {
 	if err := os.WriteFile(stamp, []byte(engine+"\n"), 0o444); err != nil {
 		t.Fatalf("write the engine stamp: %v", err)
 	}
-	// Delivered through the env file rather than t.Setenv, so callers can run in
-	// parallel.
-	envFile := filepath.Join(t.TempDir(), "agent.env")
-	if err := os.WriteFile(envFile, []byte("RDS_ENGINE_FILE="+stamp+"\n"), 0o600); err != nil {
-		t.Fatalf("write the env file: %v", err)
-	}
-	cfg := LoadConfig(envFile)
-	if cfg.EngineFile != stamp {
-		t.Fatalf("engine file = %q, want the stamp %q; is RDS_ENGINE_FILE set in the environment?", cfg.EngineFile, stamp)
-	}
+	t.Setenv("RDS_ENGINE_FILE", stamp)
+	cfg := LoadConfig(filepath.Join(t.TempDir(), "absent.env"))
 	cfg.EngineCatalog = testEngineCatalogLookup
 	return cfg
 }

spinifex/agents/rds/agent/engine_mariadb_test.go

@@ -883,7 +883,6 @@ func TestMariaDBProbe_SeparatesRecoveringFromAbsent(t *testing.T) {
 // mariadbd will not accept makes it exit during startup, so the pidfile names a
 // process that is gone and the deadline runs.
 func TestMariaDBProbe_BoundsAStalledClient(t *testing.T) {
-	t.Parallel()
 	cfg := mariadbTestProbeConfig(t)
 	cfg.EngineProbeTimeout = 20 * time.Millisecond
 	writePid(t, cfg.EnginePidFile, 4242)
@@ -947,7 +946,6 @@ func TestMariaDBProbe_MapsToTheHealthTheControlPlaneReads(t *testing.T) {
 // the server mid-recovery — a rollback that exists to break a boot loop would
 // create one, on an instance whose parameters were never at fault.
 func TestMariaDBProbe_LiveButUnreachableEngineResetsTheRollbackDeadline(t *testing.T) {
-	t.Parallel()
 	cfg := mariadbTestProbeConfig(t)
 	writePid(t, cfg.EnginePidFile, 4242)
 	probe := newEngineProbe(cfg.EnginePort, mariadbProbeState(cfg,
@@ -1154,18 +1152,12 @@ func TestMariaDBEngine_EveryResolvedDynamicValueRenders(t *testing.T) {
 	}
 }
 
-// Far below the production interval, so a hold test can wait out several polls
-// cheaply.
-const testHoldPoll = 5 * time.Millisecond
-
 // ALTER USER under a held BACKUP STAGE swaps the password in memory and then
 // waits at its commit, so a rotation issued mid-backup must not reach the
 // engine until the hold is released.
 func TestMariaDBEngine_SetPasswordWaitsForTheBackupHoldToEnd(t *testing.T) {
-	t.Parallel()
 	runner := &recordingRunner{}
 	engine := newTestMariaDBEngine(t, runner.run)
-	engine.holdPoll = testHoldPoll
 	engine.startSess = func(context.Context, command) (engineSession, error) { return &fakeSession{}, nil }
 	if err := engine.Quiesce(context.Background(), "orders-db-automated", time.Minute); err != nil {
 		t.Fatalf("Quiesce: %v", err)
@@ -1176,7 +1168,7 @@ func TestMariaDBEngine_SetPasswordWaitsForTheBackupHoldToEnd(t *testing.T) {
 	select {
 	case err := <-rotated:
 		t.Fatalf("SetPassword returned %v while the engine was held for a backup", err)
-	case <-time.After(10 * testHoldPoll):
+	case <-time.After(3 * holdPollInterval):
 	}
 
 	if err := engine.Unquiesce(context.Background()); err != nil {
@@ -1198,11 +1190,9 @@ func TestMariaDBEngine_SetPasswordWaitsForTheBackupHoldToEnd(t *testing.T) {
 // A hold that outlasts the wait fails the rotation before anything is sent, so
 // the control plane's error is the truth rather than half of it.
 func TestMariaDBEngine_SetPasswordFailsUntouchedWhenTheHoldOutlastsItsWait(t *testing.T) {
-	t.Parallel()
 	runner := &recordingRunner{}
 	engine := newTestMariaDBEngine(t, runner.run)
-	engine.holdPoll = testHoldPoll
-	engine.holdWait = 3 * testHoldPoll
+	engine.holdWait = 3 * holdPollInterval
 	engine.startSess = func(context.Context, command) (engineSession, error) { return &fakeSession{}, nil }
 	if err := engine.Quiesce(context.Background(), "orders-db-automated", time.Minute); err != nil {
 		t.Fatalf("Quiesce: %v", err)
@@ -1256,9 +1246,7 @@ func TestMariaDBEngine_SetPasswordBoundsTheStatementByTheRemainingBudget(t *test
 // A quiesce that cannot get the hold bookkeeping must give up with its command,
 // not wait on the mutex after its issuer has moved on.
 func TestMariaDBEngine_QuiesceGivesUpWaitingForTheBookkeepingWithItsContext(t *testing.T) {
-	t.Parallel()
 	engine := newTestMariaDBEngine(t, (&recordingRunner{}).run)
-	engine.holdPoll = testHoldPoll
 	started := 0
 	engine.startSess = func(context.Context, command) (engineSession, error) {
 		started++
@@ -1267,7 +1255,7 @@ func TestMariaDBEngine_QuiesceGivesUpWaitingForTheBookkeepingWithItsContext(t *t
 	engine.mu.Lock()
 	defer engine.mu.Unlock()
 
-	ctx, cancel := context.WithTimeout(context.Background(), 3*testHoldPoll)
+	ctx, cancel := context.WithTimeout(context.Background(), 3*holdPollInterval)
 	defer cancel()
 	done := make(chan error, 1)
 	go func() { done <- engine.Quiesce(ctx, "orders-db-automated", time.Minute) }()

spinifex/agents/rds/agent/health_test.go

@@ -83,7 +83,6 @@ func TestEngineProbe_ProbesTheAssignedPort(t *testing.T) {
 }
 
 func TestExecProbeRunner_PreservesTheProbeDeadline(t *testing.T) {
-	t.Parallel()
 	ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
 	defer cancel()
 

spinifex/agents/rds/agent/paramrecovery_test.go

@@ -385,7 +385,6 @@ func newTestGuard(engine parameterRecovery, probe *engineProbe) *paramGuard {
 // pg_isready exit 2 is an engine that is not there at all, which after a
 // parameter change is the boot-loop shape this breaks.
 func TestParamGuard_RollsBackReportsAndRestartsWhenTheEngineNeverComesUp(t *testing.T) {
-	t.Parallel()
 	engine := &fakeRecovery{restored: true}
 	cp := newFakeControlPlane()
 	guard := newParamGuard(engine, stubProbe(t, 2, nil), cp)
@@ -409,7 +408,6 @@ func TestParamGuard_RollsBackReportsAndRestartsWhenTheEngineNeverComesUp(t *test
 // the file can be replaced and the call still fail. The rollback has happened
 // either way, and the worse outcome is an instance left down and unreported.
 func TestParamGuard_ReportsAndRestartsWhenOnlyTheEnforcementFailed(t *testing.T) {
-	t.Parallel()
 	engine := &fakeRecovery{restored: true, restoreErr: errors.New("write the TLS enforcement rule: read-only file system")}
 	cp := newFakeControlPlane()
 	guard := newParamGuard(engine, stubProbe(t, 2, nil), cp)
@@ -433,7 +431,6 @@ func TestParamGuard_ReportsAndRestartsWhenOnlyTheEnforcementFailed(t *testing.T)
 // the instance on the set it is already failing under, so restarting it would
 // only churn a cluster that is down for a reason the rollback cannot fix.
 func TestParamGuard_DoesNotRestartWhenTheRestoreReplacedNothing(t *testing.T) {
-	t.Parallel()
 	engine := &fakeRecovery{restored: false, restoreErr: errors.New("read the last known good parameters: input/output error")}
 	cp := newFakeControlPlane()
 	guard := newParamGuard(engine, stubProbe(t, 2, nil), cp)
@@ -460,7 +457,6 @@ func TestParamGuard_DoesNothingWhenTheEngineIsServing(t *testing.T) {
 // pg_isready exit 1 is a postmaster that is up and replaying WAL. It is making
 // progress on its own, and restarting it would throw the recovery away.
 func TestParamGuard_LeavesARecoveringEngineAlone(t *testing.T) {
-	t.Parallel()
 	engine := &fakeRecovery{restored: true}
 	guard := newTestGuard(engine, stubProbe(t, 1, nil))
 	// Bounded so the deadline-resetting loop cannot run forever if it stops
@@ -477,7 +473,6 @@ func TestParamGuard_LeavesARecoveringEngineAlone(t *testing.T) {
 // The parameters are only the cause if they differ from the set the engine last
 // accepted; restarting on an identical set would churn for nothing.
 func TestParamGuard_DoesNotRestartWhenTheParametersAreAlreadyTheAcceptedSet(t *testing.T) {
-	t.Parallel()
 	engine := &fakeRecovery{restored: false}
 	newTestGuard(engine, stubProbe(t, 2, nil)).Run(t.Context())
 

spinifex/agents/rds/agent/quiesce.go

@@ -50,8 +50,6 @@ type quiesceState struct {
 	// Guarded because the expiry timer releases the hold on its own goroutine.
 	mu   sync.Mutex
 	held *quiesceHold
-	// How often a waiter looks again; zero means defaultHoldPollInterval.
-	holdPoll time.Duration
 }
 
 // The two things every quiesce needs, checked before a session is opened so a
@@ -123,21 +121,14 @@ func (q *quiesceState) lockWithin(ctx context.Context) error {
 		select {
 		case <-ctx.Done():
 			return fmt.Errorf("wait for the backup bookkeeping: %w", ctx.Err())
-		case <-time.After(q.holdPollInterval()):
+		case <-time.After(holdPollInterval):
 		}
 	}
 	return nil
 }
 
 // How often a statement waiting for a backup hold to end looks again.
-const defaultHoldPollInterval = 100 * time.Millisecond
-
-func (q *quiesceState) holdPollInterval() time.Duration {
-	if q.holdPoll > 0 {
-		return q.holdPoll
-	}
-	return defaultHoldPollInterval
-}
+const holdPollInterval = 100 * time.Millisecond
 
 // Runs fn while no backup hold is on the engine, and keeps a new one from
 // starting until it returns. A hold that outlasts wait fails the call before fn
@@ -160,7 +151,7 @@ func (q *quiesceState) outsideHold(ctx context.Context, wait time.Duration, fn f
 			return ctx.Err()
 		case <-deadline.C:
 			return fmt.Errorf("the engine is still held for a backup after %s", wait)
-		case <-time.After(q.holdPollInterval()):
+		case <-time.After(holdPollInterval):
 		}
 	}
 }

spinifex/agents/sqs/workerboot/serve_test.go

@@ -90,7 +90,7 @@ func localStore(t *testing.T, g *guest, port int) *server.Server {
 	if err != nil {
 		t.Fatal(err)
 	}
-	ns.Start()
+	go ns.Start()
 	if !ns.ReadyForConnections(10 * time.Second) {
 		t.Fatal("store server did not start")
 	}

spinifex/bootstrap/config/config.go

@@ -12,7 +12,6 @@ import (
 	"slices"
 	"strings"
 
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/zoneid"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
 	"github.com/spf13/viper"
 )
@@ -255,11 +254,9 @@ type Config struct {
 	Node string `json:"Node" mapstructure:"node"`
 	Host string `json:"Host" mapstructure:"host"` // Unique hostname or IP of this node
 	// AdvertiseIP is the off-host dial target. Empty falls back to Host for backward compat.
-	AdvertiseIP string `json:"AdvertiseIP" mapstructure:"advertise"`
-	Region      string `json:"Region" mapstructure:"region"`
-	AZ          string `json:"AZ" mapstructure:"az"`
-	// AZShortCode overrides the Region's AZ ID prefix (the apse2 in apse2-az1). Empty uses AWS's table or the derived code.
-	AZShortCode string   `json:"AZShortCode" mapstructure:"az_short_code"`
+	AdvertiseIP string   `json:"AdvertiseIP" mapstructure:"advertise"`
+	Region      string   `json:"Region" mapstructure:"region"`
+	AZ          string   `json:"AZ" mapstructure:"az"`
 	DataDir     string   `json:"DataDir" mapstructure:"data_dir"`
 	Services    []string `json:"Services" mapstructure:"services"` // Which services this node runs locally
 
@@ -910,9 +907,6 @@ func validateClusterConfig(cc *ClusterConfig) error {
 		// Range checks only. Both settings are deliberately unbounded above by
 		// anything host-aware: they are operator tunables, and the host cost is
 		// the value times the volume count for the operator to weigh.
-		if c := nodeCfg.AZShortCode; c != "" && !zoneid.ValidShortCode(c) {
-			return fmt.Errorf("config: [nodes.%s] az_short_code=%q must match [a-z0-9]{2,8}; fix it or remove the key to use the Region's default", nodeName, c)
-		}
 		if t := nodeCfg.EBS.DefaultThreads; t < 0 || t > MaxNBDKitThreads {
 			return fmt.Errorf("config: [nodes.%s.ebs] default_threads=%d out of range; use 1-%d, or remove the key for the default of %d", nodeName, t, MaxNBDKitThreads, DefaultNBDKitThreads)
 		}

spinifex/bootstrap/config/config_test.go

@@ -1680,24 +1680,3 @@ func TestNodesRunning(t *testing.T) {
 	assert.Equal(t, []string{"a", "c", "d"}, c.NodesRunning("daemon"))
 	assert.Empty(t, c.NodesRunning("nonesuch"))
 }
-
-func TestLoadConfig_AZShortCodeParsedAndInvalidRefused(t *testing.T) {
-	load := func(t *testing.T, code string) (*ClusterConfig, error) {
-		t.Helper()
-		resetViper(t)
-		path := filepath.Join(t.TempDir(), "spinifex.toml")
-		toml := fmt.Sprintf("node = \"n1\"\n\n[nodes.n1]\nregion = \"mulga-lab\"\naz = \"mulga-laba\"\naz_short_code = %q\n", code)
-		require.NoError(t, os.WriteFile(path, []byte(toml), 0600))
-		return LoadConfig(path)
-	}
-
-	cfg, err := load(t, "lab1")
-	require.NoError(t, err)
-	assert.Equal(t, "lab1", cfg.Nodes["n1"].AZShortCode)
-
-	for _, bad := range []string{"a", "UPPER", "has-dash", "toolongcode"} {
-		_, err := load(t, bad)
-		require.Error(t, err, bad)
-		assert.Contains(t, err.Error(), "az_short_code")
-	}
-}

spinifex/domains/acm/privateca.go

@@ -14,7 +14,6 @@ import (
 	"slices"
 	"sort"
 	"strings"
-	"testing"
 	"time"
 )
 
@@ -26,16 +25,6 @@ var (
 	leafKeyBits = 2048
 )
 
-// ShortenKeyBitsForTest generates tenant CA and leaf keys at bits for the rest
-// of one test and restores the previous sizes afterwards. There is one size per
-// process, so the test must not run in parallel with anything generating keys.
-func ShortenKeyBitsForTest(tb testing.TB, bits int) {
-	tb.Helper()
-	prevCA, prevLeaf := caKeyBits, leafKeyBits
-	caKeyBits, leafKeyBits = bits, bits
-	tb.Cleanup(func() { caKeyBits, leafKeyBits = prevCA, prevLeaf })
-}
-
 const (
 	// tenantCAValidity is long because the root is a one-time trust-store
 	// install on every client device: rotating it invalidates all of them at

spinifex/domains/acm/service_impl_test.go

@@ -808,7 +808,7 @@ func TestRequestCertificate_NoTenantCAIsAClientErrorNamingTheCause(t *testing.T)
 	assert.Equal(t, awserrors.ErrorResourceNotFound, code)
 	assert.NotEqual(t, awserrors.ErrorInternalError, code, "a retryable 500 hangs the provider's create")
 
-	assert.Equal(t, 400, awserrors.LookupErrorMessage("acm", code).HTTPCode, "AWS answers a missing ACM resource with 400")
+	assert.Equal(t, 404, awserrors.ErrorLookup[code].HTTPCode)
 	assert.True(t, awserrors.IsTerminal(err), "the provider must fail fast, not retry with backoff")
 
 	assert.Contains(t, message, "tenant CA", "the message must name the missing tenant CA")

spinifex/domains/admission/quota/vcpu.go

@@ -175,12 +175,11 @@ type InstanceTypeResolver func(accountID, instanceID string) (instanceType strin
 // NATSInstanceTypeResolver builds the production resolver from a node-identity
 // checked describe of the one instance. A found instance resolves even with a node
 // silent; an absence a silent node could hide is a retryable ServiceUnavailable.
-// opts reach the describe fan-out; production passes none.
-func NATSInstanceTypeResolver(natsConn *nats.Conn, nodeIDs func() []string, opts ...ec2instanceapi.DescribeOption) InstanceTypeResolver {
+func NATSInstanceTypeResolver(natsConn *nats.Conn, nodeIDs func() []string) InstanceTypeResolver {
 	return func(accountID, instanceID string) (string, bool, error) {
 		out, err := ec2instanceapi.DescribeInstancesChecked(context.Background(),
 			&ec2.DescribeInstancesInput{InstanceIds: []*string{aws.String(instanceID)}},
-			natsConn, 0, nodeIDs(), accountID, opts...)
+			natsConn, 0, nodeIDs(), accountID)
 		if err != nil {
 			if code, ok := awserrors.ResolveErrorCode(err); ok && code == awserrors.ErrorInvalidInstanceIDNotFound {
 				return "", false, nil

spinifex/domains/admission/quota/vcpu_retype_test.go

@@ -5,12 +5,10 @@ import (
 	"errors"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"testing"
-	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/internal/testkit"
-	ec2instanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/instance"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"github.com/nats-io/nats.go"
@@ -208,11 +206,6 @@ func TestInstanceTypeFromReservations(t *testing.T) {
 	}
 }
 
-// silentNodeFanoutTimeout bounds the describe fan-out in tests where a node never
-// answers, so they wait this long instead of the 3s production deadline. Local
-// replies land in milliseconds, leaving wide margin under -race.
-const silentNodeFanoutTimeout = 500 * time.Millisecond
-
 // replyAsNode answers every request on subject with reply, attributed to node.
 func replyAsNode(t *testing.T, nc *nats.Conn, subject, node string, reply any) {
 	t.Helper()
@@ -238,7 +231,6 @@ func replyAsNode(t *testing.T, nc *nats.Conn, subject, node string, reply any) {
 // Counting replies reads that as every node heard from and the instance as
 // absent, waving the retype past the cap; the gate must see node-2 is missing.
 func TestNATSInstanceTypeResolver_RefusesWhenOneNodeAnswersTwiceAndAnotherIsSilent(t *testing.T) {
-	t.Parallel()
 	s := newVCPUService(t, Limits{Enabled: true, VCPUs: 8})
 	if err := s.AddVCPU(t.Context(), testAccount, 7); err != nil {
 		t.Fatalf("seed AddVCPU: %v", err)
@@ -252,8 +244,7 @@ func TestNATSInstanceTypeResolver_RefusesWhenOneNodeAnswersTwiceAndAnotherIsSile
 		t.Fatalf("flush: %v", err)
 	}
 
-	resolve := NATSInstanceTypeResolver(nc, func() []string { return []string{"node-1", "node-2"} },
-		ec2instanceapi.WithFanoutTimeout(silentNodeFanoutTimeout))
+	resolve := NATSInstanceTypeResolver(nc, func() []string { return []string{"node-1", "node-2"} })
 	delta, err := s.EnforceRetype(t.Context(), resolve, testAccount, "i-on-node2", "m5.xlarge")
 	if err == nil {
 		t.Fatalf("EnforceRetype = (%d, nil), want a refusal while node-2 is silent", delta)
@@ -267,7 +258,6 @@ func TestNATSInstanceTypeResolver_RefusesWhenOneNodeAnswersTwiceAndAnotherIsSile
 // With every node answering, the resolver finds the instance on node-2 and the
 // gate judges the retype against the cap.
 func TestNATSInstanceTypeResolver_FindsInstanceWhenEveryNodeAnswers(t *testing.T) {
-	t.Parallel()
 	s := newVCPUService(t, Limits{Enabled: true, VCPUs: 8})
 	if err := s.AddVCPU(t.Context(), testAccount, 7); err != nil {
 		t.Fatalf("seed AddVCPU: %v", err)
@@ -293,7 +283,6 @@ func TestNATSInstanceTypeResolver_FindsInstanceWhenEveryNodeAnswers(t *testing.T
 // An instance absent from a complete listing charges nothing and returns no
 // error, so the daemon rejects the modify with the AWS NotFound error.
 func TestNATSInstanceTypeResolver_AbsentOnCompleteListingDefersToDaemon(t *testing.T) {
-	t.Parallel()
 	s := newVCPUService(t, Limits{Enabled: true, VCPUs: 8})
 	if err := s.AddVCPU(t.Context(), testAccount, 7); err != nil {
 		t.Fatalf("seed AddVCPU: %v", err)
@@ -317,7 +306,6 @@ func TestNATSInstanceTypeResolver_AbsentOnCompleteListingDefersToDaemon(t *testi
 // A retype targets a stopped instance, which the shared stopped bucket serves
 // whatever the node fan-out does, so a silent node must not block judging it.
 func TestNATSInstanceTypeResolver_FindsStoppedInstanceWhileANodeIsSilent(t *testing.T) {
-	t.Parallel()
 	s := newVCPUService(t, Limits{Enabled: true, VCPUs: 8})
 	if err := s.AddVCPU(t.Context(), testAccount, 7); err != nil {
 		t.Fatalf("seed AddVCPU: %v", err)
@@ -332,8 +320,7 @@ func TestNATSInstanceTypeResolver_FindsStoppedInstanceWhileANodeIsSilent(t *test
 		t.Fatalf("flush: %v", err)
 	}
 
-	resolve := NATSInstanceTypeResolver(nc, func() []string { return []string{"node-1", "node-2"} },
-		ec2instanceapi.WithFanoutTimeout(silentNodeFanoutTimeout))
+	resolve := NATSInstanceTypeResolver(nc, func() []string { return []string{"node-1", "node-2"} })
 	_, err := s.EnforceRetype(t.Context(), resolve, testAccount, "i-stopped", "m5.xlarge")
 	if err == nil || err.Error() != awserrors.ErrorResourceLimitExceeded {
 		t.Fatalf("EnforceRetype err = %v, want %q", err, awserrors.ErrorResourceLimitExceeded)

spinifex/domains/dns/reconcile.go

@@ -169,24 +169,25 @@ func (r *Reconciler) Run(ctx context.Context) {
 		// No revisit deadline: the desired set is a function of the buckets
 		// being watched, so a change is the only reason to run early.
 		Reconcile: func(ctx context.Context) (time.Duration, error) {
-			return r.reconcileOnce(ctx), nil
+			r.reconcileOnce(ctx)
+			return 0, nil
 		},
 		Resync: r.interval,
 	})
 }
 
 // reconcileOnce computes the converging batch and applies it, on whichever node
-// wins this cycle's election, and returns the election's revisit when it lost. The read that finds the stale records and the
+// wins this cycle's election. The read that finds the stale records and the
 // write that removes them now happen in one process, so a record re-added
 // between them is no longer deleted by a batch that predates it.
-func (r *Reconciler) reconcileOnce(ctx context.Context) time.Duration {
+func (r *Reconciler) reconcileOnce(ctx context.Context) {
 	if !r.enabled {
-		return 0
+		return
 	}
 	if r.lease != nil {
-		release, retry, elected := reconcilelock.ElectPass(ctx, r.lease)
+		release, elected := reconcilelock.AcquireLeader(ctx, r.lease)
 		if !elected {
-			return retry
+			return
 		}
 		defer release()
 	}
@@ -195,19 +196,18 @@ func (r *Reconciler) reconcileOnce(ctx context.Context) time.Duration {
 		// A corrupt zone is handled in readZone; anything reaching here left the
 		// desired state unapplied, so surface it rather than burying it in WARN.
 		slog.Error("dns reconcile: compute batch failed, retrying next cycle", "error", err)
-		return 0
+		return
 	}
 	if len(batch) == 0 {
-		return 0
+		return
 	}
 	slog.Debug("dns reconcile: converging", "changes", len(batch))
 	res, err := r.writer.ApplyBatch(&ChangeBatch{Changes: batch})
 	if err != nil {
 		slog.Warn("dns reconcile: apply failed, retrying next cycle", "changes", len(batch), "error", err)
-		return 0
+		return
 	}
 	slog.Debug("dns reconcile: converged", "changes", res.Applied, "zones", res.Zones)
-	return 0
 }
 
 // computeBatchBounded runs computeBatch with a deadline, so a pass that never

spinifex/domains/dns/reconcile_election_test.go

@@ -46,12 +46,10 @@ func TestReconcileOnce_ALoserOfTheElectionDoesNotWriteTheZone(t *testing.T) {
 	r := electingReconcilerFor(w, dnsLease(js, "node1"), func() DesiredSet {
 		return DesiredSet{Changes: []Change{upsert("lb-1.elb."+testBase, "10.200.1.9")}}
 	})
-	revisit := r.reconcileOnce(t.Context())
+	r.reconcileOnce(t.Context())
 
 	assert.Equal(t, before, objects[testBase+".toml"],
 		"a node that did not win the election must leave the zone object exactly as it found it")
-	assert.Positive(t, revisit, "the loser asks to retry so a dead holder is taken over")
-	assert.LessOrEqual(t, revisit, reconcilelock.LeaderTTL/3, "the retry comes within a renewal interval")
 
 	release()
 }

spinifex/domains/ec2/awsapi/actions.go

@@ -202,7 +202,7 @@ var ec2Actions = map[string]ec2Action{
 		return ec2instanceapi.DescribeInstanceTypes(ctx, input, c.NATS, c.ExpectedNodes, c.NodeIDs, accountID)
 	}),
 	"DescribeInstanceTypeOfferings": ec2Handler(func(ctx context.Context, input *ec2.DescribeInstanceTypeOfferingsInput, c *call, accountID string) (any, error) {
-		return ec2instanceapi.DescribeInstanceTypeOfferings(ctx, input, c.NATS, c.ExpectedNodes, accountID, c.Region, c.AZ, c.ZoneIDs)
+		return ec2instanceapi.DescribeInstanceTypeOfferings(ctx, input, c.NATS, c.ExpectedNodes, accountID, c.Region, c.AZ)
 	}),
 	"DescribeInstanceStatus": ec2Handler(func(ctx context.Context, input *ec2.DescribeInstanceStatusInput, c *call, accountID string) (any, error) {
 		return ec2instanceapi.DescribeInstanceStatus(ctx, input, c.NATS, c.ActiveNodes(ctx), accountID, c.AZ, c.InstanceStatus)
@@ -290,7 +290,7 @@ var ec2Actions = map[string]ec2Action{
 		return ec2zoneapi.DescribeRegions(input, c.Region, c.Endpoint)
 	}),
 	"DescribeAvailabilityZones": ec2Handler(func(ctx context.Context, input *ec2.DescribeAvailabilityZonesInput, c *call, accountID string) (any, error) {
-		return ec2zoneapi.DescribeAvailabilityZones(ctx, input, c.Region, c.AZ, c.ZoneIDs)
+		return ec2zoneapi.DescribeAvailabilityZones(input, c.Region, c.AZ)
 	}),
 	"DescribeVolumes": ec2Handler(func(ctx context.Context, input *ec2.DescribeVolumesInput, c *call, accountID string) (any, error) {
 		if err := ec2volumeapi.ValidateDescribeVolumesInput(input); err != nil {

spinifex/domains/ec2/awsapi/deps.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/aws/aws-sdk-go/service/ec2"
 	ec2instanceapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/instance"
-	ec2zoneapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/zone"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 )
@@ -38,8 +37,6 @@ type Deps struct {
 	ExpectedNodes int
 	NodeIDs       []string
 	AZ            string
-	// ZoneIDs supplies the pinned availability-zone IDs every zone-reporting call reads.
-	ZoneIDs ec2zoneapi.IDSource
 	// Endpoint is the gateway's own dialable base URL DescribeRegions advertises.
 	Endpoint       string
 	InstanceStatus ec2instanceapi.StatusSynthesis

spinifex/domains/ec2/awsapi/dispatch_test.go

@@ -16,7 +16,6 @@ import (
 	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"github.com/mulgadc/spinifex/spinifex/ingress/aws/dispatch"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
@@ -70,15 +69,9 @@ func formRequest(body string) *http.Request {
 func newTestHandler(t *testing.T) *Handler {
 	t.Helper()
 	_, nc, _ := testutil.StartTestJetStream(t)
-	// A daemon that answers every EC2 subject with an empty result as the one
-	// configured node, so each action reaches its domain call and returns, and
-	// identity fan-outs settle on its reply instead of waiting out the deadline.
-	sub, err := nc.Subscribe("ec2.>", func(m *nats.Msg) {
-		reply := nats.NewMsg(m.Reply)
-		reply.Header.Set(natsmsg.NodeIDHeader, "node-1")
-		reply.Data = []byte("{}")
-		_ = m.RespondMsg(reply)
-	})
+	// A daemon that answers every EC2 subject with an empty result, so each
+	// action reaches its domain call and returns instead of waiting.
+	sub, err := nc.Subscribe("ec2.>", func(m *nats.Msg) { _ = m.Respond([]byte("{}")) })
 	require.NoError(t, err)
 	t.Cleanup(func() { _ = sub.Unsubscribe() })
 

spinifex/domains/ec2/awsapi/instance/DescribeAccountInstancesStrict_test.go

@@ -38,16 +38,8 @@ func subscribeAccountRecords(t *testing.T, nc *nats.Conn, data []byte) {
 
 const strictAccount = "123456789012"
 
-// strictFanoutWindow is the collect window: strict mode always waits it out,
-// so it is short, yet still far above a loopback reply's round trip.
-const strictFanoutWindow = 500 * time.Millisecond
-
 func strictList(nc *nats.Conn, nodes ...string) (*ec2.DescribeInstancesOutput, error) {
-	return strictListWithin(nc, strictFanoutWindow, nodes...)
-}
-
-func strictListWithin(nc *nats.Conn, window time.Duration, nodes ...string) (*ec2.DescribeInstancesOutput, error) {
-	return DescribeAccountInstancesStrict(context.Background(), nc, nodes, strictAccount, WithFanoutTimeout(window))
+	return DescribeAccountInstancesStrict(context.Background(), nc, nodes, strictAccount, WithFanoutTimeout(2*time.Second))
 }
 
 // Existence comes from the record store, so an instance no node holds is still
@@ -147,7 +139,7 @@ func TestDescribeAccountInstancesStrict_LateFrameFromUnconfiguredNodeFails(t *te
 	subscribeAsNode(t, nc, "ec2.DescribeInstances", "node-1", []byte(`{}`))
 	late := mustMarshalDescribeOutput(t, strictReservations(strictInstance("i-on-stray", "running")))
 	_, err := nc.Subscribe("ec2.DescribeInstances", func(msg *nats.Msg) {
-		time.AfterFunc(250*time.Millisecond, func() {
+		time.AfterFunc(600*time.Millisecond, func() {
 			reply := nats.NewMsg(msg.Reply)
 			reply.Data = late
 			reply.Header.Set(natsmsg.NodeIDHeader, "node-stray")
@@ -157,7 +149,7 @@ func TestDescribeAccountInstancesStrict_LateFrameFromUnconfiguredNodeFails(t *te
 	require.NoError(t, err)
 	nc.Flush()
 
-	out, err := strictListWithin(nc, time.Second, "node-1")
+	out, err := strictList(nc, "node-1")
 
 	assertIncompleteListing(t, out, err)
 }

spinifex/domains/ec2/awsapi/instance/DescribeInstanceStatus.go

@@ -8,6 +8,7 @@ import (
 	"log/slog"
 	"slices"
 	"strconv"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -23,14 +24,10 @@ const (
 // also augments from the stopped-instance KV bucket. The aggregator propagates
 // the first deterministic 4xx only when no data was collected, matching
 // DescribeInstances.
-func DescribeInstanceStatus(ctx context.Context, input *ec2.DescribeInstanceStatusInput, natsConn *nats.Conn, expectedNodes int, accountID, az string, synthesis StatusSynthesis, opts ...DescribeOption) (*ec2.DescribeInstanceStatusOutput, error) {
+func DescribeInstanceStatus(ctx context.Context, input *ec2.DescribeInstanceStatusInput, natsConn *nats.Conn, expectedNodes int, accountID, az string, synthesis StatusSynthesis) (*ec2.DescribeInstanceStatusOutput, error) {
 	if input == nil {
 		input = &ec2.DescribeInstanceStatusInput{}
 	}
-	cfg := describeConfig{fanoutTimeout: defaultDescribeFanoutTimeout}
-	for _, opt := range opts {
-		opt(&cfg)
-	}
 
 	jsonData, err := json.Marshal(input)
 	if err != nil {
@@ -39,7 +36,7 @@ func DescribeInstanceStatus(ctx context.Context, input *ec2.DescribeInstanceStat
 	}
 
 	frames, sum, err := natsmsg.Gather(ctx, natsConn, "ec2.DescribeInstanceStatus", jsonData,
-		natsmsg.GatherOpts{Timeout: cfg.fanoutTimeout, ExpectedResponders: expectedNodes, AccountID: accountID})
+		natsmsg.GatherOpts{Timeout: 3 * time.Second, ExpectedResponders: expectedNodes, AccountID: accountID})
 	if err != nil {
 		return nil, err
 	}

spinifex/domains/ec2/awsapi/instance/DescribeInstanceTypeOfferings.go

@@ -26,7 +26,7 @@ var describeInstanceTypeOfferingsValidFilters = map[string]bool{
 // DescribeInstanceTypeOfferings projects the supported instance type catalogue
 // over the location list. Both inputs are read live on every call, so the answer
 // cannot advertise a type the platform is unable to place.
-func DescribeInstanceTypeOfferings(ctx context.Context, input *ec2.DescribeInstanceTypeOfferingsInput, natsConn *nats.Conn, expectedNodes int, accountID, region, az string, ids ec2zoneapi.IDSource) (*ec2.DescribeInstanceTypeOfferingsOutput, error) {
+func DescribeInstanceTypeOfferings(ctx context.Context, input *ec2.DescribeInstanceTypeOfferingsInput, natsConn *nats.Conn, expectedNodes int, accountID, region, az string) (*ec2.DescribeInstanceTypeOfferingsOutput, error) {
 	if input == nil {
 		input = &ec2.DescribeInstanceTypeOfferingsInput{}
 	}
@@ -36,7 +36,7 @@ func DescribeInstanceTypeOfferings(ctx context.Context, input *ec2.DescribeInsta
 		locationType = ec2.LocationTypeRegion
 	}
 
-	locations, err := offeringLocations(ctx, locationType, region, az, ids)
+	locations, err := offeringLocations(locationType, region, az)
 	if err != nil {
 		slog.WarnContext(ctx, "DescribeInstanceTypeOfferings: invalid location type", "location_type", locationType)
 		return nil, err
@@ -96,7 +96,7 @@ func DescribeInstanceTypeOfferings(ctx context.Context, input *ec2.DescribeInsta
 // offeringLocations resolves the locations an offering can be reported at. The
 // zone list is read rather than assumed, so the operation follows the zone model
 // if it ever widens.
-func offeringLocations(ctx context.Context, locationType, region, az string, ids ec2zoneapi.IDSource) ([]string, error) {
+func offeringLocations(locationType, region, az string) ([]string, error) {
 	switch locationType {
 	case ec2.LocationTypeRegion:
 		return []string{region}, nil
@@ -105,7 +105,7 @@ func offeringLocations(ctx context.Context, locationType, region, az string, ids
 		return nil, errors.New(awserrors.ErrorInvalidParameterValue)
 	}
 
-	zones, err := ec2zoneapi.DescribeAvailabilityZones(ctx, &ec2.DescribeAvailabilityZonesInput{}, region, az, ids)
+	zones, err := ec2zoneapi.DescribeAvailabilityZones(&ec2.DescribeAvailabilityZonesInput{}, region, az)
 	if err != nil {
 		return nil, err
 	}

spinifex/domains/ec2/awsapi/instance/DescribeInstanceTypeOfferings_test.go

@@ -46,12 +46,12 @@ func TestDescribeInstanceTypeOfferings_AvailabilityZone(t *testing.T) {
 
 	types, err := DescribeInstanceTypes(context.Background(), &ec2.DescribeInstanceTypesInput{}, nc, 1, nil, "")
 	require.NoError(t, err)
-	zones, err := ec2zoneapi.DescribeAvailabilityZones(context.Background(), &ec2.DescribeAvailabilityZonesInput{}, offeringsTestRegion, offeringsTestAZ, offeringsTestIDs)
+	zones, err := ec2zoneapi.DescribeAvailabilityZones(&ec2.DescribeAvailabilityZonesInput{}, offeringsTestRegion, offeringsTestAZ)
 	require.NoError(t, err)
 
 	output, err := DescribeInstanceTypeOfferings(context.Background(), &ec2.DescribeInstanceTypeOfferingsInput{
 		LocationType: aws.String(ec2.LocationTypeAvailabilityZone),
-	}, nc, 1, "", offeringsTestRegion, offeringsTestAZ, offeringsTestIDs)
+	}, nc, 1, "", offeringsTestRegion, offeringsTestAZ)
 	require.NoError(t, err)
 
 	var want []*ec2.InstanceTypeOffering
@@ -76,7 +76,7 @@ func TestDescribeInstanceTypeOfferings_Region(t *testing.T) {
 
 	output, err := DescribeInstanceTypeOfferings(context.Background(), &ec2.DescribeInstanceTypeOfferingsInput{
 		LocationType: aws.String(ec2.LocationTypeRegion),
-	}, nc, 1, "", offeringsTestRegion, offeringsTestAZ, offeringsTestIDs)
+	}, nc, 1, "", offeringsTestRegion, offeringsTestAZ)
 	require.NoError(t, err)
 
 	require.Len(t, output.InstanceTypeOfferings, 2)
@@ -93,7 +93,7 @@ func TestDescribeInstanceTypeOfferings_DefaultLocationTypeIsRegion(t *testing.T)
 	_, nc := startTestNATSServer(t)
 	serveInstanceTypes(t, nc, "t3.micro")
 
-	output, err := DescribeInstanceTypeOfferings(context.Background(), &ec2.DescribeInstanceTypeOfferingsInput{}, nc, 1, "", offeringsTestRegion, offeringsTestAZ, offeringsTestIDs)
+	output, err := DescribeInstanceTypeOfferings(context.Background(), &ec2.DescribeInstanceTypeOfferingsInput{}, nc, 1, "", offeringsTestRegion, offeringsTestAZ)
 	require.NoError(t, err)
 
 	require.Len(t, output.InstanceTypeOfferings, 1)
@@ -106,14 +106,14 @@ func TestDescribeInstanceTypeOfferings_AvailabilityZoneID(t *testing.T) {
 	_, nc := startTestNATSServer(t)
 	serveInstanceTypes(t, nc, "t3.micro")
 
-	zones, err := ec2zoneapi.DescribeAvailabilityZones(context.Background(), &ec2.DescribeAvailabilityZonesInput{}, offeringsTestRegion, offeringsTestAZ, offeringsTestIDs)
+	zones, err := ec2zoneapi.DescribeAvailabilityZones(&ec2.DescribeAvailabilityZonesInput{}, offeringsTestRegion, offeringsTestAZ)
 	require.NoError(t, err)
 	zoneID := aws.StringValue(zones.AvailabilityZones[0].ZoneId)
 	require.NotEqual(t, offeringsTestAZ, zoneID, "zone ID must differ from zone name for this assertion to mean anything")
 
 	output, err := DescribeInstanceTypeOfferings(context.Background(), &ec2.DescribeInstanceTypeOfferingsInput{
 		LocationType: aws.String(ec2.LocationTypeAvailabilityZoneId),
-	}, nc, 1, "", offeringsTestRegion, offeringsTestAZ, offeringsTestIDs)
+	}, nc, 1, "", offeringsTestRegion, offeringsTestAZ)
 	require.NoError(t, err)
 
 	require.Len(t, output.InstanceTypeOfferings, 1)
@@ -127,7 +127,7 @@ func TestDescribeInstanceTypeOfferings_UnknownLocationType(t *testing.T) {
 
 	_, err := DescribeInstanceTypeOfferings(context.Background(), &ec2.DescribeInstanceTypeOfferingsInput{
 		LocationType: aws.String("outpost"),
-	}, nc, 1, "", offeringsTestRegion, offeringsTestAZ, offeringsTestIDs)
+	}, nc, 1, "", offeringsTestRegion, offeringsTestAZ)
 
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorInvalidParameterValue, err.Error())
@@ -177,7 +177,7 @@ func TestDescribeInstanceTypeOfferings_Filters(t *testing.T) {
 			output, err := DescribeInstanceTypeOfferings(context.Background(), &ec2.DescribeInstanceTypeOfferingsInput{
 				LocationType: aws.String(ec2.LocationTypeAvailabilityZone),
 				Filters:      tc.filters,
-			}, nc, 1, "", offeringsTestRegion, offeringsTestAZ, offeringsTestIDs)
+			}, nc, 1, "", offeringsTestRegion, offeringsTestAZ)
 			require.NoError(t, err)
 
 			var got []string
@@ -198,7 +198,7 @@ func TestDescribeInstanceTypeOfferings_UnknownFilter(t *testing.T) {
 	_, err := DescribeInstanceTypeOfferings(context.Background(), &ec2.DescribeInstanceTypeOfferingsInput{
 		LocationType: aws.String(ec2.LocationTypeAvailabilityZone),
 		Filters:      []*ec2.Filter{{Name: aws.String("not-a-filter"), Values: []*string{aws.String("x")}}},
-	}, nc, 1, "", offeringsTestRegion, offeringsTestAZ, offeringsTestIDs)
+	}, nc, 1, "", offeringsTestRegion, offeringsTestAZ)
 
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorInvalidParameterValue, err.Error())
@@ -221,7 +221,7 @@ func TestDescribeInstanceTypeOfferings_FullTraversal(t *testing.T) {
 			LocationType: aws.String(ec2.LocationTypeAvailabilityZone),
 			MaxResults:   aws.Int64(2),
 			NextToken:    token,
-		}, nc, 1, "", offeringsTestRegion, offeringsTestAZ, offeringsTestIDs)
+		}, nc, 1, "", offeringsTestRegion, offeringsTestAZ)
 		require.NoError(t, err)
 
 		for _, offering := range output.InstanceTypeOfferings {
@@ -252,7 +252,7 @@ func TestDescribeInstanceTypeOfferings_MalformedNextToken(t *testing.T) {
 	_, err := DescribeInstanceTypeOfferings(context.Background(), &ec2.DescribeInstanceTypeOfferingsInput{
 		LocationType: aws.String(ec2.LocationTypeAvailabilityZone),
 		NextToken:    aws.String("not-a-token"),
-	}, nc, 1, "", offeringsTestRegion, offeringsTestAZ, offeringsTestIDs)
+	}, nc, 1, "", offeringsTestRegion, offeringsTestAZ)
 
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorInvalidParameterValue, err.Error())
@@ -268,7 +268,7 @@ func TestDescribeInstanceTypeOfferings_TokenPastEnd(t *testing.T) {
 	output, err := DescribeInstanceTypeOfferings(context.Background(), &ec2.DescribeInstanceTypeOfferingsInput{
 		LocationType: aws.String(ec2.LocationTypeAvailabilityZone),
 		NextToken:    aws.String("99"),
-	}, nc, 1, "", offeringsTestRegion, offeringsTestAZ, offeringsTestIDs)
+	}, nc, 1, "", offeringsTestRegion, offeringsTestAZ)
 
 	require.NoError(t, err)
 	assert.Empty(t, output.InstanceTypeOfferings)
@@ -281,7 +281,7 @@ func TestDescribeInstanceTypeOfferings_EmptyCatalogue(t *testing.T) {
 
 	output, err := DescribeInstanceTypeOfferings(context.Background(), &ec2.DescribeInstanceTypeOfferingsInput{
 		LocationType: aws.String(ec2.LocationTypeAvailabilityZone),
-	}, nc, 0, "", offeringsTestRegion, offeringsTestAZ, offeringsTestIDs)
+	}, nc, 0, "", offeringsTestRegion, offeringsTestAZ)
 
 	require.NoError(t, err)
 	require.NotNil(t, output.InstanceTypeOfferings)
@@ -299,7 +299,7 @@ func TestDescribeInstanceTypeOfferings_CatalogueError(t *testing.T) {
 
 	_, err = DescribeInstanceTypeOfferings(context.Background(), &ec2.DescribeInstanceTypeOfferingsInput{
 		LocationType: aws.String(ec2.LocationTypeAvailabilityZone),
-	}, closedNC, 1, "", offeringsTestRegion, offeringsTestAZ, offeringsTestIDs)
+	}, closedNC, 1, "", offeringsTestRegion, offeringsTestAZ)
 
 	require.Error(t, err)
 }
@@ -309,16 +309,9 @@ func TestDescribeInstanceTypeOfferings_NilInput(t *testing.T) {
 	_, nc := startTestNATSServer(t)
 	serveInstanceTypes(t, nc, "t3.micro")
 
-	output, err := DescribeInstanceTypeOfferings(context.Background(), nil, nc, 1, "", offeringsTestRegion, offeringsTestAZ, offeringsTestIDs)
+	output, err := DescribeInstanceTypeOfferings(context.Background(), nil, nc, 1, "", offeringsTestRegion, offeringsTestAZ)
 
 	require.NoError(t, err)
 	require.Len(t, output.InstanceTypeOfferings, 1)
 	assert.Equal(t, offeringsTestRegion, aws.StringValue(output.InstanceTypeOfferings[0].Location))
 }
-
-// stubZoneIDs answers one fixed ID for every zone.
-type stubZoneIDs string
-
-func (s stubZoneIDs) ZoneID(context.Context, string) (string, error) { return string(s), nil }
-
-var offeringsTestIDs ec2zoneapi.IDSource = stubZoneIDs("apse2-az1")

spinifex/domains/ec2/awsapi/instance/RunInstances.go

@@ -96,12 +96,6 @@ func ValidateRunInstancesInput(input *ec2.RunInstancesInput) (err error) {
 		return errors.New(awserrors.ErrorInvalidParameterValue)
 	}
 
-	// A volume with no device name cannot be attached anywhere, so the mapping
-	// is refused as AWS refuses an invalid device.
-	if ec2instance.UnnamedBlockDeviceMapping(input.BlockDeviceMappings) {
-		return awserrors.Errorf(awserrors.ErrorInvalidBlockDeviceMapping, "BlockDeviceMappings: a mapping names no device")
-	}
-
 	// A volume the launch creates beyond the root is empty and sized by the
 	// request, so a mapping asking for anything else is refused by name rather
 	// than served as something it did not ask for.

spinifex/domains/ec2/awsapi/instance/RunInstances_test.go

@@ -251,15 +251,6 @@ func TestParseRunInstances(t *testing.T) {
 		}
 	})
 
-	t.Run("MappingNamingNoDeviceIsRefused", func(t *testing.T) {
-		t.Parallel()
-		input := defaults
-		input.BlockDeviceMappings = []*ec2.BlockDeviceMapping{{Ebs: &ec2.EbsBlockDevice{VolumeSize: aws.Int64(6)}}}
-		err := ValidateRunInstancesInput(&input)
-		require.Error(t, err)
-		assert.Contains(t, err.Error(), awserrors.ErrorInvalidBlockDeviceMapping)
-	})
-
 	t.Run("MaxCountAtInt32IsValid", func(t *testing.T) {
 		t.Parallel()
 		input := defaults

spinifex/domains/ec2/awsapi/instance/placement_spread_test.go

@@ -180,19 +180,8 @@ func TestDistributeInstancesSpread_NoEligibleNodes(t *testing.T) {
 
 	// Capacity exists for a different instance type only, so nothing is eligible.
 	h := mockSpreadCluster(t, nc, map[string]int{}, nil, nil, "")
-	otherTypeSub, err := nc.Subscribe("spinifex.node.status", func(msg *nats.Msg) {
-		for _, node := range []string{"node-1", "node-2"} {
-			data, _ := json.Marshal(types.NodeStatusResponse{
-				Node:          node,
-				InstanceTypes: []types.InstanceTypeCap{{Name: "m5.large", Available: 4}},
-			})
-			_ = nc.Publish(msg.Reply, data)
-		}
-	})
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = otherTypeSub.Unsubscribe() })
 
-	_, err = distributeInstancesSpread(context.Background(), spreadInput(2, 2), nc, "test-account", "my-spread", 2)
+	_, err := distributeInstancesSpread(context.Background(), spreadInput(2, 2), nc, "test-account", "my-spread", 2)
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, err.Error())
 

spinifex/domains/ec2/awsapi/instance/status_synthesis_test.go

@@ -22,10 +22,6 @@ import (
 
 const synthAccount = "123456789012"
 
-// synthSilentNodeWindow bounds a fan-out that must time out because an expected
-// node never identifies itself; the answering replies are local and immediate.
-const synthSilentNodeWindow = 500 * time.Millisecond
-
 // fakeRecords is a cache stand-in: a fixed record set plus a readiness flag,
 // so a test can assert the cold-cache case as well as the warm one.
 type fakeRecords struct {
@@ -346,8 +342,7 @@ func TestSynthesis_SilentButLiveNodeShowsInsufficientData(t *testing.T) {
 		}},
 	}
 
-	out, err := DescribeInstanceStatus(context.Background(), &ec2.DescribeInstanceStatusInput{}, nc, 2, synthAccount, "az-a", synth,
-		WithFanoutTimeout(synthSilentNodeWindow))
+	out, err := DescribeInstanceStatus(context.Background(), &ec2.DescribeInstanceStatusInput{}, nc, 2, synthAccount, "az-a", synth)
 	require.NoError(t, err)
 
 	byID := statusByID(out)
@@ -423,8 +418,7 @@ func TestSynthesis_UnidentifiedFrameFallsBackToHeartbeatOnly(t *testing.T) {
 		}},
 	}
 
-	out, err := DescribeInstanceStatus(context.Background(), &ec2.DescribeInstanceStatusInput{}, nc, 1, synthAccount, "az-a", synth,
-		WithFanoutTimeout(synthSilentNodeWindow))
+	out, err := DescribeInstanceStatus(context.Background(), &ec2.DescribeInstanceStatusInput{}, nc, 1, synthAccount, "az-a", synth)
 	require.NoError(t, err)
 	require.Len(t, out.InstanceStatuses, 1,
 		"without responder identity, a live node's instance is skipped by the fallback rule")

spinifex/domains/ec2/awsapi/instance/zoneids_agree_test.go

@@ -1,61 +0,0 @@
-package instance
-
-import (
-	"testing"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/ec2"
-	testutil "github.com/mulgadc/spinifex/internal/testkit"
-	ec2zoneapi "github.com/mulgadc/spinifex/spinifex/domains/ec2/awsapi/zone"
-	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/zoneid"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// With a non-default override in force, the three calls that report a zone ID
-// read one source and so cannot disagree.
-func TestZoneIDs_ThreeProducersAgreeUnderAnOverride(t *testing.T) {
-	const region, az, account = "mulga-lab", "mulga-laba", "123456789012"
-	_, nc, js := testutil.StartTestJetStream(t)
-	testutil.StubVpcdSGResponder(t, nc)
-	serveInstanceTypes(t, nc, "t3.micro")
-
-	ids, err := zoneid.NewResolver(js, region, "ov99")
-	require.NoError(t, err)
-
-	vpcSvc, err := ec2vpc.NewVPCServiceImplWithNATS(t.Context(), nil, nc)
-	require.NoError(t, err)
-	vpcSvc.SetZoneIDSource(ids)
-	vpc, err := vpcSvc.CreateVpc(t.Context(), &ec2.CreateVpcInput{CidrBlock: aws.String("10.0.0.0/16")}, account)
-	require.NoError(t, err)
-	_, err = vpcSvc.CreateSubnet(t.Context(), &ec2.CreateSubnetInput{
-		VpcId: vpc.Vpc.VpcId, CidrBlock: aws.String("10.0.1.0/24"), AvailabilityZone: aws.String(az),
-	}, account)
-	require.NoError(t, err)
-
-	zones, err := ec2zoneapi.DescribeAvailabilityZones(t.Context(), &ec2.DescribeAvailabilityZonesInput{}, region, az, ids)
-	require.NoError(t, err)
-	require.Len(t, zones.AvailabilityZones, 1)
-	fromZones := aws.StringValue(zones.AvailabilityZones[0].ZoneId)
-
-	subnets, err := vpcSvc.DescribeSubnets(t.Context(), &ec2.DescribeSubnetsInput{}, account)
-	require.NoError(t, err)
-	var fromSubnet string
-	for _, s := range subnets.Subnets {
-		if aws.StringValue(s.AvailabilityZone) == az {
-			fromSubnet = aws.StringValue(s.AvailabilityZoneId)
-		}
-	}
-
-	offerings, err := DescribeInstanceTypeOfferings(t.Context(), &ec2.DescribeInstanceTypeOfferingsInput{
-		LocationType: aws.String(ec2.LocationTypeAvailabilityZoneId),
-	}, nc, 1, account, region, az, ids)
-	require.NoError(t, err)
-	require.NotEmpty(t, offerings.InstanceTypeOfferings)
-	fromOfferings := aws.StringValue(offerings.InstanceTypeOfferings[0].Location)
-
-	assert.Equal(t, "ov99-az1", fromZones)
-	assert.Equal(t, fromZones, fromSubnet)
-	assert.Equal(t, fromZones, fromOfferings)
-}

spinifex/domains/ec2/awsapi/zone/AvailabilityZone.go

@@ -4,24 +4,16 @@
 package zone
 
 import (
-	"context"
 	"slices"
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	filterutil "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/zoneid"
 )
 
-// IDSource returns the stored availability-zone ID for a zone name, pinning it first if needed.
-type IDSource interface {
-	ZoneID(ctx context.Context, az string) (string, error)
-}
-
-var _ IDSource = (*zoneid.Resolver)(nil)
-
 // The filter names AWS accepts on each call; any other, tag filters included,
 // is refused.
 var (
@@ -47,25 +39,18 @@ var (
 
 // DescribeAvailabilityZones implements the EC2 action for the single zone az in region. Unknown
 // ZoneNames or ZoneIds return InvalidParameterValue; non-matching filters yield an empty list.
-func DescribeAvailabilityZones(ctx context.Context, input *ec2.DescribeAvailabilityZonesInput, region string, az string, ids IDSource) (output *ec2.DescribeAvailabilityZonesOutput, err error) {
+func DescribeAvailabilityZones(input *ec2.DescribeAvailabilityZonesInput, region string, az string) (output *ec2.DescribeAvailabilityZonesOutput, err error) {
 	filters, err := parseFilters(input.Filters, describeAvailabilityZonesValidFilters)
 	if err != nil {
 		return nil, err
 	}
-	if ids == nil {
-		return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "availability zone ids are not configured")
-	}
-	zoneID, err := ids.ZoneID(ctx, az)
-	if err != nil {
-		return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "availability zone id: %v", err)
-	}
 
 	zone := &ec2.AvailabilityZone{
 		State:              aws.String("available"),
 		OptInStatus:        aws.String("opt-in-not-required"),
 		RegionName:         aws.String(region),
 		ZoneName:           aws.String(az),
-		ZoneId:             aws.String(zoneID),
+		ZoneId:             aws.String(ec2vpc.SingleZoneID),
 		GroupName:          aws.String(region),
 		NetworkBorderGroup: aws.String(region),
 		ZoneType:           aws.String("availability-zone"),
@@ -79,7 +64,7 @@ func DescribeAvailabilityZones(ctx context.Context, input *ec2.DescribeAvailabil
 		}
 	}
 	for _, id := range aws.StringValueSlice(input.ZoneIds) {
-		if id != zoneID {
+		if id != ec2vpc.SingleZoneID {
 			return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "Invalid availability zone-id: [%s]", id)
 		}
 	}

spinifex/domains/ec2/awsapi/zone/zone_test.go

@@ -3,9 +3,6 @@ package zone
 import (
 	"testing"
 
-	testutil "github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/zoneid"
-
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
@@ -13,39 +10,28 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func newIDs(t *testing.T, region, override string) *zoneid.Resolver {
-	t.Helper()
-	_, _, js := testutil.StartTestJetStream(t)
-	r, err := zoneid.NewResolver(js, region, override)
-	require.NoError(t, err)
-	return r
-}
-
 func TestDescribeAvailabilityZones(t *testing.T) {
 	tests := []struct {
 		name   string
 		region string
 		az     string
-		zoneID string
 	}{
 		{
 			name:   "Sydney",
 			region: "ap-southeast-2",
 			az:     "ap-southeast-2a",
-			zoneID: "apse2-az1",
 		},
 		{
 			name:   "US East",
 			region: "us-east-1",
 			az:     "us-east-1a",
-			zoneID: "use1-az1",
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			input := &ec2.DescribeAvailabilityZonesInput{}
-			output, err := DescribeAvailabilityZones(t.Context(), input, tt.region, tt.az, newIDs(t, tt.region, ""))
+			output, err := DescribeAvailabilityZones(input, tt.region, tt.az)
 
 			require.NoError(t, err)
 			require.NotNil(t, output)
@@ -56,7 +42,7 @@ func TestDescribeAvailabilityZones(t *testing.T) {
 			assert.Equal(t, "opt-in-not-required", *zone.OptInStatus)
 			assert.Equal(t, tt.region, *zone.RegionName)
 			assert.Equal(t, tt.az, *zone.ZoneName)
-			assert.Equal(t, tt.zoneID, *zone.ZoneId)
+			assert.Equal(t, "spinifexz1", *zone.ZoneId)
 			assert.Equal(t, tt.region, *zone.GroupName)
 			assert.Equal(t, tt.region, *zone.NetworkBorderGroup)
 			assert.Equal(t, "availability-zone", *zone.ZoneType)
@@ -65,14 +51,6 @@ func TestDescribeAvailabilityZones(t *testing.T) {
 	}
 }
 
-func TestDescribeAvailabilityZones_ZoneIDsSelectBySydneyID(t *testing.T) {
-	ids := newIDs(t, "ap-southeast-2", "")
-	out, err := DescribeAvailabilityZones(t.Context(), &ec2.DescribeAvailabilityZonesInput{ZoneIds: aws.StringSlice([]string{"apse2-az1"})}, "ap-southeast-2", "ap-southeast-2a", ids)
-	require.NoError(t, err)
-	require.Len(t, out.AvailabilityZones, 1)
-	assert.Equal(t, "ap-southeast-2a", aws.StringValue(out.AvailabilityZones[0].ZoneName))
-}
-
 func TestDescribeRegions(t *testing.T) {
 	tests := []struct {
 		name     string
@@ -126,7 +104,6 @@ func requireInvalidParameterValue(t *testing.T, err error, msg string) {
 // is as valid as an AWS one. The messages are AWS's own.
 func TestDescribeAvailabilityZones_Selection(t *testing.T) {
 	const region, az = "mulga-west", "mulga-west-a"
-	ids := newIDs(t, region, "")
 
 	for _, tc := range []struct {
 		name  string
@@ -134,7 +111,7 @@ func TestDescribeAvailabilityZones_Selection(t *testing.T) {
 		want  int
 	}{
 		{"zone name", &ec2.DescribeAvailabilityZonesInput{ZoneNames: aws.StringSlice([]string{az})}, 1},
-		{"zone id", &ec2.DescribeAvailabilityZonesInput{ZoneIds: aws.StringSlice([]string{"muw1-az1"})}, 1},
+		{"zone id", &ec2.DescribeAvailabilityZonesInput{ZoneIds: aws.StringSlice([]string{"spinifexz1"})}, 1},
 		{"zone-name filter", &ec2.DescribeAvailabilityZonesInput{Filters: filter("zone-name", az)}, 1},
 		{"wildcard filter", &ec2.DescribeAvailabilityZonesInput{Filters: filter("region-name", "mulga-*")}, 1},
 		{"state filter", &ec2.DescribeAvailabilityZonesInput{Filters: filter("state", "available")}, 1},
@@ -142,7 +119,7 @@ func TestDescribeAvailabilityZones_Selection(t *testing.T) {
 		{"filter on an absent field", &ec2.DescribeAvailabilityZonesInput{Filters: filter("parent-zone-id", "*")}, 0},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
-			out, err := DescribeAvailabilityZones(t.Context(), tc.input, region, az, ids)
+			out, err := DescribeAvailabilityZones(tc.input, region, az)
 			require.NoError(t, err)
 			assert.Len(t, out.AvailabilityZones, tc.want)
 		})
@@ -157,11 +134,10 @@ func TestDescribeAvailabilityZones_Selection(t *testing.T) {
 		{"tag filter", &ec2.DescribeAvailabilityZonesInput{Filters: filter("tag:Name", "x")}, "The filter 'tag:Name' is invalid"},
 		{"network-border-group filter", &ec2.DescribeAvailabilityZonesInput{Filters: filter("network-border-group", "x")}, "The filter 'network-border-group' is invalid"},
 		{"first unknown name in request order", &ec2.DescribeAvailabilityZonesInput{ZoneNames: aws.StringSlice([]string{az, "zz-nowhere-9z", "yy-nowhere-8y"})}, "Invalid availability zone: [zz-nowhere-9z]"},
-		{"unknown zone id", &ec2.DescribeAvailabilityZonesInput{ZoneIds: aws.StringSlice([]string{"muw1-az1", "zz-az9"})}, "Invalid availability zone-id: [zz-az9]"},
-		{"the retired constant is an unknown zone id", &ec2.DescribeAvailabilityZonesInput{ZoneIds: aws.StringSlice([]string{"spinifexz1"})}, "Invalid availability zone-id: [spinifexz1]"},
+		{"unknown zone id", &ec2.DescribeAvailabilityZonesInput{ZoneIds: aws.StringSlice([]string{"spinifexz1", "zz-az9"})}, "Invalid availability zone-id: [zz-az9]"},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
-			_, err := DescribeAvailabilityZones(t.Context(), tc.input, region, az, ids)
+			_, err := DescribeAvailabilityZones(tc.input, region, az)
 			requireInvalidParameterValue(t, err, tc.msg)
 		})
 	}
@@ -204,9 +180,3 @@ func TestDescribeRegions_Selection(t *testing.T) {
 		})
 	}
 }
-
-func TestDescribeAvailabilityZones_NoIDSourceFailsClosed(t *testing.T) {
-	_, err := DescribeAvailabilityZones(t.Context(), &ec2.DescribeAvailabilityZonesInput{}, "mulga-lab", "mulga-laba", nil)
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), awserrors.ErrorServerInternal)
-}

spinifex/domains/ec2/ebs/metadata/metadata.go

@@ -110,6 +110,23 @@ type AMI struct {
 	Tags            map[string]string `json:"tags,omitempty"`
 	SourceDigest    *ImageDigest      `json:"source_digest,omitempty"`
 	State           string            `json:"state,omitempty"`
+	// UEFIData names the image's own UEFI variable store object; nil launches
+	// from the node's VARS template.
+	UEFIData *UEFIDataRef `json:"uefi_data,omitempty"`
+}
+
+// UEFIDataRef names an image's UEFI variable store object and the firmware it
+// was written under, so a launch can check the firmware without reading it.
+type UEFIDataRef struct {
+	Key      string       `json:"key"`
+	Firmware UEFIFirmware `json:"firmware"`
+}
+
+// UEFIFirmware is the firmware a variable store was written under.
+type UEFIFirmware struct {
+	Arch       string `json:"arch"`
+	VarsSize   int64  `json:"vars_size"`
+	CodeSHA256 string `json:"code_sha256"`
 }
 
 // ImageDigest identifies the artifact an AMI was imported from, not the

spinifex/domains/ec2/ebs/metadata/metadata_test.go

@@ -73,6 +73,16 @@ func TestAMIWithoutSourceDigestStillDecodes(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, "ami-1", got.ImageID)
 	assert.Nil(t, got.SourceDigest)
+	assert.Nil(t, got.UEFIData, "a document with no uefi_data launches from the template")
+}
+
+// The UEFI store reference rides at schema version 2 under its JSON names.
+func TestAMIUEFIDataDecodesAtSchemaVersion2(t *testing.T) {
+	got, err := UnmarshalAMI([]byte(`{"schema_version":2,"image_id":"ami-1","uefi_data":` +
+		`{"key":"uefi/ami-1/k","firmware":{"arch":"x86_64","vars_size":540672,"code_sha256":"abc"}}}`))
+	require.NoError(t, err)
+	require.NotNil(t, got.UEFIData)
+	assert.Equal(t, UEFIDataRef{Key: "uefi/ami-1/k", Firmware: UEFIFirmware{Arch: "x86_64", VarsSize: 540672, CodeSHA256: "abc"}}, *got.UEFIData)
 }
 
 func TestMetadataRejectsUnknownSchemaVersion(t *testing.T) {

spinifex/domains/ec2/image/service_impl.go

@@ -18,6 +18,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	ebsencryption "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/encryption"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
 	ec2platform "github.com/mulgadc/spinifex/spinifex/domains/ec2/platform"
 	ec2snapshot "github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
@@ -39,6 +40,9 @@ type CreateImageParams struct {
 	RootVolumeID  string
 	SourceImageID string
 	IsRunning     bool // logged only: the provider's CreateSnapshot drains a live volume itself
+	// UEFI is set for an instance that boots through UEFI, whose variable
+	// store the image carries.
+	UEFI bool
 }
 
 // ImageServiceImpl handles AMI image operations with S3 storage.
@@ -49,6 +53,9 @@ type ImageServiceImpl struct {
 	natsConn   *nats.Conn
 	metadata   *ebsmetadata.Store
 	provider   ebsprovider.EBSProvider
+
+	uefiRecords UEFIStoreRecords
+	nodeID      string
 }
 
 // SetEBSProvider injects the provider boundary used by the control plane.
@@ -444,12 +451,23 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 		"rootVolumeId", params.RootVolumeID, "amiId", amiID, "snapshotId", snapshotID,
 		"isRunning", params.IsRunning)
 
-	// Step 1: Snapshot root volume
+	// Step 1: Copy the instance's UEFI variable store to the image's own key,
+	// before the snapshot, so a refusal here leaves no snapshot behind.
+	var uefiData *ebsmetadata.UEFIDataRef
+	if params.UEFI {
+		var err error
+		uefiData, err = s.copyInstanceUEFIStore(context.Background(), *input.InstanceId, amiID)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Step 2: Snapshot root volume
 	if err := s.snapshotVolume(params.RootVolumeID, snapshotID, accountID); err != nil {
 		return nil, err
 	}
 
-	// Step 2: Read source volume config for size
+	// Step 3: Read source volume config for size
 	volMeta, err := s.getVolumeMetadata(context.Background(), accountID, params.RootVolumeID)
 	if err != nil {
 		slog.Error("CreateImageFromInstance: failed to read volume metadata", "volumeId", params.RootVolumeID, "err", err)
@@ -457,7 +475,7 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 	}
 	volumeSizeGiB := volMeta.CapacityGiB
 
-	// Step 3: Read source AMI config for architecture, platform, etc.
+	// Step 4: Read source AMI config for architecture, platform, etc.
 	sourceAMI := ebsmetadata.AMI{
 		Architecture:    "x86_64",
 		PlatformDetails: "Linux/UNIX",
@@ -472,13 +490,13 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 		sourceAMI = srcCfg
 	}
 
-	// Step 4: Store snapshot metadata
+	// Step 5: Store snapshot metadata
 	if err := s.putSnapshotMetadata(context.Background(), snapshotID, params.RootVolumeID, volumeSizeGiB, accountID); err != nil {
 		slog.Error("CreateImageFromInstance: failed to write snapshot metadata", "snapshotId", snapshotID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
-	// Step 5: Build and store AMI config
+	// Step 6: Build and store AMI config
 	meta := ebsmetadata.AMI{
 		ImageID:         amiID,
 		Name:            name,
@@ -497,10 +515,12 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 		Distro:       sourceAMI.Distro,
 		DistroFamily: sourceAMI.DistroFamily,
 		// Snapshot succeeded before this point, so the image is complete.
-		State: "available",
+		State:    "available",
+		UEFIData: uefiData,
 	}
 
 	if err := s.putAMIConfig(context.Background(), amiID, meta); err != nil {
+		// A UEFI object left under the image's prefix is in the orphan report.
 		slog.Error("CreateImageFromInstance: failed to store AMI config", "amiId", amiID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
@@ -856,6 +876,17 @@ func (s *ImageServiceImpl) CopyImage(ctx context.Context, input *ec2.CopyImageIn
 
 	tags := mergeCopyImageTags(srcMeta.Tags, input.TagSpecifications, aws.BoolValue(input.CopyImageTags))
 
+	uefiData, err := s.copyImageUEFIStore(ctx, srcMeta, newImageID)
+	if err != nil {
+		slog.ErrorContext(ctx, "CopyImage: failed to copy the UEFI variable store",
+			"sourceImageId", sourceImageID, "amiId", newImageID, "orphanSnapshotId", newSnapshotID, "err", err)
+		if delErr := s.metadata.DeleteSnapshot(ctx, accountID, newSnapshotID); delErr != nil {
+			slog.ErrorContext(ctx, "CopyImage: failed to roll back orphaned snapshot metadata",
+				"snapshotId", newSnapshotID, "err", delErr)
+		}
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+
 	meta := ebsmetadata.AMI{
 		ImageID:         newImageID,
 		Name:            name,
@@ -876,12 +907,18 @@ func (s *ImageServiceImpl) CopyImage(ctx context.Context, input *ec2.CopyImageIn
 		SourceDigest:    srcMeta.SourceDigest,
 		// Zero-copy: the new config shares the source's already-durable
 		// snapshot, so the image is complete as soon as this is written.
-		State: "available",
+		State:    "available",
+		UEFIData: uefiData,
 	}
 
 	if err := s.putAMIConfig(ctx, newImageID, meta); err != nil {
+		var orphanUEFIKey string
+		if uefiData != nil {
+			orphanUEFIKey = uefiData.Key
+		}
+		// The copied UEFI object stays under the new image's prefix; the log names it.
 		slog.ErrorContext(ctx, "CopyImage: failed to write AMI config",
-			"amiId", newImageID, "orphanSnapshotId", newSnapshotID, "err", err)
+			"amiId", newImageID, "orphanSnapshotId", newSnapshotID, "orphanUEFIKey", orphanUEFIKey, "err", err)
 		// Best-effort rollback of the orphaned snapshot metadata.
 		if delErr := s.metadata.DeleteSnapshot(ctx, accountID, newSnapshotID); delErr != nil {
 			slog.ErrorContext(ctx, "CopyImage: failed to roll back orphaned snapshot metadata",
@@ -1125,10 +1162,10 @@ func pickRootSnapshotBDM(mappings []*ec2.BlockDeviceMapping, rootDeviceName *str
 	return nil
 }
 
-// DeregisterImage hard-deletes the AMI document. The pre-provider
-// {imageID}/config.json goes too, or the metadata store's legacy read fallback
-// would resurrect the image. Backing snapshot is untouched, so operators run
-// delete-snapshot separately to reclaim block storage.
+// DeregisterImage hard-deletes the image's UEFI store prefix, then the AMI
+// document, so a retry after a failed delete finishes the job. The legacy
+// {imageID}/config.json goes too, or the metadata store's read fallback would
+// resurrect the image. The backing snapshot is untouched, as on AWS.
 func (s *ImageServiceImpl) DeregisterImage(ctx context.Context, input *ec2.DeregisterImageInput, accountID string) (*ec2.DeregisterImageOutput, error) {
 	if input == nil || input.ImageId == nil || *input.ImageId == "" {
 		return nil, errors.New(awserrors.ErrorMissingParameter)
@@ -1140,6 +1177,12 @@ func (s *ImageServiceImpl) DeregisterImage(ctx context.Context, input *ec2.Dereg
 		return nil, err
 	}
 
+	// Only the image's own prefix: a copy or a source instance has its own.
+	if n, err := uefistore.DeleteObjects(ctx, s.store, s.bucketName, imageID); err != nil {
+		slog.ErrorContext(ctx, "DeregisterImage: failed to delete the UEFI variable store", "imageId", imageID, "deleted", n, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+
 	if err := s.metadata.DeleteAMI(ctx, imageID); err != nil && !objectstore.IsNoSuchKeyError(err) {
 		slog.ErrorContext(ctx, "DeregisterImage: failed to delete AMI document", "imageId", imageID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)

spinifex/domains/ec2/image/uefi.go

@@ -0,0 +1,101 @@
+package image
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+)
+
+// uefiPointerAttempts bounds how often CreateImage re-reads a running
+// instance's pointer after the object it named was pruned behind a newer one.
+const uefiPointerAttempts = 3
+
+// UEFIStoreRecords is the store-record read CreateImage makes of its source
+// instance, answered by the stream leader so it sees the last landed pointer.
+type UEFIStoreRecords interface {
+	LeaderGet(ctx context.Context, instanceID string) (*uefistore.UEFIStoreRecord, uint64, error)
+}
+
+var _ UEFIStoreRecords = (*uefistore.Store)(nil)
+
+// SetUEFIStores wires the store records CreateImage reads a UEFI instance's
+// pointer from, and the node ID it writes image store objects as.
+func (s *ImageServiceImpl) SetUEFIStores(records UEFIStoreRecords, nodeID string) {
+	s.uefiRecords = records
+	s.nodeID = nodeID
+}
+
+// copyInstanceUEFIStore copies the object instanceID's store record points at
+// to a fresh key under imageID and returns its reference, or nil when the
+// instance has no record or no pointer. The error is an AWS error code.
+func (s *ImageServiceImpl) copyInstanceUEFIStore(ctx context.Context, instanceID, imageID string) (*ebsmetadata.UEFIDataRef, error) {
+	if s.uefiRecords == nil || s.nodeID == "" {
+		slog.ErrorContext(ctx, "CreateImage: no UEFI store records are wired on this node", "instanceId", instanceID)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	// Once a live pointer is seen, a record that then vanishes is a terminate
+	// racing this request, not an instance without a store.
+	sawPointer := false
+	for range uefiPointerAttempts {
+		rec, _, err := s.uefiRecords.LeaderGet(ctx, instanceID)
+		switch {
+		case errors.Is(err, kvstore.ErrNotFound) && sawPointer:
+			return nil, errors.New(awserrors.ErrorIncorrectInstanceState)
+		case errors.Is(err, kvstore.ErrNotFound):
+			return nil, nil
+		case err != nil:
+			slog.ErrorContext(ctx, "CreateImage: failed to read the UEFI store record", "instanceId", instanceID, "err", err)
+			return nil, errors.New(awserrors.ErrorServerInternal)
+		case rec.State != uefistore.StateLive:
+			return nil, errors.New(awserrors.ErrorIncorrectInstanceState)
+		case rec.Pointer == nil:
+			return nil, nil
+		}
+		sawPointer = true
+		ptr := rec.Pointer
+		key, err := uefistore.CopyObject(ctx, s.store, s.bucketName, ptr.Key, uefistore.ObjectMeta{
+			OwnerID: imageID, WriterNode: s.nodeID, Firmware: ptr.Firmware, Source: uefistore.SourceInstance(instanceID),
+		})
+		if errors.Is(err, uefistore.ErrObjectNotFound) {
+			// A write-back moved the pointer and a prune removed this one.
+			continue
+		}
+		if err != nil {
+			slog.ErrorContext(ctx, "CreateImage: failed to copy the UEFI variable store",
+				"instanceId", instanceID, "imageId", imageID, "key", ptr.Key, "err", err)
+			return nil, errors.New(awserrors.ErrorServerInternal)
+		}
+		slog.InfoContext(ctx, "CreateImage: copied the UEFI variable store",
+			"instanceId", instanceID, "imageId", imageID, "sourceKey", ptr.Key, "key", key)
+		return uefistore.AMIRef(key, ptr.Firmware), nil
+	}
+	slog.ErrorContext(ctx, "CreateImage: the UEFI store pointer kept naming a missing object",
+		"instanceId", instanceID, "attempts", uefiPointerAttempts)
+	return nil, errors.New(awserrors.ErrorServerInternal)
+}
+
+// copyImageUEFIStore copies the store src names to a fresh key under imageID,
+// so deregistering either image leaves the other's intact. It returns nil
+// when src names none.
+func (s *ImageServiceImpl) copyImageUEFIStore(ctx context.Context, src ebsmetadata.AMI, imageID string) (*ebsmetadata.UEFIDataRef, error) {
+	store, err := uefistore.ImageVarStoreFromAMI(src)
+	if err != nil || store == nil {
+		return nil, err
+	}
+	if s.nodeID == "" {
+		return nil, errors.New("no node ID is wired to write UEFI store objects as")
+	}
+	key, err := uefistore.CopyObject(ctx, s.store, s.bucketName, store.Key, uefistore.ObjectMeta{
+		OwnerID: imageID, WriterNode: s.nodeID, Firmware: store.Firmware, Source: uefistore.SourceImage(src.ImageID),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("copy UEFI store %s to %s: %w", store.Key, imageID, err)
+	}
+	return uefistore.AMIRef(key, store.Firmware), nil
+}

spinifex/domains/ec2/image/uefi_test.go

@@ -0,0 +1,417 @@
+package image
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	awss3 "github.com/aws/aws-sdk-go/service/s3"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	uefiTestInstance = "i-0123456789abcdef0"
+	uefiTestVarsSize = 4096
+)
+
+var uefiTestFirmware = uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: uefiTestVarsSize, CodeSHA256: "code-sha"}
+
+func uefiRegion(tag string) []byte {
+	b := make([]byte, uefiTestVarsSize)
+	copy(b, tag)
+	return b
+}
+
+// uefiImageFixture is an image service wired to a real store-record bucket,
+// with a UEFI instance whose root volume CreateImage can snapshot.
+type uefiImageFixture struct {
+	svc       *ImageServiceImpl
+	snapshots *countingSnapshots
+	objects   *objectstore.MemoryObjectStore
+	records   *uefistore.Store
+	volume    ebsmetadata.Volume
+}
+
+func newUEFIImageFixture(t *testing.T) *uefiImageFixture {
+	t.Helper()
+	svc, objects, provider := setupProviderSnapshotImageService(t)
+	_, _, js := testutil.StartTestJetStream(t)
+	records := uefistore.NewStore(js)
+	require.NoError(t, records.Open(t.Context()))
+	svc.SetUEFIStores(records, "node-a")
+	volume := seedProviderVolume(t, svc, provider, "vol-uefi001", 8)
+	snapshots := &countingSnapshots{EBSProvider: provider}
+	svc.SetEBSProvider(snapshots)
+	return &uefiImageFixture{svc: svc, snapshots: snapshots, objects: objects, records: records, volume: volume}
+}
+
+// countingSnapshots counts the snapshots CreateImage asks the provider for.
+type countingSnapshots struct {
+	ebsprovider.EBSProvider
+
+	created int
+}
+
+func (c *countingSnapshots) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
+	c.created++
+	return c.EBSProvider.CreateSnapshot(ctx, req)
+}
+
+// assertNoSnapshot checks a refused CreateImage took no snapshot and wrote no
+// snapshot document.
+func (f *uefiImageFixture) assertNoSnapshot(t *testing.T) {
+	t.Helper()
+	assert.Zero(t, f.snapshots.created, "a refused CreateImage takes no snapshot")
+	snaps, err := f.svc.MetadataStore().ListAllSnapshots(t.Context())
+	require.NoError(t, err)
+	assert.Empty(t, snaps, "a refused CreateImage writes no snapshot document")
+}
+
+// putInstanceStore puts vars at a fresh key under the instance and points a
+// live store record at it.
+func (f *uefiImageFixture) putInstanceStore(t *testing.T, vars []byte) string {
+	t.Helper()
+	key := f.putObject(t, uefiTestInstance, vars)
+	_, err := f.records.Create(t.Context(), &uefistore.UEFIStoreRecord{
+		InstanceID: uefiTestInstance, AccountID: testAccountID, State: uefistore.StateLive, Generation: 1,
+		Pointer: &uefistore.UEFIVarStoreRef{Key: key, Firmware: uefiTestFirmware, WriterNode: "node-a", WrittenAt: time.Now().UTC()},
+	})
+	require.NoError(t, err)
+	return key
+}
+
+func (f *uefiImageFixture) putObject(t *testing.T, owner string, vars []byte) string {
+	t.Helper()
+	key, err := uefistore.NewObjectKey(owner)
+	require.NoError(t, err)
+	require.NoError(t, uefistore.PutObject(t.Context(), f.objects, testBucket, key, vars, uefistore.ObjectMeta{
+		OwnerID: owner, WriterNode: "node-a", Firmware: uefiTestFirmware, Source: uefistore.SourceWriteBack,
+	}))
+	return key
+}
+
+func (f *uefiImageFixture) createImage(t *testing.T, uefi bool) (string, error) {
+	t.Helper()
+	out, err := f.svc.CreateImageFromInstance(CreateImageParams{
+		Input:        &ec2.CreateImageInput{InstanceId: aws.String(uefiTestInstance), Name: aws.String("uefi-image")},
+		RootVolumeID: f.volume.VolumeID,
+		UEFI:         uefi,
+	}, testAccountID)
+	if err != nil {
+		return "", err
+	}
+	return aws.StringValue(out.ImageId), nil
+}
+
+func (f *uefiImageFixture) ami(t *testing.T, imageID string) ebsmetadata.AMI {
+	t.Helper()
+	ami, err := f.svc.MetadataStore().GetAMI(t.Context(), imageID)
+	require.NoError(t, err)
+	return ami
+}
+
+func (f *uefiImageFixture) region(t *testing.T, key string) []byte {
+	t.Helper()
+	out, err := f.objects.GetObject(t.Context(), &awss3.GetObjectInput{Bucket: aws.String(testBucket), Key: aws.String(key)})
+	require.NoError(t, err)
+	defer out.Body.Close()
+	zr, err := gzip.NewReader(out.Body)
+	require.NoError(t, err)
+	b, err := io.ReadAll(zr)
+	require.NoError(t, err)
+	return b
+}
+
+func (f *uefiImageFixture) keys(t *testing.T, owner string) []string {
+	t.Helper()
+	keys, err := uefistore.ListObjects(t.Context(), f.objects, testBucket, owner)
+	require.NoError(t, err)
+	return keys
+}
+
+func (f *uefiImageFixture) copyImage(t *testing.T, src, name string) string {
+	t.Helper()
+	out, err := f.svc.CopyImage(t.Context(), validCopyImageServiceInput(src, name), testAccountID)
+	require.NoError(t, err)
+	return aws.StringValue(out.ImageId)
+}
+
+func (f *uefiImageFixture) deregister(t *testing.T, imageID string) error {
+	t.Helper()
+	_, err := f.svc.DeregisterImage(t.Context(), &ec2.DeregisterImageInput{ImageId: aws.String(imageID)}, testAccountID)
+	return err
+}
+
+// CreateImage copies the object the instance's record points at to the
+// image's own prefix, and the AMI document names it with its firmware.
+func TestCreateImage_CopiesTheInstancesUEFIStore(t *testing.T) {
+	f := newUEFIImageFixture(t)
+	instanceKey := f.putInstanceStore(t, uefiRegion("guest variables"))
+
+	imageID, err := f.createImage(t, true)
+	require.NoError(t, err)
+
+	ami := f.ami(t, imageID)
+	require.NotNil(t, ami.UEFIData)
+	assert.Equal(t, []string{ami.UEFIData.Key}, f.keys(t, imageID), "the image has one object under its own prefix")
+	assert.Equal(t, uefiRegion("guest variables"), f.region(t, ami.UEFIData.Key))
+	assert.Equal(t, uefistore.AMIRef(ami.UEFIData.Key, uefiTestFirmware), ami.UEFIData)
+	assert.Equal(t, []string{instanceKey}, f.keys(t, uefiTestInstance), "the instance's store is left as it was")
+
+	store, err := uefistore.ImageVarStoreFromAMI(ami)
+	require.NoError(t, err)
+	assert.Equal(t, &uefistore.ImageVarStore{ImageID: imageID, Key: ami.UEFIData.Key, Firmware: uefiTestFirmware}, store,
+		"the document decodes into the store a launch seeds from")
+}
+
+// A UEFI instance with no store record or no pointer, and a BIOS instance,
+// give an image that launches from the template.
+func TestCreateImage_AnInstanceWithoutAPointerContributesNothing(t *testing.T) {
+	t.Run("no record", func(t *testing.T) {
+		f := newUEFIImageFixture(t)
+		imageID, err := f.createImage(t, true)
+		require.NoError(t, err)
+		assert.Nil(t, f.ami(t, imageID).UEFIData)
+		assert.Empty(t, f.keys(t, imageID))
+	})
+	t.Run("no pointer", func(t *testing.T) {
+		f := newUEFIImageFixture(t)
+		_, err := f.records.Create(t.Context(), &uefistore.UEFIStoreRecord{
+			InstanceID: uefiTestInstance, AccountID: testAccountID, State: uefistore.StateLive, Generation: 1,
+		})
+		require.NoError(t, err)
+		imageID, err := f.createImage(t, true)
+		require.NoError(t, err)
+		assert.Nil(t, f.ami(t, imageID).UEFIData)
+	})
+	t.Run("BIOS instance", func(t *testing.T) {
+		f := newUEFIImageFixture(t)
+		f.putInstanceStore(t, uefiRegion("never read"))
+		f.svc.SetUEFIStores(nil, "")
+		imageID, err := f.createImage(t, false)
+		require.NoError(t, err)
+		assert.Nil(t, f.ami(t, imageID).UEFIData)
+	})
+}
+
+// A UEFI instance's image is refused rather than made without its store when
+// this node cannot read store records, and a terminating store is refused.
+func TestCreateImage_RefusesWhenTheStoreCannotBeCarried(t *testing.T) {
+	t.Run("no records wired", func(t *testing.T) {
+		f := newUEFIImageFixture(t)
+		f.svc.SetUEFIStores(nil, "")
+		_, err := f.createImage(t, true)
+		require.EqualError(t, err, awserrors.ErrorServerInternal)
+		f.assertNoSnapshot(t)
+	})
+	t.Run("deleting", func(t *testing.T) {
+		f := newUEFIImageFixture(t)
+		f.putInstanceStore(t, uefiRegion("terminating"))
+		_, err := f.records.MarkDeleting(t.Context(), uefiTestInstance)
+		require.NoError(t, err)
+		_, err = f.createImage(t, true)
+		require.EqualError(t, err, awserrors.ErrorIncorrectInstanceState)
+		f.assertNoSnapshot(t)
+	})
+	t.Run("record unreadable", func(t *testing.T) {
+		f := newUEFIImageFixture(t)
+		f.svc.SetUEFIStores(failingRecords{}, "node-a")
+		_, err := f.createImage(t, true)
+		require.EqualError(t, err, awserrors.ErrorServerInternal)
+		f.assertNoSnapshot(t)
+	})
+	t.Run("object unreadable", func(t *testing.T) {
+		f := newUEFIImageFixture(t)
+		key := f.putInstanceStore(t, uefiRegion("x"))
+		_, err := f.objects.PutObject(t.Context(), &awss3.PutObjectInput{
+			Bucket: aws.String(testBucket), Key: aws.String(key), Body: bytes.NewReader([]byte("not gzip")),
+		})
+		require.NoError(t, err)
+		_, err = f.createImage(t, true)
+		require.EqualError(t, err, awserrors.ErrorServerInternal)
+		f.assertNoSnapshot(t)
+	})
+}
+
+// failingRecords cannot read any store record.
+type failingRecords struct{}
+
+func (failingRecords) LeaderGet(context.Context, string) (*uefistore.UEFIStoreRecord, uint64, error) {
+	return nil, 0, assert.AnError
+}
+
+// vanishingRecord shows a live pointer whose object is gone, then no record,
+// as a terminate's reaper leaves it.
+type vanishingRecord struct {
+	stale *uefistore.UEFIStoreRecord
+	reads int
+}
+
+func (v *vanishingRecord) LeaderGet(context.Context, string) (*uefistore.UEFIStoreRecord, uint64, error) {
+	v.reads++
+	if v.reads == 1 {
+		return v.stale, 1, nil
+	}
+	return nil, 0, kvstore.ErrNotFound
+}
+
+// A record that vanishes after CreateImage saw its live pointer is a terminate
+// racing the request, so the image is refused rather than made without its store.
+func TestCreateImage_StoreDeletedMidRequestIsIncorrectInstanceState(t *testing.T) {
+	f := newUEFIImageFixture(t)
+	f.putInstanceStore(t, uefiRegion("x"))
+	stale, _, err := f.records.Get(t.Context(), uefiTestInstance)
+	require.NoError(t, err)
+	stale.Pointer.Key = uefistore.KeyPrefix(uefiTestInstance) + "reaped"
+	f.svc.SetUEFIStores(&vanishingRecord{stale: stale}, "node-a")
+
+	_, err = f.createImage(t, true)
+
+	require.EqualError(t, err, awserrors.ErrorIncorrectInstanceState)
+	f.assertNoSnapshot(t)
+	amis, err := f.svc.MetadataStore().ListAMIs(t.Context())
+	require.NoError(t, err)
+	assert.Empty(t, amis)
+}
+
+// laggingPointer answers the first read with a pointer whose object a prune
+// has removed, and every later read with the record as stored.
+type laggingPointer struct {
+	*uefistore.Store
+
+	stale *uefistore.UEFIStoreRecord
+	reads int
+}
+
+func (l *laggingPointer) LeaderGet(ctx context.Context, id string) (*uefistore.UEFIStoreRecord, uint64, error) {
+	l.reads++
+	if l.reads == 1 {
+		return l.stale, 1, nil
+	}
+	return l.Store.LeaderGet(ctx, id)
+}
+
+// A running instance's write-back can move the pointer and prune the old
+// object between CreateImage's read and its get, so the pointer is read again.
+func TestCreateImage_RereadsAPointerWhoseObjectWasPruned(t *testing.T) {
+	f := newUEFIImageFixture(t)
+	f.putInstanceStore(t, uefiRegion("newest"))
+	stale, _, err := f.records.Get(t.Context(), uefiTestInstance)
+	require.NoError(t, err)
+	stale.Pointer.Key = uefistore.KeyPrefix(uefiTestInstance) + "pruned"
+	lagging := &laggingPointer{Store: f.records, stale: stale}
+	f.svc.SetUEFIStores(lagging, "node-a")
+
+	imageID, err := f.createImage(t, true)
+	require.NoError(t, err)
+
+	assert.Equal(t, 2, lagging.reads)
+	assert.Equal(t, uefiRegion("newest"), f.region(t, f.ami(t, imageID).UEFIData.Key))
+}
+
+// CopyImage gives the copy its own object holding the source's region.
+func TestCopyImage_GivesTheCopyItsOwnUEFIStore(t *testing.T) {
+	f := newUEFIImageFixture(t)
+	f.putInstanceStore(t, uefiRegion("source variables"))
+	src, err := f.createImage(t, true)
+	require.NoError(t, err)
+
+	cp := f.copyImage(t, src, "uefi-copy")
+
+	srcRef, cpRef := f.ami(t, src).UEFIData, f.ami(t, cp).UEFIData
+	require.NotNil(t, cpRef)
+	assert.Equal(t, []string{cpRef.Key}, f.keys(t, cp))
+	assert.Equal(t, srcRef.Firmware, cpRef.Firmware)
+	assert.Equal(t, uefiRegion("source variables"), f.region(t, cpRef.Key))
+	assert.Equal(t, []string{srcRef.Key}, f.keys(t, src), "the source keeps its own object")
+}
+
+// A source AMI with no store gives a copy with none.
+func TestCopyImage_SourceWithoutAUEFIStoreCopiesNone(t *testing.T) {
+	f := newUEFIImageFixture(t)
+	seedCopyableAMI(t, f.objects, "ami-src001", "source-ami", testAccountID, "snap-src001", "vol-src001", 8)
+
+	cp := f.copyImage(t, "ami-src001", "copy")
+
+	assert.Nil(t, f.ami(t, cp).UEFIData)
+	assert.Empty(t, f.keys(t, cp))
+}
+
+// A source store that cannot be copied fails the copy and leaves no AMI.
+func TestCopyImage_UnreadableSourceStoreFails(t *testing.T) {
+	f := newUEFIImageFixture(t)
+	seedCopyableAMI(t, f.objects, "ami-src001", "source-ami", testAccountID, "snap-src001", "vol-src001", 8)
+	src := f.ami(t, "ami-src001")
+	src.UEFIData = uefistore.AMIRef(uefistore.KeyPrefix("ami-src001")+"gone", uefiTestFirmware)
+	require.NoError(t, f.svc.MetadataStore().PutAMI(t.Context(), src))
+
+	_, err := f.svc.CopyImage(t.Context(), validCopyImageServiceInput("ami-src001", "copy"), testAccountID)
+
+	require.EqualError(t, err, awserrors.ErrorServerInternal)
+	amis, err := f.svc.MetadataStore().ListAMIs(t.Context())
+	require.NoError(t, err)
+	assert.Len(t, amis, 1, "no copy is registered without its store")
+}
+
+// TestS7_DeregisteringASourceImageLeavesItsCopysStore enforces ADR-0003:S7
+// for DeregisterImage, which account termination also calls: it deletes only
+// the deregistered image's prefix, and the copy's store is its own.
+func TestS7_DeregisteringASourceImageLeavesItsCopysStore(t *testing.T) {
+	f := newUEFIImageFixture(t)
+	instanceKey := f.putInstanceStore(t, uefiRegion("source variables"))
+	src, err := f.createImage(t, true)
+	require.NoError(t, err)
+	cp := f.copyImage(t, src, "uefi-copy")
+
+	require.NoError(t, f.deregister(t, src))
+
+	assert.Empty(t, f.keys(t, src), "the deregistered image's store is gone")
+	assert.Equal(t, uefiRegion("source variables"), f.region(t, f.ami(t, cp).UEFIData.Key),
+		"ADR-0003:S7: deregistering the source must not delete the store its copy boots from")
+	assert.Equal(t, []string{instanceKey}, f.keys(t, uefiTestInstance), "the instance's store is left as it was")
+}
+
+// failingUEFIDelete fails every delete under the UEFI root.
+type failingUEFIDelete struct{ *objectstore.MemoryObjectStore }
+
+func (s failingUEFIDelete) DeleteObject(ctx context.Context, in *awss3.DeleteObjectInput) (*awss3.DeleteObjectOutput, error) {
+	if strings.HasPrefix(aws.StringValue(in.Key), uefistore.ObjectRoot) {
+		return nil, assert.AnError
+	}
+	return s.MemoryObjectStore.DeleteObject(ctx, in)
+}
+
+// DeregisterImage deletes the store before the document, so a failed delete
+// keeps the image for a retry to finish.
+func TestDeregisterImage_FailedUEFIStoreDeleteKeepsTheImage(t *testing.T) {
+	f := newUEFIImageFixture(t)
+	f.putInstanceStore(t, uefiRegion("x"))
+	imageID, err := f.createImage(t, true)
+	require.NoError(t, err)
+	f.svc.store = failingUEFIDelete{f.objects}
+
+	require.EqualError(t, f.deregister(t, imageID), awserrors.ErrorServerInternal)
+	assert.NotNil(t, f.ami(t, imageID).UEFIData, "the document stays for the retry")
+
+	f.svc.store = f.objects
+	require.NoError(t, f.deregister(t, imageID))
+	assert.Empty(t, f.keys(t, imageID))
+	_, err = f.svc.MetadataStore().GetAMI(t.Context(), imageID)
+	assert.True(t, objectstore.IsNoSuchKeyError(err), "the retry removes the document: %v", err)
+}
+
+var _ UEFIStoreRecords = (*laggingPointer)(nil)

spinifex/domains/ec2/instance/provider_efi_volume_test.go

@@ -1,126 +0,0 @@
-package instance
-
-import (
-	"bytes"
-	"context"
-	"testing"
-
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// varsTemplateBytes stands in for a firmware VARS template. The real one is
-// 540,672 bytes, but nothing here depends on the size beyond it being exact.
-func varsTemplateBytes() []byte { return bytes.Repeat([]byte{0xAB}, 4096) }
-
-func providerEFIService(t *testing.T, provider ebsprovider.EBSProvider) *InstanceServiceImpl {
-	t.Helper()
-	return providerRootVolumeService(t, provider, rootVolumeAMILoader(), objectstore.NewMemoryObjectStore())
-}
-
-// TestCreateEFIVolume_Provider_SeedsFromTemplate is the core of the EFI
-// decouple: the variable store is allocated through the provider with the
-// template as seed bytes, so no control-plane engine is built for it.
-func TestCreateEFIVolume_Provider_SeedsFromTemplate(t *testing.T) {
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true})
-	svc := providerEFIService(t, provider)
-	template := varsTemplateBytes()
-
-	require.NoError(t, svc.createEFIVolumeViaProvider(context.Background(), "vol-efiseed001-efi", template))
-
-	volume, err := provider.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{
-		Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-efiseed001-efi",
-	})
-	require.NoError(t, err)
-	assert.Equal(t, int64(len(template)), volume.CapacityBytes,
-		"the store must be sized byte-exactly to the VARS template")
-
-	seed, ok := provider.SeedData("vol-efiseed001-efi")
-	require.True(t, ok)
-	assert.Equal(t, template, seed, "the template must reach the provider as seed bytes")
-}
-
-// A provider that rounds capacity to a GiB boundary produces a VARS region
-// pflash refuses, which would surface as an unbootable guest rather than a
-// failed launch. The mismatch has to fail the launch instead.
-func TestCreateEFIVolume_Provider_CapacityMismatchFails(t *testing.T) {
-	provider := &roundingProvider{
-		EBSProvider: ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true}),
-	}
-	svc := providerEFIService(t, provider)
-
-	err := svc.createEFIVolumeViaProvider(context.Background(), "vol-efiround01-efi", varsTemplateBytes())
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-// A relaunch reissues the same create. Reseeding would overwrite the BootOrder
-// the guest wrote on its first boot, so the second call must be a no-op.
-func TestCreateEFIVolume_Provider_RepeatDoesNotReseed(t *testing.T) {
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true})
-	svc := providerEFIService(t, provider)
-	ctx := context.Background()
-	template := varsTemplateBytes()
-
-	require.NoError(t, svc.createEFIVolumeViaProvider(ctx, "vol-efirepeat1-efi", template))
-	require.NoError(t, svc.createEFIVolumeViaProvider(ctx, "vol-efirepeat1-efi", template))
-
-	seed, ok := provider.SeedData("vol-efirepeat1-efi")
-	require.True(t, ok)
-	assert.Equal(t, template, seed)
-}
-
-// A seed above the wire bound must fail the launch with an error naming the
-// limit, not reach NATS and fail as an oversized publish.
-func TestCreateEFIVolume_Provider_OversizedTemplateFails(t *testing.T) {
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true})
-	svc := providerEFIService(t, provider)
-
-	err := svc.createEFIVolumeViaProvider(context.Background(), "vol-efitoobig1-efi", make([]byte, ebsprovider.MaxSeedBytes+1))
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-// TestPrepareEFIVolume_Provider_RegistersEBSRequest covers the whole function
-// against this host's real firmware, including the name the VM launches with.
-func TestPrepareEFIVolume_Provider_RegistersEBSRequest(t *testing.T) {
-	if _, _, _, err := vm.FirmwarePaths("x86_64"); err != nil {
-		t.Skipf("no UEFI firmware on this host: %v", err)
-	}
-
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true})
-	svc := providerEFIService(t, provider)
-	instance := &vm.VM{}
-
-	require.NoError(t, svc.prepareEFIVolume(context.Background(), "vol-efiprepare001", instance, "x86_64"))
-
-	instance.EBSRequests.Mu.Lock()
-	defer instance.EBSRequests.Mu.Unlock()
-	require.Len(t, instance.EBSRequests.Requests, 1)
-	assert.Equal(t, "vol-efiprepare001-efi", instance.EBSRequests.Requests[0].Name)
-	assert.True(t, instance.EBSRequests.Requests[0].EFI)
-	assert.False(t, instance.EBSRequests.Requests[0].Boot)
-
-	seed, ok := provider.SeedData("vol-efiprepare001-efi")
-	require.True(t, ok)
-	assert.NotEmpty(t, seed, "the store must be seeded from this host's VARS template")
-}
-
-// roundingProvider reports a GiB-rounded capacity the way a provider with no
-// sub-GiB volume granularity would.
-type roundingProvider struct {
-	ebsprovider.EBSProvider
-}
-
-func (p *roundingProvider) CreateVolume(ctx context.Context, req ebsprovider.CreateVolumeRequest) (*ebsprovider.Volume, error) {
-	volume, err := p.EBSProvider.CreateVolume(ctx, req)
-	if err != nil {
-		return nil, err
-	}
-	volume.CapacityBytes = 1 << 30
-	return volume, nil
-}

spinifex/domains/ec2/instance/service_impl.go

@@ -10,7 +10,6 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/tagmirror"
 	"log/slog"
 	"maps"
-	"os"
 	"slices"
 	"sort"
 	"strconv"
@@ -27,6 +26,7 @@ import (
 	ebsencryption "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/encryption"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
 	ebspolicy "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/policy"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instancetypes"
 	ec2platform "github.com/mulgadc/spinifex/spinifex/domains/ec2/platform"
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
@@ -136,10 +136,11 @@ func isEphemeralMapping(bdm *ec2.BlockDeviceMapping) bool {
 }
 
 // SelectRootBlockDeviceMapping returns the index of the mapping addressing the
-// root device, which is the one naming ebsmetadata.RootDeviceName, or -1 when
-// none does. A mapping naming any other device is an additional volume and the
-// root keeps the image's own mapping, as AWS does.
+// root device, which is the one naming ebsmetadata.RootDeviceName. Position
+// decides only when no mapping names it, so a caller addressing a device this
+// platform's images do not have still gets the volume they sized.
 func SelectRootBlockDeviceMapping(mappings []*ec2.BlockDeviceMapping) int {
+	first := -1
 	for i, bdm := range mappings {
 		if bdm == nil || isEphemeralMapping(bdm) {
 			continue
@@ -147,23 +148,11 @@ func SelectRootBlockDeviceMapping(mappings []*ec2.BlockDeviceMapping) int {
 		if aws.StringValue(bdm.DeviceName) == ebsmetadata.RootDeviceName {
 			return i
 		}
-	}
-	return -1
-}
-
-// UnnamedBlockDeviceMapping reports whether a non-root EBS mapping names no
-// device, which would leave its volume attached at an empty device name.
-func UnnamedBlockDeviceMapping(mappings []*ec2.BlockDeviceMapping) bool {
-	root := SelectRootBlockDeviceMapping(mappings)
-	for i, bdm := range mappings {
-		if i == root || bdm == nil || bdm.Ebs == nil || isEphemeralMapping(bdm) {
-			continue
-		}
-		if aws.StringValue(bdm.DeviceName) == "" {
-			return true
+		if first < 0 {
+			first = i
 		}
 	}
-	return false
+	return first
 }
 
 // UnservedBlockDeviceMappings names the non-root EBS mappings the launch path
@@ -1265,6 +1254,10 @@ func (s *InstanceServiceImpl) StartInstance(ctx context.Context, instance *vm.VM
 				slog.WarnContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
 				return refusal
 			}
+			if errors.Is(err, vm.ErrUEFIClaimHeld) {
+				slog.WarnContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
+				return errInsufficientCapacity
+			}
 			slog.ErrorContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
 			return errors.New(awserrors.ErrorServerInternal)
 		}
@@ -1612,18 +1605,20 @@ func (s *InstanceServiceImpl) GenerateVolumes(ctx context.Context, input *ec2.Ru
 		return nil, err
 	}
 
-	// Step 2: Create EFI variable store (only when the AMI is UEFI; BIOS
-	// guests must not allocate an orphan VARS volume).
-	if instance.BootMode == "uefi" || instance.BootMode == "uefi-preferred" {
-		arch := instanceArchitecture(s.instanceTypes[*input.InstanceType])
-		err = s.prepareEFIVolume(ctx, imageId, instance, arch)
-		if err != nil {
-			return nil, err
-		}
+	// Step 2: Seed the UEFI variable store, which is not a volume. A failure
+	// terminates the instance, and terminate deletes whatever store it left.
+	image, err := s.imageUEFIVarStore(ctx, instance, aws.StringValue(input.ImageId))
+	if err != nil {
+		slog.ErrorContext(ctx, "Failed to read the image's UEFI variable store", "instanceId", instance.ID,
+			"imageId", aws.StringValue(input.ImageId), "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if err := s.vmMgr.SeedUEFIVarStore(ctx, instance, image); err != nil {
+		slog.ErrorContext(ctx, "Failed to seed the UEFI variable store", "instanceId", instance.ID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
-	// The EFI store is internal, so it is not reported; every volume the
-	// request named is.
+	// Every volume the request named is reported.
 	volumeInfos := []VolumeInfo{
 		{
 			VolumeId:            imageId,
@@ -1643,6 +1638,23 @@ func (s *InstanceServiceImpl) GenerateVolumes(ctx context.Context, input *ec2.Ru
 	return volumeInfos, nil
 }
 
+// imageUEFIVarStore returns the store the launching AMI names for a UEFI
+// instance to seed from, or nil when the instance boots without UEFI or the
+// AMI names none.
+func (s *InstanceServiceImpl) imageUEFIVarStore(ctx context.Context, instance *vm.VM, imageID string) (*uefistore.ImageVarStore, error) {
+	if !vm.UsesUEFIVarStore(instance) || !strings.HasPrefix(imageID, "ami-") {
+		return nil, nil
+	}
+	if s.amiLoader == nil {
+		return nil, errors.New("AMI loader not initialized")
+	}
+	ami, err := s.amiLoader.GetAMIConfig(ctx, imageID)
+	if err != nil {
+		return nil, err
+	}
+	return uefistore.ImageVarStoreFromAMI(ami)
+}
+
 // prepareDataVolumes creates an empty volume for every EBS mapping that is not
 // the root and records each as a non-boot request the launcher attaches. A
 // failure unwinds the volumes this call created, so a refused launch leaves none.
@@ -1885,81 +1897,6 @@ func rootVolumeIOPS(requested int) int {
 	return requested
 }
 
-// prepareEFIVolume creates the per-VM EFI variable store, sized exactly to the
-// firmware VARS template (pflash requires byte-exact size) and seeded from it.
-// arch is "x86_64" | "arm64".
-func (s *InstanceServiceImpl) prepareEFIVolume(ctx context.Context, volumeID string, instance *vm.VM, arch string) error {
-	codePath, varsTemplate, varsSize, err := vm.FirmwarePaths(arch)
-	if err != nil {
-		slog.ErrorContext(ctx, "UEFI firmware not installed on this host", "arch", arch, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	template, err := os.ReadFile(varsTemplate)
-	if err != nil {
-		slog.ErrorContext(ctx, "Failed to read VARS template", "path", varsTemplate, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	if int64(len(template)) != varsSize {
-		slog.ErrorContext(ctx, "VARS template size mismatch between stat and read", "path", varsTemplate, "statSize", varsSize, "readSize", len(template))
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	slog.InfoContext(ctx, "Preparing EFI variable store", "arch", arch, "firmwarePath", codePath, "varsTemplate", varsTemplate, "size", varsSize)
-
-	efiVolumeName := fmt.Sprintf("%s-efi", volumeID)
-	if s.ebsProvider == nil {
-		slog.ErrorContext(ctx, "no EBS provider configured", "op", "prepareEFIVolume")
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	if err := s.createEFIVolumeViaProvider(ctx, efiVolumeName, template); err != nil {
-		return err
-	}
-
-	instance.EBSRequests.Mu.Lock()
-	instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{
-		Name: efiVolumeName,
-		Boot: false,
-		EFI:  true,
-	})
-	instance.EBSRequests.Mu.Unlock()
-
-	return nil
-}
-
-// createEFIVolumeViaProvider allocates the EFI variable store through the
-// provider, seeding it with this node's firmware VARS template. The bytes
-// travel with the request so the region matches the firmware that will run.
-func (s *InstanceServiceImpl) createEFIVolumeViaProvider(ctx context.Context, efiVolumeName string, template []byte) error {
-	slog.InfoContext(ctx, "Creating EFI variable store via EBS provider", "name", efiVolumeName, "seedBytes", len(template))
-
-	// The store is internal to the launch, never attachable and filtered out of
-	// DescribeVolumes, so it gets no ebsmetadata document the way a root volume
-	// does. Its lifecycle is the root volume's DeleteVolume.
-	created, err := s.ebsProvider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{
-		Versioned:        ebsprovider.NewVersioned(),
-		VolumeID:         efiVolumeName,
-		CapacityRange:    ebsprovider.CapacityRange{RequiredBytes: int64(len(template))},
-		AvailabilityZone: s.config.AZ,
-		SeedData:         template,
-	})
-	if err != nil {
-		slog.ErrorContext(ctx, "Provider EFI volume creation failed", "name", efiVolumeName, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	if created == nil {
-		slog.ErrorContext(ctx, "Provider returned no volume for the EFI variable store", "name", efiVolumeName)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-
-	// pflash rejects a VARS region that is not byte-exact, so a provider that
-	// rounded the capacity up to a GiB boundary yields a guest that cannot boot.
-	if created.CapacityBytes != int64(len(template)) {
-		slog.ErrorContext(ctx, "Provider EFI volume capacity does not match the VARS template",
-			"name", efiVolumeName, "wantBytes", len(template), "gotBytes", created.CapacityBytes)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	return nil
-}
-
 // checkImageArchitecture refuses an AMI whose architecture the instance type
 // does not support, as AWS does. An unrecorded architecture on either side is
 // not a mismatch.
@@ -2760,7 +2697,7 @@ func (s *InstanceServiceImpl) capacityRefusal(ctx context.Context, instance *vm.
 func (s *InstanceServiceImpl) launchClaimedInstance(ctx context.Context, instance *vm.VM, undo func()) error {
 	// Before anything is stamped or claimed, so the record handed back is the
 	// one that was claimed.
-	if err := vm.CheckUEFIVarStore(instance); err != nil {
+	if err := s.vmMgr.CheckUEFIVarStore(ctx, instance); err != nil {
 		slog.ErrorContext(ctx, "launchClaimedInstance: refusing to launch a UEFI instance without exactly one variable store",
 			"instanceId", instance.ID, "err", err)
 		undo()
@@ -2893,6 +2830,10 @@ func (s *InstanceServiceImpl) launchClaimedInstance(ctx context.Context, instanc
 		if refusal := volumeHeldElsewhereError(err); refusal != nil {
 			return refusal
 		}
+		// A live node holds the variable store, so another node may take it.
+		if errors.Is(err, vm.ErrUEFIClaimHeld) {
+			return s.capacityRefusal(ctx, instance)
+		}
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 

spinifex/domains/ec2/instance/service_impl_test.go

@@ -438,7 +438,7 @@ func TestParseVolumeParams_BlockDeviceMappingNoEbs(t *testing.T) {
 
 	p := parseVolumeParams(input)
 
-	assert.Equal(t, ebsmetadata.RootDeviceName, p.deviceName, "a mapping not naming the root leaves the root device alone")
+	assert.Equal(t, "/dev/xvda", p.deviceName)
 	assert.Equal(t, 4*1024*1024*1024, p.size, "size should stay at default without Ebs")
 	assert.True(t, p.deleteOnTermination)
 }
@@ -469,7 +469,7 @@ func TestParseVolumeParams_PartialEbs(t *testing.T) {
 
 	p := parseVolumeParams(input)
 
-	assert.Equal(t, 4*1024*1024*1024, p.size, "an unnamed mapping does not size the root")
+	assert.Equal(t, 8*1024*1024*1024, p.size)
 	assert.Equal(t, ebsmetadata.RootDeviceName, p.deviceName, "device should stay at default")
 	assert.Empty(t, p.volumeType, "volumeType should stay empty")
 	assert.Zero(t, p.iops, "iops should stay zero")
@@ -593,61 +593,23 @@ func TestParseVolumeParams_RootMappingSelectedByName(t *testing.T) {
 	assert.Equal(t, "gp3", p.volumeType)
 }
 
-func TestParseVolumeParams_LoneNonRootMappingIsADataVolume(t *testing.T) {
-	// A mapping that does not name the AMI root device adds a volume; the root
-	// keeps the image's own mapping, as AWS does.
+func TestParseVolumeParams_SingleNonRootMappingStillLaunches(t *testing.T) {
+	// ECS and EKS both launch with one mapping named /dev/vda, which is not the
+	// AMI root device name. Position decides when nothing names the root.
 	input := &ec2.RunInstancesInput{
-		ImageId: aws.String("ami-lone"),
+		ImageId: aws.String("ami-vda"),
 		BlockDeviceMappings: []*ec2.BlockDeviceMapping{
 			{
-				DeviceName: aws.String("/dev/sdf"),
-				Ebs: &ec2.EbsBlockDevice{
-					VolumeSize:          aws.Int64(6),
-					DeleteOnTermination: aws.Bool(false),
-				},
-			},
-		},
-	}
-
-	assert.Equal(t, -1, SelectRootBlockDeviceMapping(input.BlockDeviceMappings))
-
-	p := parseVolumeParams(input)
-	assert.Equal(t, 4*1024*1024*1024, p.size, "root keeps its default size")
-	assert.Equal(t, ebsmetadata.RootDeviceName, p.deviceName)
-	assert.True(t, p.deleteOnTermination, "root keeps its default")
-
-	data := parseDataVolumeParams(input)
-	require.Len(t, data, 1)
-	assert.Equal(t, "/dev/sdf", data[0].deviceName)
-	assert.Equal(t, int64(6), data[0].sizeGiB)
-	assert.False(t, data[0].deleteOnTermination)
-	assert.Empty(t, UnservedBlockDeviceMappings(input.BlockDeviceMappings))
-}
-
-func TestParseVolumeParams_LoneNonRootMappingWithoutSizeIsRefused(t *testing.T) {
-	mappings := []*ec2.BlockDeviceMapping{
-		{DeviceName: aws.String("/dev/sdf"), Ebs: &ec2.EbsBlockDevice{}},
-	}
-
-	assert.Equal(t, []string{"/dev/sdf (names no size)"}, UnservedBlockDeviceMappings(mappings))
-}
-
-func TestParseVolumeParams_LoneRootNamedMappingIsTheRoot(t *testing.T) {
-	input := &ec2.RunInstancesInput{
-		ImageId: aws.String("ami-lone-root"),
-		BlockDeviceMappings: []*ec2.BlockDeviceMapping{
-			{
-				DeviceName: aws.String(ebsmetadata.RootDeviceName),
+				DeviceName: aws.String("/dev/vda"),
 				Ebs:        &ec2.EbsBlockDevice{VolumeSize: aws.Int64(20)},
 			},
 		},
 	}
 
-	assert.Equal(t, 0, SelectRootBlockDeviceMapping(input.BlockDeviceMappings))
 	p := parseVolumeParams(input)
 	assert.Equal(t, 20*1024*1024*1024, p.size)
-	assert.Equal(t, ebsmetadata.RootDeviceName, p.deviceName)
-	assert.Empty(t, parseDataVolumeParams(input))
+	assert.Equal(t, "/dev/vda", p.deviceName)
+	assert.Empty(t, UnservedBlockDeviceMappings(input.BlockDeviceMappings))
 }
 
 func TestUnservedBlockDeviceMappings(t *testing.T) {
@@ -5276,19 +5238,3 @@ func TestPrepareRunInstances_ArchitectureMismatch(t *testing.T) {
 	}
 	assert.NoError(t, checkImageArchitecture(types["t4g.micro"], "aarch64"))
 }
-
-func TestUnnamedBlockDeviceMapping(t *testing.T) {
-	root := &ec2.BlockDeviceMapping{
-		DeviceName: aws.String(ebsmetadata.RootDeviceName),
-		Ebs:        &ec2.EbsBlockDevice{VolumeSize: aws.Int64(8)},
-	}
-	unnamed := &ec2.BlockDeviceMapping{Ebs: &ec2.EbsBlockDevice{VolumeSize: aws.Int64(4)}}
-	emptyName := &ec2.BlockDeviceMapping{DeviceName: aws.String(""), Ebs: &ec2.EbsBlockDevice{VolumeSize: aws.Int64(4)}}
-	ephemeral := &ec2.BlockDeviceMapping{VirtualName: aws.String("ephemeral0")}
-
-	assert.False(t, UnnamedBlockDeviceMapping(nil))
-	assert.False(t, UnnamedBlockDeviceMapping([]*ec2.BlockDeviceMapping{root}))
-	assert.False(t, UnnamedBlockDeviceMapping([]*ec2.BlockDeviceMapping{root, ephemeral}))
-	assert.True(t, UnnamedBlockDeviceMapping([]*ec2.BlockDeviceMapping{unnamed}))
-	assert.True(t, UnnamedBlockDeviceMapping([]*ec2.BlockDeviceMapping{root, emptyName}))
-}

spinifex/domains/ec2/instance/uefi_seed_test.go

@@ -0,0 +1,171 @@
+package instance
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const testSeedInstance = "i-0a1b2c3d4e5f60718"
+
+type seedInstanceTypes map[string]vm.InstanceTypeSpec
+
+func (r seedInstanceTypes) Resolve(name string) (vm.InstanceTypeSpec, bool) {
+	spec, ok := r[name]
+	return spec, ok
+}
+
+// uefiSeedingService is an instance service whose VM manager seeds stores into
+// a real JetStream's store records and claims and an in-memory object store,
+// under a stand-in x86_64 firmware.
+func uefiSeedingService(t *testing.T) (*InstanceServiceImpl, *ebsprovider.MemoryProvider, *uefistore.Store, *objectstore.MemoryObjectStore) {
+	t.Helper()
+	fwDir := t.TempDir()
+	code := filepath.Join(fwDir, "OVMF_CODE.fd")
+	vars := filepath.Join(fwDir, "OVMF_VARS.fd")
+	require.NoError(t, os.WriteFile(code, []byte("code"), 0o600))
+	require.NoError(t, os.WriteFile(vars, make([]byte, 4096), 0o600))
+	orig, had := vm.FirmwarePathCandidates["x86_64"]
+	vm.FirmwarePathCandidates["x86_64"] = []vm.FirmwareCandidate{{Code: code, VarsTemplate: vars}}
+	t.Cleanup(func() {
+		if had {
+			vm.FirmwarePathCandidates["x86_64"] = orig
+		} else {
+			delete(vm.FirmwarePathCandidates, "x86_64")
+		}
+	})
+
+	_, _, js := testutil.StartTestJetStream(t)
+	records := uefistore.NewStore(js)
+	require.NoError(t, records.Open(t.Context()))
+
+	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{CrashConsistentSnapshot: true, VolumeEnumeration: true})
+	seedProviderSnapshot(t, provider, "vol-origin", "snap-source")
+	objects := objectstore.NewMemoryObjectStore()
+	svc := providerRootVolumeService(t, provider, rootVolumeAMILoader(), objects)
+	svc.vmMgr = vm.NewManagerWithDeps(vm.Deps{
+		NodeID:        "node-a",
+		InstanceTypes: seedInstanceTypes{"t3.nano": {Architecture: "x86_64", VCPUs: 1, MemoryMiB: 512}},
+		UEFIVarStore: vm.UEFIVarStoreDeps{
+			Objects: objects, Bucket: testRootBucket, DataDir: t.TempDir(), Records: records,
+			Claims: uefistore.NewClaims(kvstore.OpenSyncBucket(js), "node-a"),
+		},
+	})
+	return svc, provider, records, objects
+}
+
+func uefiLaunch() (*ec2.RunInstancesInput, *vm.VM) {
+	input := &ec2.RunInstancesInput{ImageId: aws.String("ami-1"), InstanceType: aws.String("t3.nano")}
+	return input, &vm.VM{ID: testSeedInstance, AccountID: testRootAccount, InstanceType: "t3.nano", BootMode: "uefi"}
+}
+
+// A UEFI launch seeds its store as an object and a store record, and creates
+// no <root>-efi volume and no EFI request.
+func TestGenerateVolumes_UEFILaunchSeedsAStoreAndCreatesNoEFIVolume(t *testing.T) {
+	svc, provider, records, objects := uefiSeedingService(t)
+	input, instance := uefiLaunch()
+
+	infos, err := svc.GenerateVolumes(t.Context(), input, instance)
+	require.NoError(t, err)
+
+	require.Len(t, infos, 1, "only the root volume is reported")
+	instance.EBSRequests.Mu.Lock()
+	for _, req := range instance.EBSRequests.Requests {
+		assert.False(t, req.EFI, "no EFI request is appended")
+		assert.False(t, strings.HasSuffix(req.Name, "-efi"), req.Name)
+	}
+	instance.EBSRequests.Mu.Unlock()
+	listed, err := provider.ListVolumes(t.Context(), ebsprovider.ListVolumesRequest{Versioned: ebsprovider.NewVersioned()})
+	require.NoError(t, err)
+	for _, v := range listed.Volumes {
+		assert.False(t, strings.HasSuffix(v.ID, "-efi"), "no EFI volume is created: %s", v.ID)
+	}
+
+	rec, _, err := records.Get(t.Context(), testSeedInstance)
+	require.NoError(t, err)
+	assert.Equal(t, uefistore.StateLive, rec.State)
+	require.NotNil(t, rec.Pointer)
+	keys, err := uefistore.ListObjects(t.Context(), objects, testRootBucket, testSeedInstance)
+	require.NoError(t, err)
+	assert.Equal(t, []string{rec.Pointer.Key}, keys, "the pointer names the one template object")
+}
+
+// A seed that cannot be written fails the launch as InternalError, and no EFI
+// volume is created in its place.
+func TestGenerateVolumes_FailedSeedIsInternalError(t *testing.T) {
+	svc, provider, _, _ := uefiSeedingService(t)
+	svc.vmMgr = vm.NewManager()
+	input, instance := uefiLaunch()
+
+	_, err := svc.GenerateVolumes(t.Context(), input, instance)
+
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+	listed, err := provider.ListVolumes(t.Context(), ebsprovider.ListVolumesRequest{Versioned: ebsprovider.NewVersioned()})
+	require.NoError(t, err)
+	for _, v := range listed.Volumes {
+		assert.False(t, strings.HasSuffix(v.ID, "-efi"), v.ID)
+	}
+}
+
+// putImageStore gives ami-1 a store object holding vars and names it in the
+// AMI document the launch reads.
+func putImageStore(t *testing.T, svc *InstanceServiceImpl, objects *objectstore.MemoryObjectStore, key string, vars []byte) {
+	t.Helper()
+	if strings.HasPrefix(key, uefistore.KeyPrefix("ami-1")) {
+		require.NoError(t, uefistore.PutObject(t.Context(), objects, testRootBucket, key, vars, uefistore.ObjectMeta{
+			OwnerID: "ami-1", WriterNode: "node-b", Source: uefistore.SourceInstance("i-0000000000000000a"),
+			Firmware: uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: int64(len(vars)), CodeSHA256: "other"},
+		}))
+	}
+	loader := svc.amiLoader.(*fakeAMILoader)
+	ami := loader.byID["ami-1"]
+	ami.UEFIData = uefistore.AMIRef(key, uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: int64(len(vars)), CodeSHA256: "other"})
+	loader.byID["ami-1"] = ami
+}
+
+// An AMI that names a store seeds the new instance's own object from it.
+func TestGenerateVolumes_UEFILaunchSeedsFromTheAMIsStore(t *testing.T) {
+	svc, _, records, objects := uefiSeedingService(t)
+	vars := make([]byte, 4096)
+	copy(vars, "image variables")
+	putImageStore(t, svc, objects, uefistore.KeyPrefix("ami-1")+"k", vars)
+	input, instance := uefiLaunch()
+
+	_, err := svc.GenerateVolumes(t.Context(), input, instance)
+	require.NoError(t, err)
+
+	rec, _, err := records.Get(t.Context(), testSeedInstance)
+	require.NoError(t, err)
+	require.NotNil(t, rec.Pointer)
+	got, _, err := uefistore.ReadObject(t.Context(), objects, testRootBucket, rec.Pointer.Key, 4096)
+	require.NoError(t, err)
+	assert.Equal(t, vars, got)
+}
+
+// An AMI document whose store reference is outside the image's own prefix is
+// refused, and the launch records no store.
+func TestGenerateVolumes_AMIStoreOutsideItsPrefixIsInternalError(t *testing.T) {
+	svc, _, records, objects := uefiSeedingService(t)
+	putImageStore(t, svc, objects, uefistore.KeyPrefix("i-0000000000000000a")+"k", make([]byte, 4096))
+	input, instance := uefiLaunch()
+
+	_, err := svc.GenerateVolumes(t.Context(), input, instance)
+
+	require.EqualError(t, err, awserrors.ErrorServerInternal)
+	_, _, err = records.Get(t.Context(), testSeedInstance)
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+}

spinifex/domains/ec2/instance/uefistore/cleanup.go

@@ -25,9 +25,9 @@ func (s *Store) Exists(ctx context.Context, instanceID string) (bool, error) {
 	return s.kv.Exists(ctx, instanceID)
 }
 
-// MarkDeleting records that instanceID's store is to be deleted, and drops any
-// write-back still owed. It reports false with no error when the instance has
-// no store record. Only a terminate or an operator's forget may call it.
+// MarkDeleting records that instanceID's store is to be deleted. It reports
+// false with no error when the instance has no store record. Only a terminate
+// or an operator's forget may call it.
 func (s *Store) MarkDeleting(ctx context.Context, instanceID string) (bool, error) {
 	for range markAttempts {
 		rec, rev, err := s.Get(ctx, instanceID)
@@ -41,7 +41,6 @@ func (s *Store) MarkDeleting(ctx context.Context, instanceID string) (bool, erro
 			return true, nil
 		}
 		rec.State = StateDeleting
-		rec.WriteBack = nil
 		_, err = s.CompareAndSet(ctx, rec, rev)
 		if err == nil {
 			return true, nil

spinifex/domains/ec2/instance/uefistore/cleanup_test.go

@@ -37,7 +37,7 @@ func keysUnder(t *testing.T, objects objectstore.ObjectStore, prefix string) []s
 }
 
 // Deletion is final: a writer holding the revision of a deleting record, such
-// as a write-back retry, must not bring the store back to live.
+// as a late pointer move, must not bring the store back to live.
 func TestCompareAndSet_RefusesDeletingToLive(t *testing.T) {
 	s, _ := openStore(t)
 	rev, err := s.Create(t.Context(), liveRecord())
@@ -47,8 +47,7 @@ func TestCompareAndSet_RefusesDeletingToLive(t *testing.T) {
 	rev, err = s.CompareAndSet(t.Context(), deleting, rev)
 	require.NoError(t, err)
 
-	revived := liveRecord()
-	revived.WriteBack = writeBack()
+	revived := pointedRecord()
 	_, err = s.CompareAndSet(t.Context(), revived, rev)
 	require.ErrorIs(t, err, uefistore.ErrDeleting)
 
@@ -58,24 +57,9 @@ func TestCompareAndSet_RefusesDeletingToLive(t *testing.T) {
 	assert.Equal(t, uefistore.StateDeleting, got.State)
 }
 
-// A write-back owed on a deleting store would put an object behind the delete.
-func TestCompareAndSet_RefusesDeletingWithAWriteBack(t *testing.T) {
+func TestMarkDeleting_MovesLiveToDeletingAndKeepsWhatItNames(t *testing.T) {
 	s, _ := openStore(t)
-	rev, err := s.Create(t.Context(), liveRecord())
-	require.NoError(t, err)
-	rec := liveRecord()
-	rec.State = uefistore.StateDeleting
-	rec.WriteBack = writeBack()
-
-	_, err = s.CompareAndSet(t.Context(), rec, rev)
-	require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
-}
-
-func TestMarkDeleting_MovesLiveToDeletingAndDropsTheWriteBack(t *testing.T) {
-	s, _ := openStore(t)
-	rec := liveRecord()
-	rec.WriteBack = writeBack()
-	_, err := s.Create(t.Context(), rec)
+	_, err := s.Create(t.Context(), pointedRecord())
 	require.NoError(t, err)
 
 	has, err := s.MarkDeleting(t.Context(), instanceID)
@@ -85,8 +69,8 @@ func TestMarkDeleting_MovesLiveToDeletingAndDropsTheWriteBack(t *testing.T) {
 	got, rev, err := s.Get(t.Context(), instanceID)
 	require.NoError(t, err)
 	assert.Equal(t, uefistore.StateDeleting, got.State)
-	assert.Nil(t, got.WriteBack)
 	assert.Equal(t, "111122223333", got.AccountID)
+	assert.Equal(t, pointer("b"), got.Pointer, "the pointer stays readable until the prefix is gone")
 
 	has, err = s.MarkDeleting(t.Context(), instanceID)
 	require.NoError(t, err)

spinifex/domains/ec2/instance/uefistore/fence.go

@@ -0,0 +1,93 @@
+package uefistore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+)
+
+// ClaimPrefix is the kvsync prefix of every store's claim. An instance's claim
+// is ClaimPrefix+instanceID, and names the node that may hold variable changes
+// no object has yet.
+const ClaimPrefix = "uefi."
+
+// NewClaims returns node's group of store claims over store. The daemon builds
+// it once per process, as kvsync requires of every prefix.
+func NewClaims(store kvsync.KVStore, node string) *kvsync.ClaimGroup {
+	return kvsync.NewClaimGroup(store, ClaimPrefix, node)
+}
+
+// ErrFenced is a stamp or pointer move refused because the record carries a
+// claim generation other than the caller's: a later acquisition has stamped
+// it, so the caller no longer owns the store and must change nothing.
+var ErrFenced = errors.New("uefistore: store record is fenced to another claim generation")
+
+// fenceAttempts bounds the read-then-write loops of Stamp and MovePointer. A
+// lost race is re-read, so a concurrent stamp is seen and refused, not retried.
+const fenceAttempts = 5
+
+// Stamp raises instanceID's record to generation, the claim token the caller
+// has just acquired or adopted, and returns the record it stamped, whose
+// pointer the caller boots from. One above generation is ErrFenced, and one
+// deleting is ErrDeleting. A record already at generation is still written by
+// CAS, because a follower can answer the read with an older pointer.
+func (s *Store) Stamp(ctx context.Context, instanceID string, generation uint64) (*UEFIStoreRecord, error) {
+	if generation == 0 {
+		return nil, fmt.Errorf("uefistore: stamp %s: generation 0 is no claim", instanceID)
+	}
+	for range fenceAttempts {
+		rec, rev, err := s.Get(ctx, instanceID)
+		if err != nil {
+			return nil, err
+		}
+		switch {
+		case rec.State == StateDeleting:
+			return nil, fmt.Errorf("%w: %s", ErrDeleting, instanceID)
+		case rec.Generation > generation:
+			return nil, fmt.Errorf("%w: %s is at generation %d, above the claim's %d", ErrFenced, instanceID, rec.Generation, generation)
+		}
+		rec.Generation = generation
+		_, err = s.CompareAndSet(ctx, rec, rev)
+		if err == nil {
+			return rec, nil
+		}
+		if !errors.Is(err, kvstore.ErrConflict) {
+			return nil, err
+		}
+	}
+	return nil, fmt.Errorf("uefistore: stamp %s: %w after %d attempts", instanceID, kvstore.ErrConflict, fenceAttempts)
+}
+
+// MovePointer points instanceID's record at ref only while the record is at
+// generation. A record at any other generation is ErrFenced and is left
+// unchanged. The leader answers the read, because a caller drops its copy on
+// ErrFenced or ErrDeleting, and the CAS keeps a lost race from succeeding.
+func (s *Store) MovePointer(ctx context.Context, instanceID string, generation uint64, ref UEFIVarStoreRef) error {
+	if err := ref.validate(instanceID); err != nil {
+		return fmt.Errorf("%w: move %s: %w", ErrInvalidRecord, instanceID, err)
+	}
+	for range fenceAttempts {
+		rec, rev, err := s.LeaderGet(ctx, instanceID)
+		if err != nil {
+			return err
+		}
+		switch {
+		case rec.State == StateDeleting:
+			return fmt.Errorf("%w: %s", ErrDeleting, instanceID)
+		case rec.Generation != generation:
+			return fmt.Errorf("%w: %s is at generation %d, and the writer holds %d", ErrFenced, instanceID, rec.Generation, generation)
+		}
+		rec.Pointer = &ref
+		_, err = s.CompareAndSet(ctx, rec, rev)
+		if err == nil {
+			return nil
+		}
+		if !errors.Is(err, kvstore.ErrConflict) {
+			return err
+		}
+	}
+	return fmt.Errorf("uefistore: move %s: %w after %d attempts", instanceID, kvstore.ErrConflict, fenceAttempts)
+}

spinifex/domains/ec2/instance/uefistore/fence_test.go

@@ -0,0 +1,213 @@
+package uefistore_test
+
+import (
+	"testing"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func revisionOf(t *testing.T, s *uefistore.Store) uint64 {
+	t.Helper()
+	_, rev, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	return rev
+}
+
+func TestStamp_RaisesTheGenerationAndReturnsThePointerToBootFrom(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), pointedRecord())
+	require.NoError(t, err)
+
+	rec, err := s.Stamp(t.Context(), instanceID, 9)
+	require.NoError(t, err)
+	assert.Equal(t, uint64(9), rec.Generation)
+	assert.Equal(t, pointer("b"), rec.Pointer)
+
+	got, _, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, rec, got, "the returned record is the one stored")
+}
+
+// A daemon restart adopts its claim at the same generation. The stamp still
+// writes by CAS, so a pointer read from a lagging follower fails the write and
+// is read again rather than booted from.
+func TestStamp_AtTheRecordsGenerationConfirmsTheReadByCAS(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), pointedRecord())
+	require.NoError(t, err)
+	before := revisionOf(t, s)
+
+	rec, err := s.Stamp(t.Context(), instanceID, 7)
+	require.NoError(t, err)
+	assert.Equal(t, pointer("b"), rec.Pointer)
+	assert.Equal(t, uint64(7), rec.Generation)
+	assert.Greater(t, revisionOf(t, s), before, "the read must be confirmed by a revision-checked write")
+}
+
+func TestStamp_RefusesAndChangesNothing(t *testing.T) {
+	tests := []struct {
+		name       string
+		deleting   bool
+		generation uint64
+		want       error
+	}{
+		{name: "a later acquisition stamped it", generation: 6, want: uefistore.ErrFenced},
+		{name: "the store is being deleted", deleting: true, generation: 8, want: uefistore.ErrDeleting},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s, _ := openStore(t)
+			rev, err := s.Create(t.Context(), pointedRecord())
+			require.NoError(t, err)
+			if tt.deleting {
+				_, err = s.MarkDeleting(t.Context(), instanceID)
+				require.NoError(t, err)
+				rev = revisionOf(t, s)
+			}
+
+			_, err = s.Stamp(t.Context(), instanceID, tt.generation)
+			require.ErrorIs(t, err, tt.want)
+			assert.Equal(t, rev, revisionOf(t, s))
+		})
+	}
+}
+
+func TestStamp_MissingRecordIsNotFound(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Stamp(t.Context(), instanceID, 1)
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+}
+
+// The record keeps only the current version, so each move replaces the
+// pointer and names nothing it replaced.
+func TestMovePointer_ReplacesThePointer(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), pointedRecord())
+	require.NoError(t, err)
+
+	for _, name := range []string{"c", "d"} {
+		require.NoError(t, s.MovePointer(t.Context(), instanceID, 7, *pointer(name)))
+	}
+
+	got, _, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, pointer("d"), got.Pointer)
+	assert.Equal(t, uint64(7), got.Generation)
+}
+
+// A move to the key the record already names is still a CAS at the revision
+// read, so a fenced writer that read a lagging follower cannot see success.
+func TestMovePointer_ToTheCurrentKeyConfirmsTheReadByCAS(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), pointedRecord())
+	require.NoError(t, err)
+	before := revisionOf(t, s)
+
+	require.NoError(t, s.MovePointer(t.Context(), instanceID, 7, *pointer("b")))
+	assert.Greater(t, revisionOf(t, s), before)
+}
+
+func TestMovePointer_RefusesAndChangesNothing(t *testing.T) {
+	tests := []struct {
+		name       string
+		deleting   bool
+		generation uint64
+		ref        *uefistore.UEFIVarStoreRef
+		want       error
+	}{
+		{name: "an earlier generation", generation: 6, ref: pointer("c"), want: uefistore.ErrFenced},
+		{name: "a later generation than the stamp", generation: 8, ref: pointer("c"), want: uefistore.ErrFenced},
+		{name: "a deleting store", deleting: true, generation: 7, ref: pointer("c"), want: uefistore.ErrDeleting},
+		{name: "a key under another instance", generation: 7,
+			ref:  &uefistore.UEFIVarStoreRef{Key: uefistore.KeyPrefix("i-0fedcba9876543210") + "c", Firmware: pointer("c").Firmware, WriterNode: "n"},
+			want: uefistore.ErrInvalidRecord},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s, _ := openStore(t)
+			_, err := s.Create(t.Context(), pointedRecord())
+			require.NoError(t, err)
+			if tt.deleting {
+				_, err = s.MarkDeleting(t.Context(), instanceID)
+				require.NoError(t, err)
+			}
+			rev := revisionOf(t, s)
+
+			err = s.MovePointer(t.Context(), instanceID, tt.generation, *tt.ref)
+			require.ErrorIs(t, err, tt.want)
+			got, gotRev, err := s.Get(t.Context(), instanceID)
+			require.NoError(t, err)
+			assert.Equal(t, rev, gotRev)
+			assert.Equal(t, pointer("b"), got.Pointer)
+		})
+	}
+}
+
+// claimedStore is a store record and a claim group per node over one
+// JetStream, as two daemons see them.
+type claimedStore struct {
+	records *uefistore.Store
+	a, b    *kvsync.Claim
+}
+
+func newClaimedStore(t *testing.T) *claimedStore {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	records := uefistore.NewStore(js)
+	require.NoError(t, records.Open(t.Context()))
+	_, err := records.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+	return &claimedStore{
+		records: records,
+		a:       uefistore.NewClaims(kvstore.OpenSyncBucket(js), "node-a").Get(instanceID),
+		b:       uefistore.NewClaims(kvstore.OpenSyncBucket(js), "node-b").Get(instanceID),
+	}
+}
+
+// A node partitioned long enough to be taken over is fenced by the new
+// owner's stamp: its pointer move fails and the record keeps what it had.
+func TestFence_TheOldHoldersMoveFailsAfterTheNewOwnersStamp(t *testing.T) {
+	c := newClaimedStore(t)
+	first, err := c.a.TryLock(t.Context())
+	require.NoError(t, err)
+	_, err = c.records.Stamp(t.Context(), instanceID, first.Generation)
+	require.NoError(t, err)
+	require.NoError(t, c.records.MovePointer(t.Context(), instanceID, first.Generation, *pointer("b")))
+
+	second, err := c.b.TakeOver(t.Context(), first)
+	require.NoError(t, err)
+	_, err = c.records.Stamp(t.Context(), instanceID, second.Generation)
+	require.NoError(t, err)
+	rev := revisionOf(t, c.records)
+
+	err = c.records.MovePointer(t.Context(), instanceID, first.Generation, *pointer("late"))
+	require.ErrorIs(t, err, uefistore.ErrFenced)
+	got, gotRev, err := c.records.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, rev, gotRev)
+	assert.Equal(t, pointer("b"), got.Pointer)
+	assert.Equal(t, second.Generation, got.Generation)
+}
+
+// The new owner reads the pointer after it stamps, so a move the old holder
+// landed between the takeover and the stamp is the one it boots from.
+func TestFence_AMoveBetweenTakeoverAndStampIsThePointerTheNewOwnerBoots(t *testing.T) {
+	c := newClaimedStore(t)
+	first, err := c.a.TryLock(t.Context())
+	require.NoError(t, err)
+	_, err = c.records.Stamp(t.Context(), instanceID, first.Generation)
+	require.NoError(t, err)
+
+	second, err := c.b.TakeOver(t.Context(), first)
+	require.NoError(t, err)
+	require.NoError(t, c.records.MovePointer(t.Context(), instanceID, first.Generation, *pointer("landed")))
+
+	rec, err := c.records.Stamp(t.Context(), instanceID, second.Generation)
+	require.NoError(t, err)
+	assert.Equal(t, keyOf("landed"), rec.Pointer.Key)
+}

spinifex/domains/ec2/instance/uefistore/image.go

@@ -0,0 +1,145 @@
+package uefistore
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+)
+
+const (
+	// GetTimeout bounds one get of a store object, a fixed-size region.
+	GetTimeout = 10 * time.Second
+
+	// bodySlack is how far a compressed body may exceed its region before it
+	// is refused; gzip grows incompressible data by far less.
+	bodySlack = 1 << 20
+)
+
+// ErrObjectNotFound is a store object that does not exist.
+var ErrObjectNotFound = errors.New("uefistore: store object does not exist")
+
+// SourceImage is the source of a store copied from imageID's store.
+func SourceImage(imageID string) string { return "image:" + imageID }
+
+// SourceInstance is the source of an image store copied from instanceID's.
+func SourceInstance(instanceID string) string { return "instance:" + instanceID }
+
+// ImageVarStore is the store an AMI document names: one object under the
+// image's own prefix and the firmware it was written under.
+type ImageVarStore struct {
+	ImageID  string
+	Key      string
+	Firmware FirmwareIdentity
+}
+
+// ImageVarStoreFromAMI returns the store ami names, or nil when it names none.
+// A reference whose key is not one object under the image's own prefix, or
+// that names no firmware, is ErrInvalidObject.
+func ImageVarStoreFromAMI(ami ebsmetadata.AMI) (*ImageVarStore, error) {
+	ref := ami.UEFIData
+	if ref == nil {
+		return nil, nil
+	}
+	if !strings.HasPrefix(ami.ImageID, "ami-") || !ownsKey(ami.ImageID, ref.Key) {
+		return nil, fmt.Errorf("%w: %s names UEFI store %q outside %s", ErrInvalidObject, ami.ImageID, ref.Key, KeyPrefix(ami.ImageID))
+	}
+	if ref.Firmware.Arch == "" || ref.Firmware.VarsSize <= 0 {
+		return nil, fmt.Errorf("%w: %s names UEFI store %q with no firmware arch and VARS size", ErrInvalidObject, ami.ImageID, ref.Key)
+	}
+	return &ImageVarStore{
+		ImageID: ami.ImageID,
+		Key:     ref.Key,
+		Firmware: FirmwareIdentity{
+			Arch: ref.Firmware.Arch, VarsSize: ref.Firmware.VarsSize, CodeSHA256: ref.Firmware.CodeSHA256,
+		},
+	}, nil
+}
+
+// AMIRef returns the reference an AMI document records for key under fw.
+func AMIRef(key string, fw FirmwareIdentity) *ebsmetadata.UEFIDataRef {
+	return &ebsmetadata.UEFIDataRef{
+		Key:      key,
+		Firmware: ebsmetadata.UEFIFirmware{Arch: fw.Arch, VarsSize: fw.VarsSize, CodeSHA256: fw.CodeSHA256},
+	}
+}
+
+// ReadObject gets key within GetTimeout and returns its decompressed region,
+// which must be exactly varsSize bytes, with the object's metadata. A missing
+// object is ErrObjectNotFound and a malformed one ErrInvalidObject.
+func ReadObject(ctx context.Context, objects objectstore.ObjectStore, bucket, key string, varsSize int64) ([]byte, map[string]*string, error) {
+	getCtx, cancel := context.WithTimeout(ctx, GetTimeout)
+	defer cancel()
+
+	out, err := objects.GetObject(getCtx, &s3.GetObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
+	if err != nil {
+		if objectstore.IsNoSuchKeyError(err) {
+			return nil, nil, fmt.Errorf("%w: %s", ErrObjectNotFound, key)
+		}
+		return nil, nil, fmt.Errorf("get UEFI store %s: %w", key, err)
+	}
+	defer out.Body.Close()
+
+	limit := varsSize + bodySlack
+	raw, err := io.ReadAll(io.LimitReader(out.Body, limit+1))
+	if err != nil {
+		return nil, nil, fmt.Errorf("read UEFI store %s: %w", key, err)
+	}
+	if int64(len(raw)) > limit {
+		return nil, nil, fmt.Errorf("%w: %s is over %d bytes compressed", ErrInvalidObject, key, limit)
+	}
+	gz, err := gzip.NewReader(bytes.NewReader(raw))
+	if err != nil {
+		return nil, nil, fmt.Errorf("%w: %s is not gzip: %w", ErrInvalidObject, key, err)
+	}
+	region, err := io.ReadAll(io.LimitReader(gz, varsSize+1))
+	if err != nil {
+		return nil, nil, fmt.Errorf("%w: %s does not decompress: %w", ErrInvalidObject, key, err)
+	}
+	if int64(len(region)) != varsSize {
+		return nil, nil, fmt.Errorf("%w: %s holds a %d-byte region, not the %d bytes its firmware needs",
+			ErrInvalidObject, key, len(region), varsSize)
+	}
+	return region, out.Metadata, nil
+}
+
+// CopyObject reads srcKey, a region of meta.Firmware's VARS size, and puts it
+// at a fresh key under meta.OwnerID, which it returns. A meta with no CodePath
+// takes the source object's.
+func CopyObject(ctx context.Context, objects objectstore.ObjectStore, bucket, srcKey string, meta ObjectMeta) (string, error) {
+	region, srcMeta, err := ReadObject(ctx, objects, bucket, srcKey, meta.Firmware.VarsSize)
+	if err != nil {
+		return "", err
+	}
+	if meta.CodePath == "" {
+		meta.CodePath = metadataValue(srcMeta, "firmware-code-path")
+	}
+	key, err := NewObjectKey(meta.OwnerID)
+	if err != nil {
+		return "", err
+	}
+	if err := PutObject(ctx, objects, bucket, key, region, meta); err != nil {
+		return "", err
+	}
+	return key, nil
+}
+
+// metadataValue reads name from S3 object metadata, whose keys the SDK
+// returns canonicalised.
+func metadataValue(md map[string]*string, name string) string {
+	for k, v := range md {
+		if strings.EqualFold(k, name) {
+			return aws.StringValue(v)
+		}
+	}
+	return ""
+}

spinifex/domains/ec2/instance/uefistore/image_test.go

@@ -0,0 +1,70 @@
+package uefistore_test
+
+import (
+	"testing"
+
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const imageID = "ami-0123456789abcdef0"
+
+// A launch only seeds from a store under the image's own prefix with a
+// firmware it can check; an AMI with no reference seeds from the template.
+func TestImageVarStoreFromAMI(t *testing.T) {
+	fw := ebsmetadata.UEFIFirmware{Arch: "x86_64", VarsSize: 4096, CodeSHA256: "c"}
+	own := uefistore.KeyPrefix(imageID) + "k"
+
+	got, err := uefistore.ImageVarStoreFromAMI(ebsmetadata.AMI{ImageID: imageID})
+	require.NoError(t, err)
+	assert.Nil(t, got)
+
+	got, err = uefistore.ImageVarStoreFromAMI(ebsmetadata.AMI{ImageID: imageID, UEFIData: &ebsmetadata.UEFIDataRef{Key: own, Firmware: fw}})
+	require.NoError(t, err)
+	assert.Equal(t, &uefistore.ImageVarStore{
+		ImageID: imageID, Key: own, Firmware: uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: 4096, CodeSHA256: "c"},
+	}, got)
+
+	for name, ref := range map[string]ebsmetadata.UEFIDataRef{
+		"another image's key":   {Key: uefistore.KeyPrefix("ami-0fffffffffffffff0") + "k", Firmware: fw},
+		"an instance's key":     {Key: uefistore.KeyPrefix(instanceID) + "k", Firmware: fw},
+		"a nested key":          {Key: own + "/deeper", Firmware: fw},
+		"no firmware arch":      {Key: own, Firmware: ebsmetadata.UEFIFirmware{VarsSize: 4096}},
+		"no firmware VARS size": {Key: own, Firmware: ebsmetadata.UEFIFirmware{Arch: "x86_64"}},
+	} {
+		_, err := uefistore.ImageVarStoreFromAMI(ebsmetadata.AMI{ImageID: imageID, UEFIData: &ref})
+		require.ErrorIs(t, err, uefistore.ErrInvalidObject, name)
+	}
+}
+
+// A copy is a fresh key under the new owner holding the source's region, and
+// the source is left in place.
+func TestCopyObject_PutsTheRegionAtAFreshKeyUnderTheNewOwner(t *testing.T) {
+	objects := objectstore.NewMemoryObjectStore()
+	meta := testMeta(instanceID)
+	region := make([]byte, meta.Firmware.VarsSize)
+	copy(region, "variables")
+	src, err := uefistore.NewObjectKey(instanceID)
+	require.NoError(t, err)
+	require.NoError(t, uefistore.PutObject(t.Context(), objects, testBucket, src, region, meta))
+
+	dst := testMeta(imageID)
+	dst.Source = uefistore.SourceInstance(instanceID)
+	key, err := uefistore.CopyObject(t.Context(), objects, testBucket, src, dst)
+	require.NoError(t, err)
+
+	keys, err := uefistore.ListObjects(t.Context(), objects, testBucket, imageID)
+	require.NoError(t, err)
+	assert.Equal(t, []string{key}, keys)
+	got, _, err := uefistore.ReadObject(t.Context(), objects, testBucket, key, meta.Firmware.VarsSize)
+	require.NoError(t, err)
+	assert.Equal(t, region, got)
+	_, _, err = uefistore.ReadObject(t.Context(), objects, testBucket, src, meta.Firmware.VarsSize)
+	require.NoError(t, err, "the source stays")
+
+	_, err = uefistore.CopyObject(t.Context(), objects, testBucket, uefistore.KeyPrefix(instanceID)+"gone", dst)
+	require.ErrorIs(t, err, uefistore.ErrObjectNotFound)
+}

spinifex/domains/ec2/instance/uefistore/object.go

@@ -0,0 +1,121 @@
+package uefistore
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/rand"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+)
+
+const (
+	// SourceTemplate is a store seeded from the launching node's VARS template.
+	SourceTemplate = "template"
+	// SourceWriteBack is a store written back from a node's live copy.
+	SourceWriteBack = "write-back"
+
+	// PutTimeout bounds one put of a store object. The object is a fixed-size
+	// region, so a put that takes longer is a wedged object store.
+	PutTimeout = 10 * time.Second
+)
+
+// ErrInvalidObject is a store object whose key, region or metadata cannot be
+// written as the owner's store.
+var ErrInvalidObject = errors.New("uefistore: invalid store object")
+
+// ObjectMeta is what a store object records about itself, as x-amz-meta-
+// entries, so its owner, writer and firmware are known from the object alone.
+type ObjectMeta struct {
+	OwnerID    string
+	WriterNode string
+	Firmware   FirmwareIdentity
+	CodePath   string
+	Source     string
+}
+
+// Metadata returns m as object metadata, keyed by the owner's kind.
+func (m ObjectMeta) Metadata() (map[string]*string, error) {
+	var ownerKey string
+	switch {
+	case strings.HasPrefix(m.OwnerID, "i-"):
+		ownerKey = "instance-id"
+	case strings.HasPrefix(m.OwnerID, "ami-"):
+		ownerKey = "image-id"
+	default:
+		return nil, fmt.Errorf("%w: owner %q is neither an instance nor an image", ErrInvalidObject, m.OwnerID)
+	}
+	switch {
+	case m.WriterNode == "":
+		return nil, fmt.Errorf("%w: %s names no writer node", ErrInvalidObject, m.OwnerID)
+	case m.Source == "":
+		return nil, fmt.Errorf("%w: %s names no source", ErrInvalidObject, m.OwnerID)
+	case m.Firmware.Arch == "" || m.Firmware.VarsSize <= 0 || m.Firmware.CodeSHA256 == "":
+		return nil, fmt.Errorf("%w: %s has an incomplete firmware identity", ErrInvalidObject, m.OwnerID)
+	}
+	return map[string]*string{
+		ownerKey:               aws.String(m.OwnerID),
+		"writer-node":          aws.String(m.WriterNode),
+		"firmware-arch":        aws.String(m.Firmware.Arch),
+		"firmware-vars-size":   aws.String(strconv.FormatInt(m.Firmware.VarsSize, 10)),
+		"firmware-code-sha256": aws.String(m.Firmware.CodeSHA256),
+		"firmware-code-path":   aws.String(m.CodePath),
+		"source":               aws.String(m.Source),
+	}, nil
+}
+
+// NewObjectKey returns a fresh key under ownerID's prefix, named by a random
+// version 4 UUID, so no write ever replaces an object a record points at.
+func NewObjectKey(ownerID string) (string, error) {
+	var b [16]byte
+	if _, err := rand.Read(b[:]); err != nil {
+		return "", fmt.Errorf("generate UEFI store key for %s: %w", ownerID, err)
+	}
+	b[6] = b[6]&0x0f | 0x40
+	b[8] = b[8]&0x3f | 0x80
+	return fmt.Sprintf("%s%x-%x-%x-%x-%x", KeyPrefix(ownerID), b[0:4], b[4:6], b[6:8], b[8:10], b[10:16]), nil
+}
+
+// PutObject writes region gzip-compressed to key with meta, within
+// PutTimeout. The region must be exactly the firmware's VARS size, because
+// pflash refuses any other.
+func PutObject(ctx context.Context, objects objectstore.ObjectStore, bucket, key string, region []byte, meta ObjectMeta) error {
+	if !ownsKey(meta.OwnerID, key) {
+		return fmt.Errorf("%w: key %q is not under %s", ErrInvalidObject, key, KeyPrefix(meta.OwnerID))
+	}
+	if int64(len(region)) != meta.Firmware.VarsSize {
+		return fmt.Errorf("%w: %s is a %d-byte region under firmware with a %d-byte VARS region",
+			ErrInvalidObject, key, len(region), meta.Firmware.VarsSize)
+	}
+	md, err := meta.Metadata()
+	if err != nil {
+		return err
+	}
+	var body bytes.Buffer
+	zw := gzip.NewWriter(&body)
+	if _, err := zw.Write(region); err != nil {
+		return fmt.Errorf("compress UEFI store %s: %w", key, err)
+	}
+	if err := zw.Close(); err != nil {
+		return fmt.Errorf("compress UEFI store %s: %w", key, err)
+	}
+
+	putCtx, cancel := context.WithTimeout(ctx, PutTimeout)
+	defer cancel()
+	if _, err := objects.PutObject(putCtx, &s3.PutObjectInput{
+		Bucket:   aws.String(bucket),
+		Key:      aws.String(key),
+		Body:     bytes.NewReader(body.Bytes()),
+		Metadata: md,
+	}); err != nil {
+		return fmt.Errorf("put UEFI store %s: %w", key, err)
+	}
+	return nil
+}

spinifex/domains/ec2/instance/uefistore/object_test.go

@@ -0,0 +1,164 @@
+package uefistore_test
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"errors"
+	"io"
+	"regexp"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const testBucket = "instances"
+
+func testMeta(owner string) uefistore.ObjectMeta {
+	return uefistore.ObjectMeta{
+		OwnerID:    owner,
+		WriterNode: "node-1",
+		Firmware:   uefistore.FirmwareIdentity{Arch: "arm64", VarsSize: 1 << 16, CodeSHA256: "abc123"},
+		CodePath:   "/usr/share/AAVMF/AAVMF_CODE.fd",
+		Source:     uefistore.SourceTemplate,
+	}
+}
+
+var uuidV4 = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$`)
+
+// Every key is a fresh version 4 UUID directly under the owner's prefix, so
+// a write never lands on an object a record already points at.
+func TestNewObjectKey_IsAFreshUUIDUnderTheOwnersPrefix(t *testing.T) {
+	seen := map[string]bool{}
+	for range 64 {
+		key, err := uefistore.NewObjectKey(instanceID)
+		require.NoError(t, err)
+		name, ok := bytes.CutPrefix([]byte(key), []byte(uefistore.KeyPrefix(instanceID)))
+		require.True(t, ok, key)
+		assert.Regexp(t, uuidV4, string(name))
+		assert.False(t, seen[key], "key %s repeated", key)
+		seen[key] = true
+	}
+}
+
+func TestObjectMeta_Metadata_NamesTheOwnerByKind(t *testing.T) {
+	md, err := testMeta(instanceID).Metadata()
+	require.NoError(t, err)
+	assert.Equal(t, map[string]string{
+		"instance-id":          instanceID,
+		"writer-node":          "node-1",
+		"firmware-arch":        "arm64",
+		"firmware-vars-size":   "65536",
+		"firmware-code-sha256": "abc123",
+		"firmware-code-path":   "/usr/share/AAVMF/AAVMF_CODE.fd",
+		"source":               "template",
+	}, aws.StringValueMap(md))
+
+	img := testMeta("ami-0123456789abcdef0")
+	img.Source = "image:ami-0fedcba9876543210"
+	md, err = img.Metadata()
+	require.NoError(t, err)
+	assert.Equal(t, "ami-0123456789abcdef0", aws.StringValue(md["image-id"]))
+	assert.NotContains(t, md, "instance-id")
+	assert.Equal(t, "image:ami-0fedcba9876543210", aws.StringValue(md["source"]))
+}
+
+func TestObjectMeta_Metadata_RefusesAnIncompleteObject(t *testing.T) {
+	tests := map[string]func(*uefistore.ObjectMeta){
+		"unknown owner":  func(m *uefistore.ObjectMeta) { m.OwnerID = "vol-1" },
+		"no writer":      func(m *uefistore.ObjectMeta) { m.WriterNode = "" },
+		"no source":      func(m *uefistore.ObjectMeta) { m.Source = "" },
+		"no arch":        func(m *uefistore.ObjectMeta) { m.Firmware.Arch = "" },
+		"no vars size":   func(m *uefistore.ObjectMeta) { m.Firmware.VarsSize = 0 },
+		"no code sha256": func(m *uefistore.ObjectMeta) { m.Firmware.CodeSHA256 = "" },
+	}
+	for name, mutate := range tests {
+		t.Run(name, func(t *testing.T) {
+			m := testMeta(instanceID)
+			mutate(&m)
+			_, err := m.Metadata()
+			require.ErrorIs(t, err, uefistore.ErrInvalidObject)
+		})
+	}
+}
+
+// The 64 MiB arm64 region is mostly erased flash, so it is stored compressed
+// and decompresses to the exact region.
+func TestPutObject_StoresTheRegionCompressed(t *testing.T) {
+	objects := objectstore.NewMemoryObjectStore()
+	meta := testMeta(instanceID)
+	region := bytes.Repeat([]byte{0xff}, int(meta.Firmware.VarsSize))
+	copy(region, "BootOrder")
+	key, err := uefistore.NewObjectKey(instanceID)
+	require.NoError(t, err)
+
+	require.NoError(t, uefistore.PutObject(t.Context(), objects, testBucket, key, region, meta))
+
+	out, err := objects.GetObject(t.Context(), &s3.GetObjectInput{Bucket: aws.String(testBucket), Key: aws.String(key)})
+	require.NoError(t, err)
+	body, err := io.ReadAll(out.Body)
+	require.NoError(t, err)
+	assert.Less(t, len(body), len(region)/10, "the erased region compresses")
+	gz, err := gzip.NewReader(bytes.NewReader(body))
+	require.NoError(t, err)
+	got, err := io.ReadAll(gz)
+	require.NoError(t, err)
+	assert.Equal(t, region, got)
+}
+
+// pflash needs a byte-exact region, and a key outside the owner's prefix is
+// one no store record owns; either is refused before anything is put.
+func TestPutObject_RefusesWhatTheOwnerCannotBoot(t *testing.T) {
+	meta := testMeta(instanceID)
+	good, err := uefistore.NewObjectKey(instanceID)
+	require.NoError(t, err)
+	other, err := uefistore.NewObjectKey("i-0fedcba9876543210")
+	require.NoError(t, err)
+	tests := []struct {
+		name   string
+		key    string
+		region []byte
+	}{
+		{name: "short region", key: good, region: make([]byte, meta.Firmware.VarsSize-1)},
+		{name: "long region", key: good, region: make([]byte, meta.Firmware.VarsSize+1)},
+		{name: "another owner's key", key: other, region: make([]byte, meta.Firmware.VarsSize)},
+		{name: "nested key", key: uefistore.KeyPrefix(instanceID) + "a/b", region: make([]byte, meta.Firmware.VarsSize)},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			objects := objectstore.NewMemoryObjectStore()
+			err := uefistore.PutObject(t.Context(), objects, testBucket, tt.key, tt.region, meta)
+			require.ErrorIs(t, err, uefistore.ErrInvalidObject)
+			out, listErr := objects.ListObjectsV2(t.Context(), &s3.ListObjectsV2Input{Bucket: aws.String(testBucket)})
+			require.NoError(t, listErr)
+			assert.Empty(t, out.Contents)
+		})
+	}
+}
+
+type failingPut struct {
+	*objectstore.MemoryObjectStore
+}
+
+func (failingPut) PutObject(ctx context.Context, _ *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
+	if _, ok := ctx.Deadline(); !ok {
+		return nil, errors.New("put had no deadline")
+	}
+	return nil, errors.New("predastore unreachable")
+}
+
+// A put runs under a deadline, so a wedged object store fails the write
+// rather than holding the caller, and its error reaches the caller.
+func TestPutObject_IsBoundedAndReportsAFailedPut(t *testing.T) {
+	key, err := uefistore.NewObjectKey(instanceID)
+	require.NoError(t, err)
+	meta := testMeta(instanceID)
+	err = uefistore.PutObject(t.Context(), failingPut{objectstore.NewMemoryObjectStore()}, testBucket, key, make([]byte, meta.Firmware.VarsSize), meta)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "predastore unreachable")
+}

spinifex/domains/ec2/instance/uefistore/prune.go

@@ -0,0 +1,73 @@
+package uefistore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+)
+
+// PruneMinAge is how old an object must be before a prune may delete it, so a
+// write-back whose pointer move has not landed yet never loses its object.
+const PruneMinAge = 10 * time.Minute
+
+// PruneResult is what one prune of a prefix did.
+type PruneResult struct {
+	Deleted int
+	// Young counts objects other than keep that were too new to delete, so the
+	// caller prunes the prefix again later.
+	Young int
+}
+
+// PruneObjects deletes every object under ownerID's prefix except keep that was
+// last modified before cutoff. keep must be the key the store record points at;
+// an empty or foreign keep deletes nothing. An object with no modification time
+// counts as young.
+func PruneObjects(ctx context.Context, objects objectstore.ObjectStore, bucket, ownerID, keep string, cutoff time.Time) (PruneResult, error) {
+	var res PruneResult
+	if err := checkOwnedKey(ownerID, keep); err != nil {
+		return res, fmt.Errorf("uefistore: refusing to prune %s: %w", ownerID, err)
+	}
+	contents, _, err := objectstore.ListAll(ctx, objects, &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucket),
+		Prefix: aws.String(KeyPrefix(ownerID)),
+	})
+	if err != nil {
+		return res, fmt.Errorf("uefistore: list %s: %w", KeyPrefix(ownerID), err)
+	}
+	for _, obj := range contents {
+		key := aws.StringValue(obj.Key)
+		if key == keep {
+			continue
+		}
+		if obj.LastModified == nil || !obj.LastModified.Before(cutoff) {
+			res.Young++
+			continue
+		}
+		_, err := objects.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: aws.String(bucket), Key: aws.String(key)})
+		if err != nil && !objectstore.IsNoSuchKeyError(err) {
+			return res, fmt.Errorf("uefistore: delete %s: %w", key, err)
+		}
+		res.Deleted++
+	}
+	return res, nil
+}
+
+// checkOwnedKey fails a key that is empty or not one object under ownerID's
+// prefix.
+func checkOwnedKey(ownerID, key string) error {
+	if !validOwner(ownerID) {
+		return fmt.Errorf("%q is not an instance or image ID", ownerID)
+	}
+	if key == "" {
+		return errors.New("no key to keep")
+	}
+	if !ownsKey(ownerID, key) {
+		return fmt.Errorf("%s is not under %s", key, KeyPrefix(ownerID))
+	}
+	return nil
+}

spinifex/domains/ec2/instance/uefistore/prune_test.go

@@ -0,0 +1,84 @@
+package uefistore_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// datedStore lists each object with the modification time the test set; a
+// key it has none for is listed with none.
+type datedStore struct {
+	*objectstore.MemoryObjectStore
+
+	modified map[string]time.Time
+}
+
+func (s *datedStore) ListObjectsV2(ctx context.Context, in *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
+	out, err := s.MemoryObjectStore.ListObjectsV2(ctx, in)
+	if err != nil {
+		return nil, err
+	}
+	for _, obj := range out.Contents {
+		obj.LastModified = nil
+		if at, ok := s.modified[aws.StringValue(obj.Key)]; ok {
+			obj.LastModified = aws.Time(at)
+		}
+	}
+	return out, nil
+}
+
+// A prune keeps the key it is told to and every object not older than the
+// cutoff, counting those it kept for age, and an undated object counts as
+// young (ADR-0003:S7).
+func TestS7_PruneObjectsKeepsThePointerAndEveryYoungObject(t *testing.T) {
+	const owner = "i-0123456789abcdef0"
+	prefix := uefistore.KeyPrefix(owner)
+	cutoff := time.Unix(1700000000, 0)
+	keep, stale, young, undated, other := prefix+"keep", prefix+"stale", prefix+"young", prefix+"undated", uefistore.KeyPrefix("i-0fedcba9876543210")+"stale"
+	store := &datedStore{MemoryObjectStore: objectstore.NewMemoryObjectStore(), modified: map[string]time.Time{
+		keep:  cutoff.Add(-time.Hour),
+		stale: cutoff.Add(-time.Second),
+		young: cutoff,
+		other: cutoff.Add(-time.Hour),
+	}}
+	for _, k := range []string{keep, stale, young, undated, other} {
+		putObject(t, store, k)
+	}
+
+	res, err := uefistore.PruneObjects(t.Context(), store, bucket, owner, keep, cutoff)
+
+	require.NoError(t, err)
+	assert.Equal(t, uefistore.PruneResult{Deleted: 1, Young: 2}, res)
+	assert.ElementsMatch(t, []string{keep, young, undated}, keysUnder(t, store, prefix),
+		"ADR-0003:S7: a prune must keep the pointed object and every object younger than the cutoff")
+	assert.Equal(t, []string{other}, keysUnder(t, store, uefistore.KeyPrefix("i-0fedcba9876543210")), "another owner's prefix is untouched")
+}
+
+// With no key to keep, or one outside the owner's prefix, a prune cannot tell
+// the pointer from a leftover, so it deletes nothing (ADR-0003:S7).
+func TestS7_PruneObjectsWithoutAKeyToKeepDeletesNothing(t *testing.T) {
+	const owner = "i-0123456789abcdef0"
+	prefix := uefistore.KeyPrefix(owner)
+	store := &datedStore{MemoryObjectStore: objectstore.NewMemoryObjectStore(), modified: map[string]time.Time{
+		prefix + "stale": time.Unix(1, 0),
+	}}
+	putObject(t, store, prefix+"stale")
+
+	for _, keep := range []string{"", uefistore.KeyPrefix("i-0fedcba9876543210") + "stale", prefix + "nested/key"} {
+		_, err := uefistore.PruneObjects(t.Context(), store, bucket, owner, keep, time.Now())
+		require.Error(t, err, keep)
+	}
+	_, err := uefistore.PruneObjects(t.Context(), store, bucket, "i-*", prefix+"stale", time.Now())
+	require.Error(t, err)
+
+	assert.Equal(t, []string{prefix + "stale"}, keysUnder(t, store, prefix),
+		"ADR-0003:S7: a prune with no valid pointer to keep must delete nothing")
+}

spinifex/domains/ec2/instance/uefistore/uefistore.go

@@ -1,6 +1,6 @@
 // Package uefistore holds the types that describe an instance's UEFI variable
-// store and the bucket of store records that carry its deletion and write-back
-// obligations past the life of the instance record.
+// store, the bucket of store records that carry its pointer and deletion
+// obligation past the life of the instance record, and the claim that fences it.
 package uefistore
 
 import (
@@ -53,22 +53,27 @@ type FirmwareIdentity struct {
 // UEFIStoreRecord is the durable owner of one instance's variable store
 // objects. It outlives the instance record, so an obligation recorded here is
 // not lost when a terminated record expires.
+//
+// Pointer names the only version of the store the record keeps, the object the
+// instance boots from, and only a writer whose claim token has generation
+// Generation may move it.
 type UEFIStoreRecord struct {
-	SchemaVersion uint16         `json:"schema_version"`
-	InstanceID    string         `json:"instance_id"`
-	AccountID     string         `json:"account_id"`
-	State         string         `json:"state"`
-	WriteBack     *UEFIWriteBack `json:"write_back,omitempty"`
+	SchemaVersion uint16           `json:"schema_version"`
+	InstanceID    string           `json:"instance_id"`
+	AccountID     string           `json:"account_id"`
+	State         string           `json:"state"`
+	Generation    uint64           `json:"generation"`
+	Pointer       *UEFIVarStoreRef `json:"pointer,omitempty"`
 }
 
-// UEFIWriteBack is a write-back that failed and is still owed. Key is the
-// object every retry puts to, so retries leave no extra objects; SeededFrom is
-// the pointer Node seeded from, which a retry must still find on the record.
-type UEFIWriteBack struct {
-	Node       string    `json:"node"`
-	Key        string    `json:"key"`
-	SeededFrom string    `json:"seeded_from"`
-	Since      time.Time `json:"since"`
+// UEFIVarStoreRef names the store object an instance launches from, with the
+// firmware it was written under, so a launch can check it without reading the
+// object's metadata.
+type UEFIVarStoreRef struct {
+	Key        string           `json:"key"`
+	Firmware   FirmwareIdentity `json:"firmware"`
+	WriterNode string           `json:"writer_node"`
+	WrittenAt  time.Time        `json:"written_at"`
 }
 
 // KeyPrefix is the object-store prefix every store object of ownerID (an
@@ -83,8 +88,8 @@ func ownsKey(ownerID, key string) bool {
 	return ok && name != "" && !strings.Contains(name, "/")
 }
 
-// validate checks rec against key. A write-back key outside the instance's own
-// prefix is refused because a superseded obligation deletes that key.
+// validate checks rec against key. The pointer must name one object directly
+// under the instance's own prefix, because a deletion acts on it.
 func (rec *UEFIStoreRecord) validate(key string) error {
 	switch {
 	case rec.SchemaVersion > RecordSchemaVersion:
@@ -98,29 +103,32 @@ func (rec *UEFIStoreRecord) validate(key string) error {
 		return fmt.Errorf("%w: instance_id %s stored at key %s", ErrInvalidRecord, rec.InstanceID, key)
 	case rec.State != StateLive && rec.State != StateDeleting:
 		return fmt.Errorf("%w: %s has state %q", ErrInvalidRecord, key, rec.State)
-	case rec.State == StateDeleting && rec.WriteBack != nil:
-		// A write-back retry puts an object, which would land behind the delete.
-		return fmt.Errorf("%w: %s is deleting with a write-back still owed", ErrInvalidRecord, key)
 	}
-	if wb := rec.WriteBack; wb != nil {
-		switch {
-		case wb.Node == "":
-			return fmt.Errorf("%w: %s write-back names no node", ErrInvalidRecord, key)
-		case wb.Since.IsZero():
-			return fmt.Errorf("%w: %s write-back has no since", ErrInvalidRecord, key)
-		case !ownsKey(rec.InstanceID, wb.Key):
-			return fmt.Errorf("%w: %s write-back key %q is not under %s", ErrInvalidRecord, key, wb.Key, KeyPrefix(rec.InstanceID))
-		case !ownsKey(rec.InstanceID, wb.SeededFrom):
-			return fmt.Errorf("%w: %s write-back seeded_from %q is not under %s", ErrInvalidRecord, key, wb.SeededFrom, KeyPrefix(rec.InstanceID))
+	if p := rec.Pointer; p != nil {
+		if err := p.validate(rec.InstanceID); err != nil {
+			return fmt.Errorf("%w: %s pointer: %w", ErrInvalidRecord, key, err)
 		}
 	}
 	return nil
 }
 
+// validate checks ref as a pointer of instanceID's store.
+func (ref *UEFIVarStoreRef) validate(instanceID string) error {
+	switch {
+	case !ownsKey(instanceID, ref.Key):
+		return fmt.Errorf("key %q is not under %s", ref.Key, KeyPrefix(instanceID))
+	case ref.Firmware.Arch == "" || ref.Firmware.VarsSize <= 0:
+		return fmt.Errorf("key %q names no firmware arch and VARS size", ref.Key)
+	case ref.WriterNode == "":
+		return fmt.Errorf("key %q names no writer node", ref.Key)
+	}
+	return nil
+}
+
 func bucketConfig() kvstore.Config {
 	return kvstore.Config{
 		Name:        BucketName,
-		Description: "Spinifex UEFI variable store records (deletion and write-back obligations)",
+		Description: "Spinifex UEFI variable store records (pointers and deletion obligations)",
 		History:     1,
 		// Left false: a lost bucket has lost obligations, and an empty
 		// recreated one would report every store as having none.
@@ -172,6 +180,19 @@ func (s *Store) Get(ctx context.Context, instanceID string) (*UEFIStoreRecord, u
 	return rec, rev, nil
 }
 
+// LeaderGet is Get answered by the stream leader, for a caller that deletes
+// objects by what the record does not point at.
+func (s *Store) LeaderGet(ctx context.Context, instanceID string) (*UEFIStoreRecord, uint64, error) {
+	rec, rev, err := s.kv.LeaderGet(ctx, instanceID)
+	if err != nil {
+		return nil, 0, err
+	}
+	if err := rec.validate(instanceID); err != nil {
+		return nil, 0, err
+	}
+	return rec, rev, nil
+}
+
 // Create writes a new record, which must be live: a store is recorded before
 // its first object is written. An existing record is kvstore.ErrExists. Every
 // write stamps rec with RecordSchemaVersion.

spinifex/domains/ec2/instance/uefistore/uefistore_test.go

@@ -30,15 +30,25 @@ func liveRecord() *uefistore.UEFIStoreRecord {
 	return &uefistore.UEFIStoreRecord{InstanceID: instanceID, AccountID: "111122223333", State: uefistore.StateLive}
 }
 
-func writeBack() *uefistore.UEFIWriteBack {
-	return &uefistore.UEFIWriteBack{
-		Node:       "node-2",
-		Key:        uefistore.KeyPrefix(instanceID) + "b",
-		SeededFrom: uefistore.KeyPrefix(instanceID) + "a",
-		Since:      time.Date(2026, 11, 3, 1, 2, 3, 0, time.UTC),
+func pointer(name string) *uefistore.UEFIVarStoreRef {
+	return &uefistore.UEFIVarStoreRef{
+		Key:        keyOf(name),
+		Firmware:   uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: 540672, CodeSHA256: "c0de"},
+		WriterNode: "node-2",
+		WrittenAt:  time.Date(2026, 11, 3, 1, 2, 3, 0, time.UTC),
 	}
 }
 
+func keyOf(name string) string { return uefistore.KeyPrefix(instanceID) + name }
+
+// pointedRecord is a live record stamped at generation 7 that points at "b".
+func pointedRecord() *uefistore.UEFIStoreRecord {
+	rec := liveRecord()
+	rec.Generation = 7
+	rec.Pointer = pointer("b")
+	return rec
+}
+
 func rawPut(t *testing.T, js jetstream.JetStream, key, body string) {
 	t.Helper()
 	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
@@ -66,8 +76,7 @@ func TestOpen_CreatesBucketWithNoTTLAndStampsVersion(t *testing.T) {
 
 func TestCreate_RoundTripsAndStampsSchemaVersion(t *testing.T) {
 	s, _ := openStore(t)
-	rec := liveRecord()
-	rec.WriteBack = writeBack()
+	rec := pointedRecord()
 
 	rev, err := s.Create(t.Context(), rec)
 	require.NoError(t, err)
@@ -82,9 +91,7 @@ func TestCreate_RoundTripsAndStampsSchemaVersion(t *testing.T) {
 // The wire names are what every release reads, so they are pinned.
 func TestRecord_WireShape(t *testing.T) {
 	s, js := openStore(t)
-	rec := liveRecord()
-	rec.WriteBack = writeBack()
-	_, err := s.Create(t.Context(), rec)
+	_, err := s.Create(t.Context(), pointedRecord())
 	require.NoError(t, err)
 
 	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
@@ -96,11 +103,12 @@ func TestRecord_WireShape(t *testing.T) {
 		"instance_id": "i-0123456789abcdef0",
 		"account_id": "111122223333",
 		"state": "live",
-		"write_back": {
-			"node": "node-2",
+		"generation": 7,
+		"pointer": {
 			"key": "uefi/i-0123456789abcdef0/b",
-			"seeded_from": "uefi/i-0123456789abcdef0/a",
-			"since": "2026-11-03T01:02:03Z"
+			"firmware": {"arch": "x86_64", "vars_size": 540672, "code_sha256": "c0de"},
+			"writer_node": "node-2",
+			"written_at": "2026-11-03T01:02:03Z"
 		}
 	}`, string(entry.Value()))
 }
@@ -144,8 +152,7 @@ func TestCompareAndSet_StaleRevisionConflictsAndWritesNothing(t *testing.T) {
 	first, err := s.Create(t.Context(), liveRecord())
 	require.NoError(t, err)
 
-	moved := liveRecord()
-	moved.WriteBack = writeBack()
+	moved := pointedRecord()
 	second, err := s.CompareAndSet(t.Context(), moved, first)
 	require.NoError(t, err)
 	assert.Greater(t, second, first)
@@ -159,7 +166,7 @@ func TestCompareAndSet_StaleRevisionConflictsAndWritesNothing(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, second, rev)
 	assert.Equal(t, uefistore.StateLive, got.State)
-	assert.Equal(t, writeBack(), got.WriteBack)
+	assert.Equal(t, pointer("b"), got.Pointer)
 }
 
 // Two actors that read the same revision race to change it; exactly one may
@@ -179,12 +186,11 @@ func TestCompareAndSet_ConcurrentWritersFromOneReadHaveOneWinner(t *testing.T) {
 	)
 	for i := range writers {
 		wg.Go(func() {
-			rec := liveRecord()
-			rec.WriteBack = writeBack()
-			rec.WriteBack.Node = fmt.Sprintf("node-%d", i)
+			rec := pointedRecord()
+			rec.Pointer.WriterNode = fmt.Sprintf("node-%d", i)
 			if _, err := s.CompareAndSet(context.Background(), rec, rev); err == nil {
 				mu.Lock()
-				wins = append(wins, rec.WriteBack.Node)
+				wins = append(wins, rec.Pointer.WriterNode)
 				mu.Unlock()
 			} else {
 				assert.ErrorIs(t, err, kvstore.ErrConflict)
@@ -196,7 +202,7 @@ func TestCompareAndSet_ConcurrentWritersFromOneReadHaveOneWinner(t *testing.T) {
 	require.Len(t, wins, 1)
 	got, _, err := s.Get(t.Context(), instanceID)
 	require.NoError(t, err)
-	assert.Equal(t, wins[0], got.WriteBack.Node)
+	assert.Equal(t, wins[0], got.Pointer.WriterNode)
 }
 
 func TestCompareAndDelete_StaleRevisionKeepsRecord(t *testing.T) {
@@ -217,29 +223,20 @@ func TestCompareAndDelete_StaleRevisionKeepsRecord(t *testing.T) {
 	require.ErrorIs(t, err, kvstore.ErrNotFound)
 }
 
-// A superseded obligation deletes its recorded key, so a key outside the
+// A deletion acts on the key the record points at, so a key outside the
 // instance's own prefix must never reach the bucket.
 func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
+	other := uefistore.KeyPrefix("i-0fedcba9876543210") + "b"
 	cases := map[string]func(*uefistore.UEFIStoreRecord){
-		"unknown state":            func(r *uefistore.UEFIStoreRecord) { r.State = "gone" },
-		"not an instance ID":       func(r *uefistore.UEFIStoreRecord) { r.InstanceID = "ami-0123456789abcdef0" },
-		"write-back with no node":  func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Node = "" },
-		"write-back with no since": func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Since = time.Time{} },
-		"write-back key under another instance": func(r *uefistore.UEFIStoreRecord) {
-			r.WriteBack.Key = uefistore.KeyPrefix("i-0fedcba9876543210") + "b"
-		},
-		"write-back key under an image": func(r *uefistore.UEFIStoreRecord) {
-			r.WriteBack.Key = uefistore.KeyPrefix("ami-0123456789abcdef0") + "b"
-		},
-		"write-back key is the prefix itself": func(r *uefistore.UEFIStoreRecord) {
-			r.WriteBack.Key = uefistore.KeyPrefix(instanceID)
-		},
-		"write-back key nested below the prefix": func(r *uefistore.UEFIStoreRecord) {
-			r.WriteBack.Key = uefistore.KeyPrefix(instanceID) + "x/y"
-		},
-		"write-back seeded from another instance": func(r *uefistore.UEFIStoreRecord) {
-			r.WriteBack.SeededFrom = uefistore.KeyPrefix("i-0fedcba9876543210") + "a"
-		},
+		"unknown state":                   func(r *uefistore.UEFIStoreRecord) { r.State = "gone" },
+		"not an instance ID":              func(r *uefistore.UEFIStoreRecord) { r.InstanceID = "ami-0123456789abcdef0" },
+		"pointer under another instance":  func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = other },
+		"pointer under an image":          func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = uefistore.KeyPrefix("ami-0123456789abcdef0") + "b" },
+		"pointer is the prefix itself":    func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = uefistore.KeyPrefix(instanceID) },
+		"pointer nested below the prefix": func(r *uefistore.UEFIStoreRecord) { r.Pointer.Key = keyOf("x/y") },
+		"pointer with no firmware arch":   func(r *uefistore.UEFIStoreRecord) { r.Pointer.Firmware.Arch = "" },
+		"pointer with no VARS size":       func(r *uefistore.UEFIStoreRecord) { r.Pointer.Firmware.VarsSize = 0 },
+		"pointer with no writer node":     func(r *uefistore.UEFIStoreRecord) { r.Pointer.WriterNode = "" },
 	}
 	for name, mutate := range cases {
 		t.Run(name, func(t *testing.T) {
@@ -247,8 +244,7 @@ func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
 			rev, err := s.Create(t.Context(), liveRecord())
 			require.NoError(t, err)
 
-			rec := liveRecord()
-			rec.WriteBack = writeBack()
+			rec := pointedRecord()
 			mutate(rec)
 			_, err = s.CompareAndSet(t.Context(), rec, rev)
 			require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
@@ -256,7 +252,7 @@ func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
 			got, gotRev, err := s.Get(t.Context(), instanceID)
 			require.NoError(t, err)
 			assert.Equal(t, rev, gotRev, "an invalid record must not be written")
-			assert.Nil(t, got.WriteBack)
+			assert.Nil(t, got.Pointer)
 		})
 	}
 }
@@ -265,11 +261,11 @@ func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
 // is reported rather than decoded into something a reaper would act on.
 func TestGet_RefusesRecordsItCannotTrust(t *testing.T) {
 	cases := map[string]string{
-		"newer schema":       `{"schema_version":2,"instance_id":"i-0123456789abcdef0","state":"live"}`,
-		"no schema":          `{"instance_id":"i-0123456789abcdef0","state":"live"}`,
-		"another instance":   `{"schema_version":1,"instance_id":"i-0fedcba9876543210","state":"deleting"}`,
-		"unknown state":      `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"purged"}`,
-		"foreign write-back": `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"live","write_back":{"node":"n","key":"uefi/i-0fedcba9876543210/b","seeded_from":"uefi/i-0123456789abcdef0/a","since":"2026-11-03T00:00:00Z"}}`,
+		"newer schema":     `{"schema_version":2,"instance_id":"i-0123456789abcdef0","state":"live"}`,
+		"no schema":        `{"instance_id":"i-0123456789abcdef0","state":"live"}`,
+		"another instance": `{"schema_version":1,"instance_id":"i-0fedcba9876543210","state":"deleting"}`,
+		"unknown state":    `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"purged"}`,
+		"foreign pointer":  `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"live","generation":3,"pointer":{"key":"uefi/i-0fedcba9876543210/b","firmware":{"arch":"x86_64","vars_size":540672},"writer_node":"n"}}`,
 	}
 	for name, body := range cases {
 		t.Run(name, func(t *testing.T) {

spinifex/domains/ec2/instance/volume_held_elsewhere_test.go

@@ -102,3 +102,38 @@ func TestStartPaths_ReportAVolumeHeldElsewhereByCode(t *testing.T) {
 		assert.Equal(t, awserrors.ErrorIncorrectState, awserrors.ValidErrorCodeFromError(err))
 	})
 }
+
+// uefiClaimHeldMounter fails the launch the way a start does while a live
+// node holds the instance's UEFI variable store.
+type uefiClaimHeldMounter struct{ raceVolumeMounter }
+
+func (uefiClaimHeldMounter) Mount(context.Context, *vm.VM) error {
+	return fmt.Errorf("%w: node-a holds i-uefi: %w", vm.ErrUEFIClaimHeld, vm.ErrMountRetryable)
+}
+
+// A held UEFI store is a capacity refusal on both start paths, so the caller
+// tries another node rather than reporting a fault.
+func TestStartPaths_ReportAHeldUEFIStoreAsCapacity(t *testing.T) {
+	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
+	types, _ := defaultPrepareInstanceTypes()
+
+	t.Run("claimed relaunch", func(t *testing.T) {
+		mgr := vm.NewManager()
+		mgr.SetDeps(vm.Deps{NodeID: "node-b", VolumeMounter: uefiClaimHeldMounter{}})
+		svc := &InstanceServiceImpl{resourceMgr: &fakeResourceCapacityProvider{instanceTypes: types}, vmMgr: mgr}
+
+		err := svc.launchClaimedInstance(t.Context(), &vm.VM{ID: "i-uefi-claim", AccountID: "acc", InstanceType: "t3.micro", Status: vm.StatePending}, func() {})
+		assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, awserrors.ValidErrorCodeFromError(err))
+	})
+
+	t.Run("live owner", func(t *testing.T) {
+		const id = "i-uefi-live"
+		mgr := mgrWith(map[string]*vm.VM{id: {ID: id, Status: vm.StateStopped, InstanceType: "t3.micro"}})
+		mgr.SetDeps(vm.Deps{NodeID: "node-b", VolumeMounter: uefiClaimHeldMounter{}})
+		v, _ := mgr.Get(id)
+		svc := &InstanceServiceImpl{resourceMgr: &fakeResourceCapacityProvider{instanceTypes: types}, vmMgr: mgr}
+
+		err := svc.StartInstance(t.Context(), v, ec2v1.EC2InstanceCommand{ID: id})
+		assert.Equal(t, awserrors.ErrorInsufficientInstanceCapacity, awserrors.ValidErrorCodeFromError(err))
+	})
+}

spinifex/domains/ec2/key/service_impl.go

@@ -48,15 +48,8 @@ type KeyServiceImpl struct {
 	// Optional: injected after construction. Nil leaves the central tag store
 	// untouched by create, import and delete.
 	centralTags CentralTagStore
-
-	// rsaKeyBits sizes the RSA keys CreateKeyPair generates. Zero takes
-	// defaultRSAKeyBits, which is what production runs; only tests set it.
-	rsaKeyBits int
 }
 
-// defaultRSAKeyBits is the size of the RSA key pairs AWS CreateKeyPair returns.
-const defaultRSAKeyBits = 2048
-
 // CentralTagStore keeps the central tag index in step with key-pair records, so
 // DescribeTags sees creation tags. Implemented by domains/ec2/tags.TagsServiceImpl.
 type CentralTagStore interface {
@@ -176,7 +169,7 @@ func (s *KeyServiceImpl) CreateKeyPair(ctx context.Context, input *ec2.CreateKey
 	}
 
 	// Generate the key in process; the private half never reaches local disk.
-	privateKeyData, publicKey, err := generateKeyPair(keyType, s.rsaBits())
+	privateKeyData, publicKey, err := generateKeyPair(keyType)
 	if err != nil {
 		slog.ErrorContext(ctx, "Failed to generate key pair", "keyName", keyName, "keyType", keyType, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
@@ -262,18 +255,10 @@ func (s *KeyServiceImpl) CreateKeyPair(ctx context.Context, input *ec2.CreateKey
 	return output, nil
 }
 
-// rsaBits returns the RSA key size CreateKeyPair generates.
-func (s *KeyServiceImpl) rsaBits() int {
-	if s.rsaKeyBits > 0 {
-		return s.rsaKeyBits
-	}
-	return defaultRSAKeyBits
-}
-
 // generateKeyPair generates a key pair of the given EC2 key type, returning the
 // private key in the PEM container AWS hands back for that type alongside the
-// public half. rsaBits sizes an RSA key and is ignored for ED25519.
-func generateKeyPair(keyType string, rsaBits int) ([]byte, ssh.PublicKey, error) {
+// public half.
+func generateKeyPair(keyType string) ([]byte, ssh.PublicKey, error) {
 	var block *pem.Block
 	var cryptoPub crypto.PublicKey
 
@@ -290,10 +275,10 @@ func generateKeyPair(keyType string, rsaBits int) ([]byte, ssh.PublicKey, error)
 		cryptoPub = pub
 
 	case "rsa":
-		// RSA in PKCS#1 PEM, as AWS returns it. OpenSSH reads that format for SSH
-		// either way, but GetPasswordData's --priv-launch-key cannot read the
-		// OpenSSH container at all.
-		priv, err := rsa.GenerateKey(rand.Reader, rsaBits)
+		// RSA 2048-bit in PKCS#1 PEM, as AWS returns it. OpenSSH reads that format
+		// for SSH either way, but GetPasswordData's --priv-launch-key cannot read
+		// the OpenSSH container at all.
+		priv, err := rsa.GenerateKey(rand.Reader, 2048)
 		if err != nil {
 			return nil, nil, fmt.Errorf("generate rsa key: %w", err)
 		}

spinifex/domains/ec2/key/service_impl_test.go

@@ -2,7 +2,6 @@ package key
 
 import (
 	"context"
-	"crypto/rsa"
 	"crypto/sha1"
 	"crypto/sha256"
 	"crypto/x509"
@@ -101,12 +100,9 @@ const (
 	testRSAPrivKeyImportedFingerprint = "66:81:50:28:0a:74:9e:48:9f:d8:79:da:27:86:22:79"
 )
 
-// newTestKeyService generates RSA keys at 1024 bits, the smallest crypto/rsa
-// accepts, since only TestCreateKeyPair_RSA checks the production size.
 func newTestKeyService() (*KeyServiceImpl, *objectstore.MemoryObjectStore) {
 	store := objectstore.NewMemoryObjectStore()
 	svc := NewKeyServiceImplWithStore(store, testBucket)
-	svc.rsaKeyBits = 1024
 	return svc, store
 }
 
@@ -187,7 +183,7 @@ func TestCreateKeyPair_ED25519(t *testing.T) {
 }
 
 func TestCreateKeyPair_RSA(t *testing.T) {
-	svc := NewKeyServiceImplWithStore(objectstore.NewMemoryObjectStore(), testBucket)
+	svc, _ := newTestKeyService()
 
 	out, err := svc.CreateKeyPair(context.Background(), &ec2.CreateKeyPairInput{
 		KeyName: aws.String("my-rsa-key"),
@@ -212,11 +208,6 @@ func TestCreateKeyPair_RSA(t *testing.T) {
 	require.NoError(t, err)
 	sum := sha1.Sum(der)
 
-	// AWS generates 2048-bit RSA key pairs.
-	rsaKey, ok := rawKey.(*rsa.PrivateKey)
-	require.True(t, ok, "RSA key material parsed as %T", rawKey)
-	assert.Equal(t, 2048, rsaKey.N.BitLen())
-
 	assert.Equal(t, colonHex(sum[:]), *out.KeyFingerprint)
 	// 20 hex pairs -- a SHA-1, not the 16-pair MD5 this used to return.
 	assert.Regexp(t, `^([0-9a-f]{2}:){19}[0-9a-f]{2}$`, *out.KeyFingerprint)

spinifex/domains/ec2/natgw/lifecycle_invariants_test.go

@@ -17,18 +17,18 @@ import (
 
 // TestRLC1_NatGatewayDeleteNotFoundOnAbsent enforces the Common Resource
 // Lifecycle Contract rule #1 (AWS-faithful delete, per-service): the EC2
-// DeleteNatGateway API returns NatGatewayNotFound for an absent NAT
+// DeleteNatGateway API returns InvalidNatGatewayID.NotFound for an absent NAT
 // gateway, not success. Idempotent convergence belongs to destroy orchestration, which
 // tolerates NotFound via awserrors.IsNotFound; the public API stays AWS compatible.
 func TestRLC1_NatGatewayDeleteNotFoundOnAbsent(t *testing.T) {
 	svc := setupTestService(t)
 
 	_, err := svc.DeleteNatGateway(t.Context(), &ec2.DeleteNatGatewayInput{
-		NatGatewayId: aws.String("nat-0a1b2c3d4e5f60718"),
+		NatGatewayId: aws.String("nat-absent00000000"),
 	}, testAccountID)
 
-	require.Errorf(t, err, "DeleteNatGateway on an absent NAT gateway must return %s, not success (RLC rule #1 AWS-faithful delete): destroy orchestration tolerates NotFound, the API must not", awserrors.ErrorNatGatewayNotFound)
-	assert.ErrorContains(t, err, awserrors.ErrorNatGatewayNotFound, "DeleteNatGateway on an absent NAT gateway must return the canonical NotFound (RLC rule #1)")
+	require.Errorf(t, err, "DeleteNatGateway on an absent NAT gateway must return %s, not success (RLC rule #1 AWS-faithful delete): destroy orchestration tolerates NotFound, the API must not", awserrors.ErrorInvalidNatGatewayIDNotFound)
+	assert.ErrorContains(t, err, awserrors.ErrorInvalidNatGatewayIDNotFound, "DeleteNatGateway on an absent NAT gateway must return the canonical NotFound (RLC rule #1)")
 }
 
 // TestRLC3_NatGatewayDeletesWhileRouted enforces Q-98: a NAT gateway is

spinifex/domains/ec2/natgw/service_impl.go

@@ -190,9 +190,9 @@ func (s *NatGatewayServiceImpl) DeleteNatGateway(ctx context.Context, input *ec2
 	entry, err := s.natgwKV.Get(ctx, key)
 	if err != nil {
 		if errors.Is(err, jetstream.ErrKeyNotFound) {
-			// AWS-faithful: an absent NAT gateway is NatGatewayNotFound (provider
+			// AWS-faithful: an absent NAT gateway is NotFound (provider
 			// tolerates it on destroy); destroy orchestration tolerates it too.
-			return nil, errors.New(awserrors.ErrorNatGatewayNotFound)
+			return nil, errors.New(awserrors.ErrorInvalidNatGatewayIDNotFound)
 		}
 		slog.ErrorContext(ctx, "Failed to read NAT Gateway", "natGatewayId", natgwID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
@@ -419,7 +419,7 @@ func (s *NatGatewayServiceImpl) DescribeNatGateways(ctx context.Context, input *
 		entry, err := s.deletedNatgwKV.Get(ctx, key)
 		if err != nil {
 			if errors.Is(err, jetstream.ErrKeyNotFound) {
-				return nil, errors.New(awserrors.ErrorNatGatewayNotFound)
+				return nil, errors.New(awserrors.ErrorInvalidNatGatewayIDNotFound)
 			}
 			return nil, errors.New(awserrors.ErrorServerInternal)
 		}

spinifex/domains/ec2/natgw/service_impl_test.go

@@ -205,27 +205,9 @@ func TestDescribeNatGateways_ByID(t *testing.T) {
 
 	// Non-existent ID
 	_, err = svc.DescribeNatGateways(t.Context(), &ec2.DescribeNatGatewaysInput{
-		NatGatewayIds: []*string{aws.String("nat-0a1b2c3d4e5f60718")},
+		NatGatewayIds: []*string{aws.String("nat-nope")},
 	}, testAccountID)
-	assert.EqualError(t, err, awserrors.ErrorNatGatewayNotFound)
-}
-
-// AWS fails the whole call when any named ID is absent, rather than returning
-// the IDs it does hold.
-func TestDescribeNatGateways_UnknownIDAmongKnownFailsWholeCall(t *testing.T) {
-	svc := setupTestService(t)
-	createOut, err := svc.CreateNatGateway(t.Context(), &ec2.CreateNatGatewayInput{
-		SubnetId:     aws.String("subnet-pub1"),
-		AllocationId: aws.String("eipalloc-test1"),
-	}, testAccountID)
-	require.NoError(t, err)
-
-	out, err := svc.DescribeNatGateways(t.Context(), &ec2.DescribeNatGatewaysInput{
-		NatGatewayIds: []*string{createOut.NatGateway.NatGatewayId, aws.String("nat-0a1b2c3d4e5f60718")},
-	}, testAccountID)
-
-	assert.EqualError(t, err, awserrors.ErrorNatGatewayNotFound)
-	assert.Nil(t, out, "no partial list when one ID is absent")
+	assert.EqualError(t, err, awserrors.ErrorInvalidNatGatewayIDNotFound)
 }
 
 func TestDescribeNatGateways_FilterByState(t *testing.T) {
@@ -295,9 +277,9 @@ func TestDescribeNatGateways_DeletedGatewayVisible(t *testing.T) {
 
 	// Non-existent ID should still error
 	_, err = svc.DescribeNatGateways(t.Context(), &ec2.DescribeNatGatewaysInput{
-		NatGatewayIds: []*string{aws.String("nat-0a1b2c3d4e5f60718")},
+		NatGatewayIds: []*string{aws.String("nat-never-existed")},
 	}, testAccountID)
-	assert.EqualError(t, err, awserrors.ErrorNatGatewayNotFound)
+	assert.EqualError(t, err, awserrors.ErrorInvalidNatGatewayIDNotFound)
 }
 
 func TestDescribeNatGateways_FilterByNatGatewayId(t *testing.T) {

spinifex/domains/ec2/routetable/service_impl_test.go

@@ -388,22 +388,6 @@ func TestCreateRoute(t *testing.T) {
 	assert.Equal(t, "igw-test1", record.Routes[1].GatewayId)
 }
 
-// AWS answers a route to an unknown NAT gateway with InvalidNatGatewayID.NotFound,
-// unlike the NAT gateway describe and delete calls, which use NatGatewayNotFound.
-func TestCreateRoute_UnknownNatGatewayKeepsInvalidIDCode(t *testing.T) {
-	t.Parallel()
-	svc := setupTestService(t)
-	rtbID := createTestRtb(t, svc)
-
-	_, err := svc.CreateRoute(t.Context(), &ec2.CreateRouteInput{
-		RouteTableId:         aws.String(rtbID),
-		DestinationCidrBlock: aws.String("0.0.0.0/0"),
-		NatGatewayId:         aws.String("nat-0a1b2c3d4e5f60718"),
-	}, testAccountID)
-
-	assert.EqualError(t, err, awserrors.ErrorInvalidNatGatewayIDNotFound)
-}
-
 func TestCreateRoute_DuplicateDestination(t *testing.T) {
 	t.Parallel()
 	svc := setupTestService(t)

spinifex/domains/ec2/vpc/service_impl.go

@@ -7,9 +7,7 @@ import (
 	"fmt"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/tagmirror"
 	"log/slog"
-	"maps"
 	"net"
-	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -25,7 +23,6 @@ import (
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/paging"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/zoneid"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
@@ -46,6 +43,11 @@ const (
 	KVBucketVPCsVersion       = 1
 	KVBucketSubnetsVersion    = 1
 	KVBucketVNICounterVersion = 1
+
+	// SingleZoneID is the only availability-zone id the gateway exposes. The
+	// deployment is single-AZ, so every subnet and every DescribeAvailabilityZones
+	// result reports it (AWS clients resolve AZ topology by zone-id, not name).
+	SingleZoneID = "spinifexz1"
 )
 
 // VPCRecord represents a stored VPC. AZ is stamped at create time; empty AZ
@@ -83,7 +85,6 @@ type VPCServiceImpl struct {
 	natsConn   *nats.Conn
 	vpcKV      jetstream.KeyValue
 	subnetKV   jetstream.KeyValue
-	zoneIDs    ZoneIDSource
 	vniKV      jetstream.KeyValue
 	eniKV      jetstream.KeyValue
 	sgKV       jetstream.KeyValue
@@ -132,14 +133,6 @@ func (s *VPCServiceImpl) SetPrivateDNSNamer(namer func(privateIP string) string)
 	s.privateDNSName = namer
 }
 
-// ZoneIDSource returns the stored availability-zone ID for a zone name.
-type ZoneIDSource interface {
-	ZoneID(ctx context.Context, az string) (string, error)
-}
-
-// SetZoneIDSource sets where subnets read their availability-zone ID. Without one the ID is omitted.
-func (s *VPCServiceImpl) SetZoneIDSource(src ZoneIDSource) { s.zoneIDs = src }
-
 // SetCentralTagStore injects the central tag store so create paths project
 // their record tags into it and delete paths clear them. Nil leaves both
 // unchanged.
@@ -791,16 +784,6 @@ func (s *VPCServiceImpl) CreateSubnet(ctx context.Context, input *ec2.CreateSubn
 		az = s.config.AZ
 	}
 
-	// Pin the zone's ID before anything is written, so a refused zone leaves no subnet behind.
-	zoneID, err := s.zoneIDFor(ctx, az)
-	if err != nil {
-		if errors.Is(err, zoneid.ErrTooManyZones) {
-			return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "The Region already has the maximum of %d distinct availability zones", zoneid.MaxZones)
-		}
-		slog.ErrorContext(ctx, "CreateSubnet: zone id pin failed", "az", az, "err", err)
-		return nil, errors.New(awserrors.ErrorServerInternal)
-	}
-
 	subnetID := awsidentifiers.GenerateResourceID("subnet")
 
 	// Calculate available IPs (total hosts minus AWS reserved: network, router, DNS, future, broadcast)
@@ -833,7 +816,9 @@ func (s *VPCServiceImpl) CreateSubnet(ctx context.Context, input *ec2.CreateSubn
 	// Publish vpc.create-subnet event for vpcd topology translation
 	s.publishSubnetEvent(networkv1.SubnetCreateSubject, record.SubnetId, record.VpcId, record.CidrBlock)
 
-	return &ec2.CreateSubnetOutput{Subnet: s.subnetRecordToEC2(&record, totalHosts, accountID, zoneID)}, nil
+	return &ec2.CreateSubnetOutput{
+		Subnet: s.subnetRecordToEC2(&record, totalHosts, accountID),
+	}, nil
 }
 
 // DeleteSubnet deletes a subnet.
@@ -1058,12 +1043,7 @@ func (s *VPCServiceImpl) DescribeSubnets(ctx context.Context, input *ec2.Describ
 			continue
 		}
 
-		zoneID, err := s.zoneIDFor(ctx, record.AvailabilityZone)
-		if err != nil {
-			slog.ErrorContext(ctx, "subnet zone id lookup failed", "subnetId", record.SubnetId, "az", record.AvailabilityZone, "err", err)
-			return nil, errors.New(awserrors.ErrorServerInternal)
-		}
-		if len(parsedFilters) > 0 && !subnetMatchesFilters(&record, parsedFilters, aws.StringValue(zoneID)) {
+		if len(parsedFilters) > 0 && !subnetMatchesFilters(&record, parsedFilters) {
 			continue
 		}
 
@@ -1075,7 +1055,7 @@ func (s *VPCServiceImpl) DescribeSubnets(ctx context.Context, input *ec2.Describ
 			availableIPs = max((1<<(32-ones))-5, 0) //#nosec G115 - ones from validated CIDR
 		}
 
-		subnets = append(subnets, s.subnetRecordToEC2(&record, availableIPs, accountID, zoneID))
+		subnets = append(subnets, s.subnetRecordToEC2(&record, availableIPs, accountID))
 	}
 
 	// If specific subnet IDs were requested but not found, return error
@@ -1265,19 +1245,18 @@ var describeSubnetsPaging = paging.EC2{
 
 // describeSubnetsValidFilters defines the set of filter names accepted by DescribeSubnets.
 var describeSubnetsValidFilters = map[string]bool{
-	"subnet-id":            true,
-	"vpc-id":               true,
-	"availability-zone":    true,
-	"availability-zone-id": true,
-	"cidr-block":           true,
-	"state":                true,
-	"default-for-az":       true,
-	"tag-key":              true,
-	"tag-value":            true,
+	"subnet-id":         true,
+	"vpc-id":            true,
+	"availability-zone": true,
+	"cidr-block":        true,
+	"state":             true,
+	"default-for-az":    true,
+	"tag-key":           true,
+	"tag-value":         true,
 }
 
 // subnetMatchesFilters checks whether a SubnetRecord satisfies all parsed awsfilters.
-func subnetMatchesFilters(record *SubnetRecord, filters map[string][]string, zoneID string) bool {
+func subnetMatchesFilters(record *SubnetRecord, filters map[string][]string) bool {
 	for name, values := range filters {
 		if awsfilters.IsTagFilter(name) {
 			continue
@@ -1291,8 +1270,6 @@ func subnetMatchesFilters(record *SubnetRecord, filters map[string][]string, zon
 			field = record.VpcId
 		case "availability-zone":
 			field = record.AvailabilityZone
-		case "availability-zone-id":
-			field = zoneID
 		case "cidr-block":
 			field = record.CidrBlock
 		case "state":
@@ -1336,25 +1313,13 @@ func (s *VPCServiceImpl) vpcRecordToEC2(record *VPCRecord, accountID string) *ec
 	return vpc
 }
 
-// zoneIDFor returns the stored zone ID for az, pinning it if needed. It is nil when there is no source or no zone name.
-func (s *VPCServiceImpl) zoneIDFor(ctx context.Context, az string) (*string, error) {
-	if s.zoneIDs == nil || az == "" {
-		return nil, nil
-	}
-	id, err := s.zoneIDs.ZoneID(ctx, az)
-	if err != nil {
-		return nil, err
-	}
-	return aws.String(id), nil
-}
-
-func (s *VPCServiceImpl) subnetRecordToEC2(record *SubnetRecord, availableIPs int, accountID string, zoneID *string) *ec2.Subnet {
+func (s *VPCServiceImpl) subnetRecordToEC2(record *SubnetRecord, availableIPs int, accountID string) *ec2.Subnet {
 	subnet := &ec2.Subnet{
 		SubnetId:                aws.String(record.SubnetId),
 		VpcId:                   aws.String(record.VpcId),
 		CidrBlock:               aws.String(record.CidrBlock),
 		AvailabilityZone:        aws.String(record.AvailabilityZone),
-		AvailabilityZoneId:      zoneID,
+		AvailabilityZoneId:      aws.String(SingleZoneID),
 		State:                   aws.String(record.State),
 		DefaultForAz:            aws.Bool(record.IsDefault),
 		AvailableIpAddressCount: aws.Int64(int64(availableIPs)),
@@ -1714,35 +1679,3 @@ func (s *VPCServiceImpl) publishSubnetEvent(topic, subnetId, vpcId, cidrBlock st
 	}
 	client.CreateSubnet(evt)
 }
-
-// SubnetAvailabilityZones returns the distinct zone names held by subnet records of every account, sorted.
-func (s *VPCServiceImpl) SubnetAvailabilityZones(ctx context.Context) ([]string, error) {
-	keys, err := s.subnetKV.Keys(ctx)
-	if err != nil {
-		if errors.Is(err, jetstream.ErrNoKeysFound) {
-			return nil, nil
-		}
-		return nil, fmt.Errorf("list subnets: %w", err)
-	}
-	seen := map[string]struct{}{}
-	for _, k := range keys {
-		if k == kvstore.VersionKey {
-			continue
-		}
-		entry, err := s.subnetKV.Get(ctx, k)
-		if errors.Is(err, jetstream.ErrKeyNotFound) {
-			continue
-		}
-		if err != nil {
-			return nil, fmt.Errorf("read subnet %s: %w", k, err)
-		}
-		var rec SubnetRecord
-		if err := json.Unmarshal(entry.Value(), &rec); err != nil {
-			return nil, fmt.Errorf("decode subnet %s: %w", k, err)
-		}
-		if rec.AvailabilityZone != "" {
-			seen[rec.AvailabilityZone] = struct{}{}
-		}
-	}
-	return slices.Sorted(maps.Keys(seen)), nil
-}

spinifex/domains/ec2/vpc/service_impl_test.go

@@ -3,22 +3,15 @@ package vpc
 import (
 	"context"
 	"encoding/json"
-	"errors"
-	"fmt"
-	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	networkv1 "github.com/mulgadc/spinifex/contracts/network/v1"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/zoneid"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go"
-	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -2561,135 +2554,3 @@ func TestVpc_ModifyVpcAttribute_MultipleAttributes(t *testing.T) {
 	assert.Contains(t, msg, "enableDnsSupport")
 	assert.Contains(t, msg, "enableDnsHostnames")
 }
-
-func TestSubnetAvailabilityZones_DistinctSortedAcrossAccounts(t *testing.T) {
-	svc := setupTestVPCService(t)
-	put := func(key, az string) {
-		data, err := json.Marshal(SubnetRecord{SubnetId: key, AvailabilityZone: az})
-		require.NoError(t, err)
-		_, err = svc.subnetKV.Put(t.Context(), key, data)
-		require.NoError(t, err)
-	}
-
-	empty, err := svc.SubnetAvailabilityZones(t.Context())
-	require.NoError(t, err)
-	assert.Empty(t, empty)
-
-	put("111111111111.subnet-1", "rack-b")
-	put("222222222222.subnet-2", "rack-a")
-	put("222222222222.subnet-3", "rack-b")
-	put("222222222222.subnet-4", "")
-
-	got, err := svc.SubnetAvailabilityZones(t.Context())
-	require.NoError(t, err)
-	assert.Equal(t, []string{"rack-a", "rack-b"}, got)
-}
-
-func TestDescribeSubnets_AvailabilityZoneIDFilter(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
-	testutil.StubVpcdSGResponder(t, nc)
-	svc, err := NewVPCServiceImplWithNATS(t.Context(), nil, nc)
-	require.NoError(t, err)
-	ids, err := zoneid.NewResolver(js, "mulga-lab", "")
-	require.NoError(t, err)
-	svc.SetZoneIDSource(ids)
-
-	vpcID := createTestVPC(t, svc, "10.0.0.0/16")
-	mk := func(cidr, az string) string {
-		out, err := svc.CreateSubnet(t.Context(), &ec2.CreateSubnetInput{
-			VpcId: aws.String(vpcID), CidrBlock: aws.String(cidr), AvailabilityZone: aws.String(az),
-		}, testAccountID)
-		require.NoError(t, err)
-		return aws.StringValue(out.Subnet.SubnetId)
-	}
-	inA := mk("10.0.1.0/24", "mulga-laba")
-	inB := mk("10.0.2.0/24", "mulga-labb")
-	idA, err := ids.ZoneID(t.Context(), "mulga-laba")
-	require.NoError(t, err)
-	idB, err := ids.ZoneID(t.Context(), "mulga-labb")
-	require.NoError(t, err)
-	require.NotEqual(t, idA, idB)
-
-	by := func(id string) []string {
-		out, err := svc.DescribeSubnets(t.Context(), &ec2.DescribeSubnetsInput{
-			Filters: []*ec2.Filter{{Name: aws.String("availability-zone-id"), Values: aws.StringSlice([]string{id})}},
-		}, testAccountID)
-		require.NoError(t, err)
-		var got []string
-		for _, s := range out.Subnets {
-			got = append(got, aws.StringValue(s.SubnetId))
-		}
-		return got
-	}
-	assert.Equal(t, []string{inA}, by(idA))
-	assert.Equal(t, []string{inB}, by(idB))
-	assert.Empty(t, by("zz9-az9"), "an unknown zone id matches nothing and is not an error")
-}
-
-func TestCreateSubnet_RefusesA65thDistinctZone(t *testing.T) {
-	_, nc, js := testutil.StartTestJetStream(t)
-	testutil.StubVpcdSGResponder(t, nc)
-	svc, err := NewVPCServiceImplWithNATS(t.Context(), nil, nc)
-	require.NoError(t, err)
-	ids, err := zoneid.NewResolver(js, "mulga-lab", "")
-	require.NoError(t, err)
-	svc.SetZoneIDSource(ids)
-	vpcID := createTestVPC(t, svc, "10.0.0.0/16")
-
-	create := func(i int, az string) error {
-		_, err := svc.CreateSubnet(t.Context(), &ec2.CreateSubnetInput{
-			VpcId: aws.String(vpcID), CidrBlock: aws.String(fmt.Sprintf("10.0.%d.0/24", i)), AvailabilityZone: aws.String(az),
-		}, testAccountID)
-		return err
-	}
-	for i := 1; i <= zoneid.MaxZones; i++ {
-		require.NoError(t, create(i, fmt.Sprintf("rack-%02d", i)))
-	}
-	pinned, err := ids.Pin(t.Context(), "rack-01", "rack-64")
-	require.NoError(t, err)
-
-	err = create(65, "rack-65")
-	testutil.RequireAWSError(t, err, awserrors.ErrorInvalidParameterValue, "The Region already has the maximum of 64 distinct availability zones")
-
-	out, err := svc.DescribeSubnets(t.Context(), &ec2.DescribeSubnetsInput{}, testAccountID)
-	require.NoError(t, err)
-	assert.Len(t, out.Subnets, zoneid.MaxZones, "the refused subnet must not exist")
-	again, err := ids.Pin(t.Context(), "rack-01", "rack-64")
-	require.NoError(t, err)
-	assert.Equal(t, pinned, again)
-
-	require.NoError(t, create(66, "rack-01"), "an already pinned zone still takes new subnets")
-}
-
-type failingZoneIDs struct{}
-
-func (failingZoneIDs) ZoneID(context.Context, string) (string, error) {
-	return "", errors.New("zone store down")
-}
-
-func TestCreateSubnet_ZoneStoreFailureWritesNothingAndPublishesNothing(t *testing.T) {
-	svc, nc := setupTestVPCServiceWithNC(t)
-	svc.SetZoneIDSource(failingZoneIDs{})
-	vpcID := createTestVPC(t, svc, "10.0.0.0/16")
-
-	var events atomic.Int32
-	sub, err := nc.Subscribe(networkv1.SubnetCreateSubject, func(*nats.Msg) { events.Add(1) })
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = sub.Unsubscribe() })
-	require.NoError(t, nc.Flush())
-
-	_, err = svc.CreateSubnet(t.Context(), &ec2.CreateSubnetInput{
-		VpcId: aws.String(vpcID), CidrBlock: aws.String("10.0.1.0/24"), AvailabilityZone: aws.String("mulga-laba"),
-	}, testAccountID)
-	testutil.RequireAWSError(t, err, awserrors.ErrorServerInternal, "")
-
-	require.NoError(t, nc.Flush())
-	keys, err := svc.subnetKV.Keys(t.Context())
-	if err != nil {
-		require.ErrorIs(t, err, jetstream.ErrNoKeysFound)
-	}
-	for _, k := range keys {
-		assert.Equal(t, kvstore.VersionKey, k, "no subnet record may exist")
-	}
-	assert.Zero(t, events.Load(), "no topology event may be published")
-}

spinifex/domains/ecs/scheduler_test.go

@@ -16,7 +16,6 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/mulgadc/spinifex/internal/testkit"
-	"github.com/mulgadc/spinifex/spinifex/domains/ecs/taskdefinition"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 	"github.com/nats-io/nats.go"
@@ -137,59 +136,3 @@ func TestECSInstanceRolePolicyDoc_GrantsAssumeRole(t *testing.T) {
 	assert.Contains(t, doc, `"ecs:*"`)
 	assert.Contains(t, doc, `"ecr:GetAuthorizationToken"`)
 }
-
-// containerInstanceStatus is i-1's status in the web cluster.
-func containerInstanceStatus(t *testing.T, svc *Service) string {
-	t.Helper()
-	di, err := svc.DescribeContainerInstances(context.Background(), &ecs.DescribeContainerInstancesInput{
-		Cluster: aws.String("web"), ContainerInstances: []*string{aws.String("i-1")},
-	}, testAccountID)
-	require.NoError(t, err)
-	return aws.StringValue(di.ContainerInstances[0].Status)
-}
-
-// A leader that loses the lease must drop the bus and stop reaping, or two
-// nodes would mark instances DRAINING and stop their tasks at once. Re-elected,
-// it takes both back up.
-func TestScheduler_LosingTheLeaseDropsTheBusAndTheReaperUntilReElection(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
-	svc := NewService(nc, testRegion, "").WithDeps(Deps{IAM: newStubIAM()})
-	_, err := svc.CreateCluster(context.Background(), &ecs.CreateClusterInput{ClusterName: aws.String("web")}, testAccountID)
-	require.NoError(t, err)
-	registerInstance(t, svc, "web", "i-1", 1024, 2048)
-	kv, err := svc.bucket(t.Context(), testAccountID)
-	require.NoError(t, err)
-	var rec InstanceRecord
-	_, err = taskdefinition.GetJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec)
-	require.NoError(t, err)
-	rec.LastSeen = time.Now().UTC().Add(-2 * heartbeatTimeout)
-	require.NoError(t, taskdefinition.PutJSON(t.Context(), kv, InstanceKey("web", "i-1"), &rec))
-
-	store := kvstore.OpenSyncBucket(testutil.NewJetStream(t, nc))
-	sc := NewScheduler(nc, svc, kvsync.NewLease(store, KeyScheduler, "holder-a", 300*time.Millisecond))
-	ctx, cancel := context.WithCancel(t.Context())
-	done := make(chan struct{})
-	go func() { defer close(done); kvsync.RunElection(ctx, sc.lease, sc.onGained, sc.unsubscribeBus) }()
-	t.Cleanup(func() { cancel(); <-done })
-
-	held := func() bool { return sc.lease.Held() && sc.subCount() == 4 }
-	require.Eventually(t, held, 5*time.Second, 10*time.Millisecond, "the leader subscribes to the four bus subjects")
-
-	syncKV, err := store.KV(t.Context())
-	require.NoError(t, err)
-	_, err = syncKV.Put(t.Context(), KeyScheduler, []byte(`{"holder":"holder-b","generation":0}`))
-	require.NoError(t, err)
-	require.Eventually(t, func() bool { return !sc.lease.Held() && sc.subCount() == 0 }, 5*time.Second, 10*time.Millisecond,
-		"a lost lease must drop every bus subscription")
-
-	revisit, err := sc.timerPass(t.Context())
-	require.NoError(t, err)
-	assert.Equal(t, leaseRefresh, revisit)
-	assert.Equal(t, "ACTIVE", containerInstanceStatus(t, svc), "a node that lost the lease must not reap")
-
-	require.NoError(t, syncKV.Delete(t.Context(), KeyScheduler))
-	require.Eventually(t, held, 5*time.Second, 10*time.Millisecond, "re-elected, the bus comes back")
-	_, err = sc.timerPass(t.Context())
-	require.NoError(t, err)
-	assert.Equal(t, "DRAINING", containerInstanceStatus(t, svc), "the re-elected leader reaps again")
-}

spinifex/domains/eks/awsapi/gateway_wire_test.go

@@ -14,7 +14,6 @@ import (
 	ekscluster "github.com/mulgadc/spinifex/spinifex/domains/eks/cluster"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -56,7 +55,7 @@ func TestEKSRequest_InternalAddonsWireBody(t *testing.T) {
 		ControlPlaneNodes: []ekscluster.ControlPlaneNode{{InstanceID: cpInstanceID}},
 	}))
 
-	svc, err := ekscluster.NewEKSServiceImpl(ekscluster.EKSServiceDeps{NATSConn: nc, Region: authzRegion, Leases: ekscluster.NewClusterLeases(kvstore.OpenSyncBucket(js), "test")})
+	svc, err := ekscluster.NewEKSServiceImpl(ekscluster.EKSServiceDeps{NATSConn: nc, Region: authzRegion})
 	require.NoError(t, err)
 	sub, err := nc.Subscribe("eks.ListStagedAddonManifests", func(m *nats.Msg) {
 		acct := natsmsg.AccountIDFromMsg(m)
@@ -113,7 +112,7 @@ func TestEKSRequest_InternalWorkersWireBody(t *testing.T) {
 		require.NoError(t, ekscluster.PutNodegroupRecord(t.Context(), kv, rec))
 	}
 
-	svc, err := ekscluster.NewEKSServiceImpl(ekscluster.EKSServiceDeps{NATSConn: nc, Region: authzRegion, Leases: ekscluster.NewClusterLeases(kvstore.OpenSyncBucket(js), "test")})
+	svc, err := ekscluster.NewEKSServiceImpl(ekscluster.EKSServiceDeps{NATSConn: nc, Region: authzRegion})
 	require.NoError(t, err)
 	sub, err := nc.Subscribe("eks.ListManagedWorkers", func(m *nats.Msg) {
 		acct := natsmsg.AccountIDFromMsg(m)