fix(ci): select E2E shards with -test.skip so opt-in tests stay off

TestRebootLoop runs when named in -test.run, and the shard regex named
every test, which added ten reboots to one shard. Each leg now skips
the other shards' tests and leaves -test.run empty.

.github/e2e-cells.json

@@ -110,7 +110,8 @@
     "arch": "x86_64",
     "runner": "ci-single-nvme",
     "extra": "eks",
-    "suite": "eks"
+    "suite": "eks",
+    "timeout_minutes": 60
   },
   {
     "cell": 26,

.github/e2e-weights/single.txt

@@ -0,0 +1,36 @@
+# Median seconds per top-level test in the single suite, from E2E push runs
+# 42920949994, 42959551411 and 43080409096. e2e-shard.sh balances on these.
+TestVPCEgressPaths 301
+TestGuestChurnDurability 197
+TestInstanceStatus 140
+TestLaunchTemplateBoot 134
+TestIMDS 121
+TestEarlyRebootLiveness 104
+TestIMDSDatapathSurvivesVswitchdRestart 102
+TestAccountScoping 100
+TestVolumeLifecycle 98
+TestCreateImage 97
+TestSGPolicyDatapath 77
+TestSGReachabilityPolicy 71
+TestWANEgressSMTPBlock 70
+TestIAMInstanceProfileAssociation 51
+TestPartialBlockConcurrentWriteDurability 50
+TestSpotInstanceLifecycle 50
+TestAttachToStopped 44
+TestStopStart 44
+TestClusterStatsCLI 9
+TestInstanceClusterStats 5
+TestIAMAuthorization 1
+TestKVReplication 1
+TestSnapshotLifecycle 1
+TestCertIssuance 0
+TestConsoleOutput 0
+TestDiscovery 0
+TestEnvironment 0
+TestImageCatalogReachable 0
+TestInstanceMetadata 0
+TestNegativeErrorPaths 0
+TestRebootLoop 0
+TestSnapshotBackedLaunch 0
+TestTLSPosture 0
+TestVolumeStatus 0

.github/scripts/e2e-select.sh

@@ -16,6 +16,12 @@ FLOOR_MIN=60
 BASE_RUNNER=ci-single
 SERVICE_RUNNER=ci-pseudo
 
+# The single suite is most of the push gate's wall clock, so it is split across
+# this many ci-single legs. The rest of the base set moves to a ci-pseudo leg,
+# keeping the ci-single legs to the runners there are.
+SINGLE_SHARDS="${E2E_SINGLE_SHARDS:-3}"
+[[ "$SINGLE_SHARDS" =~ ^[1-9][0-9]*$ ]] || { printf '[e2e-select] error: E2E_SINGLE_SHARDS=%s\n' "$SINGLE_SHARDS" >&2; exit 1; }
+
 log() { printf '[e2e-select] %s\n' "$*" >&2; }
 die() { log "error: $*"; exit 1; }
 
@@ -54,19 +60,31 @@ prefixed() {
   printf '%s' "$out"
 }
 
-# leg <name> <runner> <suites>: one matrix entry, or nothing for no suites.
+# leg <name> <runner> <suites> [<shard> <shards>]: one matrix entry, or nothing
+# for no suites. shard is 0 for a leg that runs its suites whole.
 leg() {
   [ -n "$3" ] || return 0
   jq -nc --arg leg "$1" --arg runner "$2" --arg suites "$(prefixed "$3")" \
-    --argjson timeout "$(budget "$3")" \
-    '{leg: $leg, runner: $runner, suites: $suites, timeout: $timeout}'
+    --argjson timeout "$(budget "$3")" --argjson shard "${4:-0}" --argjson shards "${5:-0}" \
+    '{leg: $leg, runner: $runner, suites: $suites, timeout: $timeout, shard: $shard, shards: $shards}'
 }
 
 # emit <base-single> <service-single> <multi>
 emit() {
-  local legs
-  legs=$( { leg base "${BASE_RUNNER}" "$1"; leg service "${SERVICE_RUNNER}" "$2"; } | jq -sc .)
-  log "base leg: ${1:-none}"
+  local legs rest k
+  if [ "$SINGLE_SHARDS" -gt 1 ] && in_list single "$1"; then
+    rest=$(for s in $1; do [ "$s" = single ] || printf '%s ' "$s"; done)
+    legs=$( {
+      for ((k = 1; k <= SINGLE_SHARDS; k++)); do leg "single-$k" "${BASE_RUNNER}" single "$k" "$SINGLE_SHARDS"; done
+      leg base "${SERVICE_RUNNER}" "${rest% }"
+      leg service "${SERVICE_RUNNER}" "$2"
+    } | jq -sc .)
+    log "single legs: ${SINGLE_SHARDS} shards on ${BASE_RUNNER}"
+    log "base leg: ${rest:-none}"
+  else
+    legs=$( { leg base "${BASE_RUNNER}" "$1"; leg service "${SERVICE_RUNNER}" "$2"; } | jq -sc .)
+    log "base leg: ${1:-none}"
+  fi
   log "service leg: ${2:-none}"
   log "multi: ${3:-none}"
   printf 'single_legs=%s\n' "$legs"

.github/scripts/e2e-select_test.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+[ -n "${BASH_VERSION:-}" ] || exec bash "$0" "$@"
+# Tests for e2e-select.sh's single-suite sharding: the shard legs, where the
+# rest of the base set goes, and the unsharded fallback.
+set -uo pipefail
+
+here=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+select="$here/e2e-select.sh"
+
+fail=0
+check() { if [ "$2" = "$3" ]; then echo "ok   $1"; else echo "FAIL $1: want '$3', got '$2'"; fail=1; fi; }
+
+legs() { sed -n 's/^single_legs=//p'; }
+
+out=$(bash "$select" suites "e2e-single e2e-iam" 2>/dev/null | legs)
+check "three single shards on ci-single" \
+  "$(jq -c '[.[]|select(.suites=="e2e-single")|[.leg,.runner,.shard,.shards]]' <<<"$out")" \
+  '[["single-1","ci-single",1,3],["single-2","ci-single",2,3],["single-3","ci-single",3,3]]'
+check "the rest of the base set runs whole on ci-pseudo" \
+  "$(jq -c '[.[]|select(.leg=="base")|[.runner,.suites,.shard]]' <<<"$out")" '[["ci-pseudo","e2e-iam",0]]'
+
+out=$(bash "$select" suites "e2e-single" 2>/dev/null | legs)
+check "no base leg when single is all there is" "$(jq -c '[.[]|.leg]' <<<"$out")" '["single-1","single-2","single-3"]'
+
+out=$(bash "$select" suites "e2e-iam" 2>/dev/null | legs)
+check "no shards without the single suite" "$(jq -c '[.[]|[.leg,.runner,.suites]]' <<<"$out")" '[["base","ci-single","e2e-iam"]]'
+
+out=$(E2E_SINGLE_SHARDS=1 bash "$select" suites "e2e-single e2e-iam" 2>/dev/null | legs)
+check "one shard keeps the old single leg" "$(jq -c '[.[]|[.leg,.runner,.suites]]' <<<"$out")" '[["base","ci-single","e2e-single e2e-iam"]]'
+
+E2E_SINGLE_SHARDS=0 bash "$select" suites "e2e-single" >/dev/null 2>&1
+check "rejects a zero shard count" "$?" "1"
+
+exit "$fail"

.github/scripts/e2e-shard.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# Usage: e2e-shard.sh [--skip] <shard> <shards> [weights-file] < test-names
+#
+# Prints a regex matching shard <shard> (1-based) of <shards> from the
+# top-level test names on stdin; with --skip, one matching every other shard,
+# for -test.skip, so -test.run never names a test and opt-in tests stay off.
+# Every leg runs this on the same list, so every test lands in exactly one
+# shard, including tests added since the weights were measured.
+#
+# Longest-first onto the least-loaded shard, using measured seconds from the
+# weights file ("TestName seconds" per line) and DEFAULT_WEIGHT_S otherwise.
+# Prints nothing for an empty shard; the caller skips the suite. Exits 3 when
+# stdin holds no test names, so a failed listing cannot pass as an empty shard.
+set -euo pipefail
+
+mode=run
+if [ "${1:-}" = --skip ]; then mode=skip; shift; fi
+shard="${1:?usage: $0 <shard> <shards> [weights-file] < names}"
+shards="${2:?usage: $0 <shard> <shards> [weights-file] < names}"
+weights="${3:-/dev/null}"
+default_w="${DEFAULT_WEIGHT_S:-60}"
+
+[[ "$shard" =~ ^[0-9]+$ && "$shards" =~ ^[0-9]+$ ]] || { echo "e2e-shard: shard and shards must be integers" >&2; exit 2; }
+[ "$shards" -ge 1 ] && [ "$shard" -ge 1 ] && [ "$shard" -le "$shards" ] ||
+  { echo "e2e-shard: shard $shard is not in 1..$shards" >&2; exit 2; }
+[ -r "$weights" ] || { echo "e2e-shard: cannot read $weights" >&2; exit 2; }
+
+awk -v mode="$mode" -v shard="$shard" -v shards="$shards" -v dflt="$default_w" -v wfile="$weights" '
+  BEGIN { while ((getline line < wfile) > 0) { split(line, f, " "); if (f[1] ~ /^Test/ && f[2] ~ /^[0-9]+$/) w[f[1]] = f[2] } }
+  $1 ~ /^Test[A-Za-z0-9_]*$/ && !seen[$1]++ { n++; name[n] = $1; wt[n] = ($1 in w) ? w[$1] : dflt }
+  END {
+    if (n == 0) { print "e2e-shard: no Test names on stdin" > "/dev/stderr"; exit 3 }
+    # Heaviest first, ties by name, so every caller orders identically.
+    for (i = 2; i <= n; i++) {
+      kn = name[i]; kw = wt[i]
+      for (j = i - 1; j >= 1 && (wt[j] < kw || (wt[j] == kw && name[j] > kn)); j--) {
+        name[j + 1] = name[j]; wt[j + 1] = wt[j]
+      }
+      name[j + 1] = kn; wt[j + 1] = kw
+    }
+    for (s = 1; s <= shards; s++) load[s] = 0
+    out = ""
+    for (i = 1; i <= n; i++) {
+      best = 1
+      for (s = 2; s <= shards; s++) if (load[s] < load[best]) best = s
+      load[best] += wt[i]
+      if ((best == shard) == (mode == "run")) out = out (out == "" ? "" : "|") name[i]
+    }
+    if (out != "") print "^(" out ")$"
+  }
+'

.github/scripts/e2e-shard_test.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+[ -n "${BASH_VERSION:-}" ] || exec bash "$0" "$@"
+# Tests for e2e-shard.sh: the shards partition the input exactly, balance by
+# weight, and reject bad arguments.
+set -uo pipefail
+
+here=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+shard="$here/e2e-shard.sh"
+tmp=$(mktemp -d)
+trap 'rm -rf "$tmp"' EXIT
+
+fail=0
+check() { if [ "$2" = "$3" ]; then echo "ok   $1"; else echo "FAIL $1: want '$3', got '$2'"; fail=1; fi; }
+
+# Names the regexes select, one per line, sorted.
+selected() { sed -E 's/^\^\(//; s/\)\$$//' | tr '|' '\n' | grep -v '^$' | sort; }
+
+printf 'TestA\nTestB\nTestC\nTestD\nTestE\nTestF\nTestG\nBenchmarkX\nExampleY\nTestA\n' >"$tmp/names"
+printf 'TestA 300\nTestB 200\nTestC 150\n# comment\nTestZ 999\n' >"$tmp/w"
+
+all=""
+for k in 1 2 3; do
+  all+=$("$shard" "$k" 3 "$tmp/w" <"$tmp/names" | selected)$'\n'
+done
+check "every test in exactly one shard" "$(grep -v '^$' <<<"$all" | sort | tr '\n' ' ')" "TestA TestB TestC TestD TestE TestF TestG "
+check "benchmarks, examples and duplicates are dropped" "$(grep -c -E 'Benchmark|Example' <<<"$all")" "0"
+
+# 300 | 200 | 150, then the four 60s defaults onto the lightest each time.
+check "shard 1 takes the heaviest" "$("$shard" 1 3 "$tmp/w" <"$tmp/names")" '^(TestA)$'
+check "lighter shards absorb the defaults" "$("$shard" 3 3 "$tmp/w" <"$tmp/names")" '^(TestC|TestD|TestF)$'
+
+check "--skip matches every other shard" \
+  "$("$shard" --skip 3 3 "$tmp/w" <"$tmp/names" | selected | tr '\n' ' ')" "TestA TestB TestE TestG "
+check "--skip of a lone shard matches nothing" "$("$shard" --skip 1 1 <"$tmp/names")" ""
+
+check "one shard selects everything" "$("$shard" 1 1 <"$tmp/names" | selected | wc -l)" "7"
+check "an empty shard prints nothing" "$(printf 'TestA\n' | "$shard" 2 2)" ""
+check "the same input gives the same answer" "$("$shard" 2 3 "$tmp/w" <"$tmp/names")" "$("$shard" 2 3 "$tmp/w" <"$tmp/names")"
+
+"$shard" 4 3 </dev/null 2>/dev/null
+check "rejects a shard past the count" "$?" "2"
+"$shard" 1 x </dev/null 2>/dev/null
+check "rejects a non-integer count" "$?" "2"
+"$shard" 1 2 "$tmp/missing" </dev/null 2>/dev/null
+check "rejects an unreadable weights file" "$?" "2"
+"$shard" 1 2 </dev/null 2>/dev/null
+check "fails on an empty listing" "$?" "3"
+printf 'ok  \tgithub.com/x\nBenchmarkX\n' | "$shard" 1 2 2>/dev/null
+check "fails on a listing with no tests" "$?" "3"
+
+exit "$fail"

.github/scripts/hv-lock.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# Usage: hv-lock.sh <env.tfvars> -- <command> [args...]
+#    or: . hv-lock.sh; hv_lock_acquire <env.tfvars>   (held until the shell exits)
+#
+# Holds an exclusive lock on every hypervisor the tofu env names. Concurrent
+# destroy, scrub and apply against one libvirtd drop its connections, so
+# provisioning is serialised per hypervisor, not per env.
+#
+# Locks are files on the runner host, so they only exclude runners that share
+# it; every e2e runner lives on one host. Hosts are locked in sorted order, so
+# two envs sharing hypervisors cannot deadlock.
+#
+# Env: HV_LOCK_DIR ($HOME/.cache/spinifex-ci/hv-locks), HV_LOCK_TIMEOUT_S (1800).
+
+# hv_lock_hosts <env.tfvars>: the distinct node names, sorted, one per line.
+hv_lock_hosts() {
+  grep -v '^[[:space:]]*#' "$1" |
+    sed -nE 's/.*\{[[:space:]]*name[[:space:]]*=[[:space:]]*"([^"]+)".*/\1/p' | sort -u
+}
+
+# hv_lock_acquire <env.tfvars>: locks every host, or returns 1 on timeout and
+# 2 on a file it cannot use. The descriptors stay open in the calling shell.
+hv_lock_acquire() {
+  local tfvars="$1" dir="${HV_LOCK_DIR:-$HOME/.cache/spinifex-ci/hv-locks}"
+  local timeout_s="${HV_LOCK_TIMEOUT_S:-1800}" hosts host fd start remaining
+  if [ ! -r "$tfvars" ]; then
+    echo "::error::hv-lock: cannot read $tfvars" >&2
+    return 2
+  fi
+  hosts=$(hv_lock_hosts "$tfvars")
+  if [ -z "$hosts" ]; then
+    echo "::error::hv-lock: no node names in $tfvars; refusing to run unlocked" >&2
+    return 2
+  fi
+  mkdir -p "$dir" || { echo "::error::hv-lock: cannot create $dir" >&2; return 2; }
+  start=$SECONDS
+  for host in $hosts; do
+    exec {fd}>"$dir/$host.lock" || { echo "::error::hv-lock: cannot open $dir/$host.lock" >&2; return 2; }
+    if ! flock -n "$fd"; then
+      echo "hv-lock: waiting for $host"
+      remaining=$(( timeout_s - (SECONDS - start) ))
+      if [ "$remaining" -le 0 ] || ! flock -w "$remaining" "$fd"; then
+        echo "::error::hv-lock: $host still locked after ${timeout_s}s; another job is provisioning on it" >&2
+        return 1
+      fi
+    fi
+  done
+  echo "hv-lock: holding $(tr '\n' ' ' <<<"$hosts")(waited $(( SECONDS - start ))s)"
+}
+
+if [ "${BASH_SOURCE[0]}" = "$0" ]; then
+  set -uo pipefail
+  usage() { echo "usage: $0 <env.tfvars> -- <command> [args...]" >&2; exit 2; }
+  [ $# -ge 3 ] && [ "$2" = "--" ] || usage
+  tfvars="$1"
+  shift 2
+  hv_lock_acquire "$tfvars" || exit $?
+  "$@"
+fi

.github/scripts/hv-lock_test.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+[ -n "${BASH_VERSION:-}" ] || exec bash "$0" "$@"
+# Tests for hv-lock.sh: host parsing, exclusion between envs sharing a
+# hypervisor, independence of envs that share none, and the timeout.
+set -uo pipefail
+
+here=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+lock="$here/hv-lock.sh"
+tmp=$(mktemp -d)
+trap 'rm -rf "$tmp"' EXIT
+export HV_LOCK_DIR="$tmp/locks"
+
+fail=0
+check() { if [ "$2" = "$3" ]; then echo "ok   $1"; else echo "FAIL $1: want '$3', got '$2'"; fail=1; fi; }
+
+cat >"$tmp/a.tfvars" <<'EOF'
+cluster_name = "envA"
+nodes = [
+  { name = "casuarina", wan_ip = "192.168.0.22", storage_pool = "drive2-pool" },
+  # { name = "banksia", wan_ip = "192.168.0.99", storage_pool = "drive1-pool" },
+  { name = "bottlebrush", wan_ip = "192.168.0.23", storage_pool = "drive2-pool" },
+]
+EOF
+cat >"$tmp/b.tfvars" <<'EOF'
+nodes = [
+  { name = "bottlebrush", wan_ip = "192.168.0.40", storage_pool = "drive1-pool" },
+]
+EOF
+cat >"$tmp/c.tfvars" <<'EOF'
+nodes = [
+  { name = "wattle", wan_ip = "192.168.0.50", storage_pool = "drive1-pool" },
+]
+EOF
+printf 'cluster_name = "empty"\n' >"$tmp/empty.tfvars"
+
+out=$("$lock" "$tmp/a.tfvars" -- echo ran)
+check "runs the command" "$(tail -n1 <<<"$out")" "ran"
+check "locks named hosts, skips commented ones" "$(ls "$HV_LOCK_DIR" | tr '\n' ' ')" "bottlebrush.lock casuarina.lock "
+
+"$lock" "$tmp/a.tfvars" -- false
+check "returns the command's status" "$?" "1"
+
+"$lock" "$tmp/empty.tfvars" -- true 2>/dev/null
+check "refuses a file with no nodes" "$?" "2"
+
+"$lock" "$tmp/a.tfvars" true 2>/dev/null
+check "requires the -- separator" "$?" "2"
+
+# Sourced: the lock is held by the calling shell until it exits.
+# shellcheck source=hv-lock.sh
+( . "$lock"; hv_lock_acquire "$tmp/c.tfvars" >/dev/null; sleep 2 ) &
+sleep 0.5
+HV_LOCK_TIMEOUT_S=1 "$lock" "$tmp/c.tfvars" -- true >/dev/null 2>&1
+check "a sourced acquire holds the lock in its shell" "$?" "1"
+wait
+"$lock" "$tmp/c.tfvars" -- true >/dev/null
+check "and releases it when that shell exits" "$?" "0"
+
+# A holds bottlebrush for 2s; B shares it and must wait, C shares nothing.
+"$lock" "$tmp/a.tfvars" -- sleep 2 >/dev/null &
+sleep 0.5
+t0=$SECONDS
+HV_LOCK_TIMEOUT_S=10 "$lock" "$tmp/b.tfvars" -- true >/dev/null
+check "a shared hypervisor waits" "$(( SECONDS - t0 >= 1 ))" "1"
+wait
+
+"$lock" "$tmp/a.tfvars" -- sleep 3 >/dev/null &
+sleep 0.5
+t0=$SECONDS
+"$lock" "$tmp/c.tfvars" -- true >/dev/null
+check "a disjoint env does not wait" "$(( SECONDS - t0 ))" "0"
+HV_LOCK_TIMEOUT_S=1 "$lock" "$tmp/b.tfvars" -- true >/dev/null 2>&1
+check "gives up after the timeout" "$?" "1"
+wait
+
+exit "$fail"

.github/scripts/suite-sets.sh

@@ -203,6 +203,9 @@ e2e_suite_timeout() {
     # sweep after its last write-back, on top of three stop/starts and a boot.
     # A pass is estimated at 40 minutes, unmeasured; the rest is the watch's cap.
     uefistore) echo "60m" ;;
+    # eks finishes rather than hangs: passing runs took 26-29m, and one ran
+    # out at 30m inside DeleteCluster. The default left it minutes of headroom.
+    eks) echo "45m" ;;
     *) echo "30m" ;;
   esac
 }

.github/workflows/actions.lock

@@ -65,7 +65,7 @@ workflows:
         - 'actions/setup-go@v7.0.0'
         - 'actions/upload-artifact@v7.0.2'
         - 'docker/setup-buildx-action@v4.4.1'
-        - 'softprops/action-gh-release@v3.0.3'
+        - 'softprops/action-gh-release@v3.0.4'
     '.github/workflows/segscan-oracle.yml':
         - 'actions/checkout@v7.0.1'
         - 'actions/create-github-app-token@v3.3.0'
@@ -79,7 +79,7 @@ workflows:
         - 'actions/checkout@v7.0.1'
         - 'actions/setup-node@v7.1.0'
         - 'actions/upload-artifact@v7.0.2'
-        - 'pnpm/action-setup@v6.1.0'
+        - 'pnpm/action-setup@v6.1.1'
 dependencies:
     'actions/cache@v6.1.0':
         ref: 'v6.1.0'
@@ -131,13 +131,13 @@ dependencies:
         commit: 'sha1-dfe3c3f87815947d99a8997f908cb6525fc44e9e'
         owner_id: 761456
         repo_id: 258271226
-    'pnpm/action-setup@v6.1.0':
-        ref: 'v6.1.0'
-        commit: 'sha1-ea17c68df8912ef543352723c149a84f56e3d413'
+    'pnpm/action-setup@v6.1.1':
+        ref: 'v6.1.1'
+        commit: 'sha1-73e46992e05b19abd31f2f3c20e0e7370d762115'
         owner_id: 21320719
         repo_id: 262289747
-    'softprops/action-gh-release@v3.0.3':
-        ref: 'v3.0.3'
-        commit: 'sha1-efb35369e0ad2afab669f228072c1b0d510eae64'
+    'softprops/action-gh-release@v3.0.4':
+        ref: 'v3.0.4'
+        commit: 'sha1-8a0cd651e5b2365bd640854a3fa5dfc70e4875f3'
         owner_id: 2242
         repo_id: 204253808

.github/workflows/e2e-ddil.yml

@@ -98,6 +98,8 @@ jobs:
         run: |
           set -euo pipefail
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/hv-lock.sh"
+          hv_lock_acquire "envs/${SPINIFEX_CI_ENV}.tfvars"
           export TF_VAR_git_branch="${GIT_BRANCH}"
           quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
@@ -226,6 +228,9 @@ jobs:
         run: |
           set -euo pipefail
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/hv-lock.sh"
+          hv_lock_acquire "envs/${SPINIFEX_CI_ENV}.tfvars" || {
+            echo "::error::teardown skipped: ${SPINIFEX_CI_ENV} is left up for the next job's pre-clean destroy"; exit 1; }
           quiet_run "tofu workspace select" "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
           quiet_run "telemetry flush"       "$TOFU_LOG" timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" || true
           retry_run "tofu destroy"          "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false

.github/workflows/e2e-nightly.yml

@@ -131,9 +131,11 @@ jobs:
       matrix.host }}${{ matrix.extra && format('/{0}', matrix.extra) || '' }})"
     needs: select
     if: needs.select.outputs.any == 'true'
+    # Six is the ci-single and ci-multi runner count. Provisioning is serialised
+    # per hypervisor by hv-lock.sh, which is what made more than three unsafe.
     strategy:
       fail-fast: false
-      max-parallel: 3
+      max-parallel: 6
       matrix:
         include: ${{ fromJSON(needs.select.outputs.cells) }}
     runs-on: [self-hosted, "${{ matrix.runner }}"]
@@ -230,6 +232,8 @@ jobs:
           esac
 
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/hv-lock.sh"
+          hv_lock_acquire "envs/${SPINIFEX_CI_ENV}.tfvars"
           tofu init -input=false >> "$TOFU_LOG" 2>&1
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
           tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars ${TOFU_VAR_ARGS} -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
@@ -905,6 +909,9 @@ jobs:
             oracle-*) TOFU_VAR_ARGS="-var=host_image_override=${ORACLE_GOLD_IMAGE} -var=host_os=oracle" ;;
           esac
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/hv-lock.sh"
+          hv_lock_acquire "envs/${SPINIFEX_CI_ENV}.tfvars" || {
+            echo "::error::teardown skipped: ${SPINIFEX_CI_ENV} is left up for the next job's pre-clean destroy"; exit 1; }
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
           timeout 120 ./flush-telemetry.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
           # shellcheck disable=SC2086
@@ -1412,7 +1419,7 @@ jobs:
   # also overrides cancellation and keeps the run starting jobs after a cancel.
   performance:
     name: cell-23 (predastore/throughput)
-    needs: [matrix, baremetal, iso-single, iso-multi]
+    needs: [matrix, baremetal, iso-single, iso-multi, stress]
     if: >-
       !cancelled() && (
       inputs.cells == '' ||
@@ -1518,12 +1525,11 @@ jobs:
   # Fault injection, not throughput. Cell-23 asks how fast predastore is when
   # every node is healthy; this asks whether it survives one that is not.
   #
-  # After cell-23 rather than beside it: both want a quiet single runner, and a
-  # benchmark sharing a machine with a fault injection run measures the wrong
-  # thing. The port offset keeps them apart even so.
+  # Beside the cells rather than after them: it measures survival, not speed.
+  # Cell-23 waits for it, so the benchmark still never shares a machine with it.
   stress:
     name: cell-24 (predastore/fault-injection)
-    needs: [matrix, baremetal, iso-single, iso-multi, performance]
+    needs: [select]
     if: >-
       !cancelled() && (
       inputs.cells == '' ||

.github/workflows/e2e-suites.yml

@@ -174,6 +174,8 @@ jobs:
         run: |
           set -euo pipefail
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/hv-lock.sh"
+          hv_lock_acquire "envs/${SPINIFEX_CI_ENV}.tfvars"
           export TF_VAR_git_branch="${GIT_BRANCH}"
           quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
@@ -500,6 +502,9 @@ jobs:
         run: |
           set +e
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/hv-lock.sh"
+          hv_lock_acquire "envs/${SPINIFEX_CI_ENV}.tfvars" || {
+            echo "::error::teardown skipped: ${SPINIFEX_CI_ENV} is left up for the next job's pre-clean destroy"; exit 1; }
           if [ -n "${WAN_IP}" ] && [ -n "${SSH_KEY}" ]; then
             ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i "${SSH_KEY}" \
               tf-user@${WAN_IP} \
@@ -624,6 +629,8 @@ jobs:
         run: |
           set -euo pipefail
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/hv-lock.sh"
+          hv_lock_acquire "envs/${SPINIFEX_CI_ENV}.tfvars"
           export TF_VAR_git_branch="${GIT_BRANCH}"
           quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
@@ -883,6 +890,9 @@ jobs:
         run: |
           set +e
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/hv-lock.sh"
+          hv_lock_acquire "envs/${SPINIFEX_CI_ENV}.tfvars" || {
+            echo "::error::teardown skipped: ${SPINIFEX_CI_ENV} is left up for the next job's pre-clean destroy"; exit 1; }
           if [ -n "${NODES_CSV}" ] && [ -n "${SSH_KEY}" ]; then
             IFS=',' read -ra NODE_LIST <<< "${NODES_CSV}"
             for ip in "${NODE_LIST[@]}"; do

.github/workflows/e2e.yml

@@ -31,6 +31,12 @@ permissions:
   statuses: write
   actions: read
 
+# A newer push to dev supersedes the run in flight, which frees its env slots;
+# main and dispatches keep every run. The superseded commit reports failure.
+concurrency:
+  group: e2e-${{ github.event_name == 'push' && github.ref == 'refs/heads/dev' && 'dev-push' || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'push' && github.ref == 'refs/heads/dev' }}
+
 env:
   GIT_BRANCH: ${{ github.ref_name }}
   TEST_RUN: ${{ inputs.test_run || '' }}
@@ -65,6 +71,8 @@ jobs:
           EVENT: ${{ github.event_name }}
           BEFORE: ${{ github.event.before }}
           IN_SUITES: ${{ inputs.suites }}
+          # A test_run regex already narrows the suite, so it runs in one leg.
+          E2E_SINGLE_SHARDS: ${{ inputs.test_run && '1' || '' }}
         run: |
           set -euo pipefail
           SELECT=.github/scripts/e2e-select.sh
@@ -93,15 +101,18 @@ jobs:
       # Both legs can run on banksia at once, so every host path carries the leg.
       TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}-${{ matrix.leg }}.log
       ARTIFACT_DIR: /tmp/spinifex-e2e-artifacts-${{ github.run_id }}-${{ matrix.leg }}
+      # Non-zero on a single-suite shard leg: which share of the suite it runs.
+      SHARD: ${{ matrix.shard }}
+      SHARDS: ${{ matrix.shards }}
       # The base leg keeps the cell name it always had, so its history in
       # Elastic and telemetry continues; any other leg appends its name.
-      E2E_CELL: ${{ matrix.leg == 'base' && 'single' || format('single-{0}', matrix.leg) }}
+      E2E_CELL: ${{ matrix.leg == 'base' && 'single' || startsWith(matrix.leg, 'single-') && matrix.leg || format('single-{0}', matrix.leg) }}
       # Job-level env wins over the workflow-level default. This job and
       # e2e_multi share one run id and both name their hosts node1/node2/node3,
       # so without ci.cell their telemetry collapses into one bucket. This job
       # is where "Bootstrap via Binary Install" bakes SPINIFEX_OTEL_ATTRS into
       # the VM's telemetry.env, so the label has to be set here.
-      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=${{ matrix.leg == 'base' && 'single' || format('single-{0}', matrix.leg) }}
+      SPINIFEX_OTEL_ATTRS: ci.run_id=${{ github.run_id }},ci.workflow=${{ github.workflow }},ci.cell=${{ matrix.leg == 'base' && 'single' || startsWith(matrix.leg, 'single-') && matrix.leg || format('single-{0}', matrix.leg) }}
     steps:
       - name: Reap Orphan Procs
         run: |
@@ -187,6 +198,8 @@ jobs:
         run: |
           set -euo pipefail
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/hv-lock.sh"
+          hv_lock_acquire "envs/${SPINIFEX_CI_ENV}.tfvars"
           export TF_VAR_git_branch="${GIT_BRANCH}"
           quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
@@ -459,6 +472,28 @@ jobs:
           else
             TEST_RUN_FLAG=""
           fi
+          # A shard leg runs only the single suite, so the regex applies to it alone.
+          # Every leg lists the same binary, so the shards partition the suite.
+          if [ "${SHARDS:-0}" -gt 0 ]; then
+            BIN="${GITHUB_WORKSPACE}/spinifex/tests/e2e/_bin/single.test"
+            # Listed separately so a failed listing fails the leg; the sharder
+            # also refuses a listing with no tests in it.
+            # The leg skips the other shards' tests rather than naming its own:
+            # a test named in -test.run counts as opted in (TestRebootLoop).
+            SHARDER="${GITHUB_WORKSPACE}/spinifex/.github/scripts/e2e-shard.sh"
+            WEIGHTS="${GITHUB_WORKSPACE}/spinifex/.github/e2e-weights/single.txt"
+            TEST_LIST=$("${BIN}" -test.list '.*') &&
+              SHARD_RE=$("${SHARDER}" "${SHARD}" "${SHARDS}" "${WEIGHTS}" <<<"${TEST_LIST}") &&
+              SKIP_RE=$("${SHARDER}" --skip "${SHARD}" "${SHARDS}" "${WEIGHTS}" <<<"${TEST_LIST}") || {
+              echo "::error::cannot list or shard the single suite"; echo "rc=1" >> "$GITHUB_OUTPUT"; exit 0; }
+            if [ -z "${SHARD_RE}" ]; then
+              echo "shard ${SHARD}/${SHARDS} has no tests; skipping"
+              echo "rc=0" >> "$GITHUB_OUTPUT"
+              exit 0
+            fi
+            echo "shard ${SHARD}/${SHARDS}: $(tr '|' '\n' <<<"${SHARD_RE}" | wc -l) tests: ${SHARD_RE}"
+            if [ -n "${SKIP_RE}" ]; then TEST_RUN_FLAG="-test.skip='${SKIP_RE}'"; fi
+          fi
           VM_HOST="tf-user@${{ steps.vm.outputs.wan_ip }}"
           SSH_KEY="${{ steps.vm.outputs.ssh_key }}"
           cleanup_remote() {
@@ -594,6 +629,9 @@ jobs:
         run: |
           set +e
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/hv-lock.sh"
+          hv_lock_acquire "envs/${SPINIFEX_CI_ENV}.tfvars" || {
+            echo "::error::teardown skipped: ${SPINIFEX_CI_ENV} is left up for the next job's pre-clean destroy"; exit 1; }
           if [ -n "${{ steps.vm.outputs.wan_ip }}" ] && [ -n "${{ steps.vm.outputs.ssh_key }}" ]; then
             ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
               -i "${{ steps.vm.outputs.ssh_key }}" \
@@ -738,6 +776,8 @@ jobs:
         run: |
           set -euo pipefail
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/hv-lock.sh"
+          hv_lock_acquire "envs/${SPINIFEX_CI_ENV}.tfvars"
           export TF_VAR_git_branch="${GIT_BRANCH}"
           quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
@@ -1053,6 +1093,9 @@ jobs:
         run: |
           set +e
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/hv-lock.sh"
+          hv_lock_acquire "envs/${SPINIFEX_CI_ENV}.tfvars" || {
+            echo "::error::teardown skipped: ${SPINIFEX_CI_ENV} is left up for the next job's pre-clean destroy"; exit 1; }
           if [ -n "${{ steps.cluster.outputs.node1_ip }}" ] && [ -n "${{ steps.cluster.outputs.ssh_key }}" ]; then
             IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
             for ip in "${NODE_LIST[@]}"; do
@@ -1145,7 +1188,9 @@ jobs:
     name: ES CI Run Summary
     needs: [plan, e2e_single, e2e_multi]
     if: always()
-    runs-on: [self-hosted, ci-single]
+    # Any self-hosted runner reaches the sink, and this holds no env. Pinned
+    # to ci-single it waited up to 40 minutes behind a nightly's cells.
+    runs-on: [self-hosted]
     timeout-minutes: 5
     permissions:
       actions: read

.github/workflows/perf-3node.yml

@@ -167,6 +167,8 @@ jobs:
         run: |
           set -euo pipefail
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/hv-lock.sh"
+          hv_lock_acquire "envs/${SPINIFEX_CI_ENV}.tfvars"
           export TF_VAR_git_branch="${GIT_BRANCH}"
           quiet_run "tofu init"                "$TOFU_LOG" tofu init -input=false
           quiet_run "tofu workspace select"    "$TOFU_LOG" tofu workspace select -or-create "${SPINIFEX_CI_ENV}"
@@ -384,6 +386,9 @@ jobs:
         run: |
           set +e
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/hv-lock.sh"
+          hv_lock_acquire "envs/${SPINIFEX_CI_ENV}.tfvars" || {
+            echo "::error::teardown skipped: ${SPINIFEX_CI_ENV} is left up for the next job's pre-clean destroy"; exit 1; }
           UID_SELF=$(id -u)
           WS="${GITHUB_WORKSPACE}"
           reap() {

.github/workflows/publish-system-ami.yml

@@ -1,21 +1,20 @@
 name: Publish System AMIs
 
-# Builds the system AMIs (libguestfs, on the runner) and publishes each
-# compressed qcow2 + .sha256 sidecar to the Cloudflare R2 bucket that backs
-# iso.mulgadc.com/system-ami/. Six images ship:
-#   spinifex-eks-node       Alpine EKS K3s node (server|agent, role at boot)
-#   spinifex-ecs-node       Alpine ECS container instance
-#   spinifex-eks-node-gpu   Ubuntu EKS K3s GPU worker (NVIDIA driver + toolkit)
-#   spinifex-ecs-node-gpu   Ubuntu ECS GPU container instance
-#   spinifex-rds-postgres   Alpine RDS PostgreSQL instance
-#   spinifex-rds-mariadb    Alpine RDS MariaDB instance
+# Builds the system AMIs and publishes each compressed qcow2 + .sha256 sidecar
+# to the Cloudflare R2 bucket that backs iso.mulgadc.com/system-ami/. The Alpine
+# images build with libguestfs (build-system-image.sh); the Ubuntu GPU images
+# build with mkosi, which installs from resolute-updates and resolute-security,
+# so each release ships the newest kernel. Eight images ship:
+#   spinifex-eks-node               Alpine EKS K3s node (server|agent, role at boot)
+#   spinifex-ecs-node               Alpine ECS container instance
+#   spinifex-rds-postgres           Alpine RDS PostgreSQL instance
+#   spinifex-rds-mariadb            Alpine RDS MariaDB instance
+#   spinifex-eks-node-gpu           Ubuntu EKS K3s GPU worker (mkosi)
+#   spinifex-ecs-node-gpu           Ubuntu ECS GPU container instance (mkosi)
+#   ubuntu-26.04-nvidia-gpu-x86_64  Ubuntu NVIDIA GPU base image (mkosi)
+#   ubuntu-26.04-amd-gpu-x86_64     Ubuntu AMD ROCm GPU base image (mkosi)
 # Operators then install them on any cluster with:
-#   spx admin images import --name spinifex-eks-node
-#   spx admin images import --name spinifex-ecs-node
-#   spx admin images import --name spinifex-eks-node-gpu
-#   spx admin images import --name spinifex-ecs-node-gpu
-#   spx admin images import --name spinifex-rds-postgres
-#   spx admin images import --name spinifex-rds-mariadb
+#   spx admin images import --name <catalog key>
 #
 # Publishing overwrites the mirror artifact, so it runs only from release.yml
 # once a tag has passed its gates, with siblings at the tags go.mod requires.
@@ -54,9 +53,12 @@ jobs:
           VARIANT: ${{ inputs.variant }}
         run: |
           set -euo pipefail
-          cpu='{"name":"spinifex-eks-node","manifest":"scripts/images/eks-node/manifest.conf"},{"name":"spinifex-ecs-node","manifest":"scripts/images/ecs-agent/manifest.conf"}'
-          gpu='{"name":"spinifex-eks-node-gpu","manifest":"scripts/images/eks-node-gpu/manifest.conf"},{"name":"spinifex-ecs-node-gpu","manifest":"scripts/images/ecs-agent-gpu/manifest.conf"}'
-          rds='{"name":"spinifex-rds-postgres","manifest":"scripts/images/rds-postgres/manifest.conf"},{"name":"spinifex-rds-mariadb","manifest":"scripts/images/rds-mariadb/manifest.conf"}'
+          # builder=legacy entries carry a manifest for build-system-image.sh;
+          # builder=mkosi entries carry the mkosi-build.sh --image name and the
+          # R2 object base, which must equal the catalog URL basename.
+          cpu='{"name":"spinifex-eks-node","builder":"legacy","manifest":"scripts/images/eks-node/manifest.conf"},{"name":"spinifex-ecs-node","builder":"legacy","manifest":"scripts/images/ecs-agent/manifest.conf"}'
+          gpu='{"name":"spinifex-eks-node-gpu","builder":"mkosi","image":"spinifex-eks-node-gpu","object":"spinifex-eks-node-gpu-x86_64"},{"name":"spinifex-ecs-node-gpu","builder":"mkosi","image":"spinifex-ecs-node-gpu","object":"spinifex-ecs-node-gpu-x86_64"},{"name":"ubuntu-26.04-nvidia-gpu","builder":"mkosi","image":"ubuntu-gpu-nvidia","object":"ubuntu-26.04-nvidia-gpu-x86_64"},{"name":"ubuntu-26.04-amd-gpu","builder":"mkosi","image":"ubuntu-gpu-amd","object":"ubuntu-26.04-amd-gpu-x86_64"}'
+          rds='{"name":"spinifex-rds-postgres","builder":"legacy","manifest":"scripts/images/rds-postgres/manifest.conf"},{"name":"spinifex-rds-mariadb","builder":"legacy","manifest":"scripts/images/rds-mariadb/manifest.conf"}'
           case "${VARIANT}" in
             cpu) items="${cpu}" ;;
             gpu) items="${gpu}" ;;
@@ -132,6 +134,8 @@ jobs:
           go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Install build + upload tooling
+        # mkosi legs build and upload inside the pinned builder container.
+        if: matrix.builder == 'legacy'
         run: |
           set -euo pipefail
           sudo apt-get update -qq
@@ -144,6 +148,7 @@ jobs:
           fi
 
       - name: Build + publish ${{ matrix.name }} to R2
+        if: matrix.builder == 'legacy'
         working-directory: spinifex
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
@@ -168,3 +173,34 @@ jobs:
             FLAGS+=(--dry-run)
           fi
           ./scripts/publish-system-image.sh "${MANIFEST}" "${FLAGS[@]}"
+
+      - name: Build + publish ${{ matrix.name }} to R2 (mkosi)
+        if: matrix.builder == 'mkosi'
+        working-directory: spinifex
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.CF_R2_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_SECRET_ACCESS_KEY }}
+          R2_ENDPOINT: ${{ secrets.CF_R2_ENDPOINT_URL }}
+          R2_BUCKET: spinifex-iso
+          DRY_RUN: ${{ inputs.dry_run }}
+          IMAGE: ${{ matrix.image }}
+          OBJECT: ${{ matrix.object }}
+          # On-disk scratch for the same reason as the legacy legs: the GPU raw
+          # and its qcow2 do not fit a RAM-backed /tmp.
+          MKOSI_OUTPUT_DIR: ${{ runner.temp }}/mkosi-output
+        run: |
+          set -euo pipefail
+          mkdir -p "${MKOSI_OUTPUT_DIR}"
+          ./scripts/mkosi-build.sh --image "${IMAGE}" -- --manifest-format=json
+          # The kernel this image boots, from mkosi's package manifest.
+          KERNEL=$(jq -r '.packages[] | select(.name | test("^linux-image-[0-9]")) | "\(.name) \(.version)"' "${MKOSI_OUTPUT_DIR}/${IMAGE}.manifest")
+          if [[ -z "${KERNEL}" ]]; then
+            echo "::error::no linux-image-<version> package in ${IMAGE}.manifest" >&2
+            exit 1
+          fi
+          echo "kernel: ${KERNEL}"
+          FLAGS=(--image "${IMAGE}" --name "${OBJECT}")
+          if [[ "${DRY_RUN}" == "true" ]]; then
+            FLAGS+=(--dry-run)
+          fi
+          ./scripts/mkosi-publish.sh "${FLAGS[@]}"

.github/workflows/release.yml

@@ -150,7 +150,7 @@ jobs:
           path: artifacts
 
       - name: Create Draft GitHub Release
-        uses: softprops/action-gh-release@v3.0.3
+        uses: softprops/action-gh-release@v3.0.4
         with:
           draft: true
           files: |
@@ -218,6 +218,8 @@ jobs:
         working-directory: mulga/scripts/tofu-cluster
         run: |
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/hv-lock.sh"
+          hv_lock_acquire "envs/${SPINIFEX_CI_ENV}.tfvars"
           tofu init -input=false >> "$TOFU_LOG" 2>&1
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
           tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
@@ -236,6 +238,9 @@ jobs:
         working-directory: mulga/scripts/tofu-cluster
         run: |
           source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/hv-lock.sh"
+          hv_lock_acquire "envs/${SPINIFEX_CI_ENV}.tfvars" || {
+            echo "::error::teardown skipped: ${SPINIFEX_CI_ENV} is left up for the next job's pre-clean destroy"; exit 1; }
           tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
           retry_run "tofu destroy" "$TOFU_LOG" tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false
           ./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true

.github/workflows/spinifex.yml

@@ -263,9 +263,21 @@ jobs:
           path: ~/.cache/go-build
           key: gobuild-${{ github.job }}-${{ github.run_id }}
 
+  # Three parallel jobs: run serially, integration and the gated suites made
+  # this the longest PR check. lint_and_security_result reports all three.
   lint_and_security:
-    name: Lint + Security + Integration
+    name: ${{ matrix.name }}
     runs-on: ubuntu-26.04
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - check: lint
+            name: Lint and Govulncheck
+          - check: integration
+            name: Integration
+          - check: gated
+            name: Gated Suites
 
     steps:
       - name: Checkout Spinifex
@@ -338,8 +350,10 @@ jobs:
         uses: actions/cache/restore@v6.1.0
         with:
           path: ~/.cache/go-build
-          key: gobuild-${{ github.job }}-${{ github.run_id }}
-          restore-keys: gobuild-${{ github.job }}-
+          key: gobuild-${{ github.job }}-${{ matrix.check }}-${{ github.run_id }}
+          restore-keys: |
+            gobuild-${{ github.job }}-${{ matrix.check }}-
+            gobuild-${{ github.job }}-
 
       # Go refreshes an entry's mtime on use only when it is over an hour old, so
       # restored entries are backdated past that before any Go command runs and
@@ -352,20 +366,22 @@ jobs:
         run: go work init ./spinifex ./viperblock ./predastore ./northstar ./bluebottle
 
       - name: Lint
+        if: ${{ matrix.check == 'lint' }}
         uses: golangci/golangci-lint-action@v9.3.0
         with:
           version: v2.14.0
           working-directory: spinifex
 
       - name: Govulncheck
+        if: ${{ matrix.check == 'lint' && !cancelled() }}
         run: make -C spinifex govulncheck
 
       - name: Run Integration Tests
-        if: ${{ !cancelled() }}
+        if: ${{ matrix.check == 'integration' }}
         run: make -C spinifex test-integration
 
       - name: Run Gated Suites
-        if: ${{ !cancelled() }}
+        if: ${{ matrix.check == 'gated' }}
         run: make -C spinifex test-gated GOTESTFLAGS=-count=1
 
       # Keeps what this run read or wrote and drops the rest, so the saved entry
@@ -380,4 +396,19 @@ jobs:
         uses: actions/cache/save@v6.1.0
         with:
           path: ~/.cache/go-build
-          key: gobuild-${{ github.job }}-${{ github.run_id }}
+          key: gobuild-${{ github.job }}-${{ matrix.check }}-${{ github.run_id }}
+
+  # The context main's ruleset requires, passing only when all three jobs pass.
+  lint_and_security_result:
+    name: Lint and Security
+    if: ${{ always() }}
+    needs: [lint_and_security]
+    runs-on: ubuntu-26.04
+    timeout-minutes: 5
+    steps:
+      - name: Require lint, integration and gated suites
+        env:
+          RESULT: ${{ needs.lint_and_security.result }}
+        run: |
+          echo "lint, integration and gated suites: ${RESULT}"
+          [ "${RESULT}" = success ]

.github/workflows/ui.yml

@@ -51,7 +51,7 @@ jobs:
           sparse-checkout: spinifex/runtime/roles/spinifexui/frontend
 
       - name: Setup pnpm
-        uses: pnpm/action-setup@v6.1.0
+        uses: pnpm/action-setup@v6.1.1
         with:
           package_json_file: spinifex/runtime/roles/spinifexui/frontend/package.json
 
@@ -127,7 +127,7 @@ jobs:
           sparse-checkout: spinifex/runtime/roles/spinifexui/frontend
 
       - name: Setup pnpm
-        uses: pnpm/action-setup@v6.1.0
+        uses: pnpm/action-setup@v6.1.1
         with:
           package_json_file: spinifex/runtime/roles/spinifexui/frontend/package.json
 

Makefile

@@ -276,8 +276,15 @@ install-microvm: $(MICROVM_ARTIFACTS) $(QBOOT_ARTIFACTS) ## Install microVM arti
 # -j2 overlaps the only two heavy gates, lint and test-cover: alone each leaves
 # most of the machine idle, so pairing them cuts a cold run by about a quarter.
 # -Otarget keeps each gate's output in one block instead of interleaving them.
+# The shell suites mostly wait rather than compute, so they run beside the -j2
+# Go gates instead of taking one of their two slots.
 preflight:
-	@$(MAKE) --no-print-directory -j2 -Otarget QUIET=1 lint govulncheck test-cover diff-coverage test-harness test-build-scripts
+	@sh_log=$$(mktemp); trap 'rm -f "$$sh_log"' EXIT; \
+	$(MAKE) --no-print-directory QUIET=1 test-build-scripts > "$$sh_log" 2>&1 & sh_pid=$$!; \
+	$(MAKE) --no-print-directory -j2 -Otarget QUIET=1 lint govulncheck test-cover diff-coverage test-harness; go_rc=$$?; \
+	wait $$sh_pid; sh_rc=$$?; \
+	cat "$$sh_log"; \
+	[ "$$go_rc" -eq 0 ] && [ "$$sh_rc" -eq 0 ]
 	@echo -e "\n ✅ Preflight passed — safe to commit."
 
 # Shell suites + shellcheck for build/scripts/, the systemd-unit helpers that
@@ -291,27 +298,24 @@ preflight:
 # deletion, not a style note.
 NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/install-node.sh scripts/smoke-test.sh scripts/node-reset.sh scripts/uninstall-spx.sh scripts/lib/distro.sh scripts/lib/stage-sqs-store-microvm.sh
 
+# One target per suite so make runs them in parallel; -k reports every failing
+# suite rather than the first, and -Otarget keeps each suite's output together.
+SHELL_SUITES := $(addprefix shell-suite/,$(sort $(shell find build/scripts -name '*_test.sh'; \
+	find scripts -maxdepth 1 -name '*_test.sh'; find scripts/lib scripts/appliance -name '*_test.sh'; \
+	find scripts/terraform/latitude-spx .github/scripts -maxdepth 1 -name '*_test.sh')))
+.PHONY: $(SHELL_SUITES)
+$(SHELL_SUITES): shell-suite/%:
+	@t=$*; log=$$(mktemp); trap 'rm -f "$$log"' EXIT; $(_RUNSH)
+
 test-build-scripts:
-	@$(_HDR) "\n....Running build/scripts/**/*_test.sh...."
-	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$(find build/scripts -name '*_test.sh' | sort); do \
-		$(_RUNSH); \
-	done
-	@$(_HDR) "\n....Running scripts/*_test.sh, scripts/lib/*_test.sh and scripts/appliance/*_test.sh...."
-	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$( (find scripts -maxdepth 1 -name '*_test.sh'; find scripts/lib scripts/appliance -name '*_test.sh') | sort); do \
-		$(_RUNSH); \
-	done
-	@$(_HDR) "\n....Running scripts/terraform/latitude-spx/*_test.sh...."
-	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$(find scripts/terraform/latitude-spx -maxdepth 1 -name '*_test.sh' | sort); do \
-		$(_RUNSH); \
-	done
-	@$(_HDR) "\n....Running .github/scripts/*_test.sh...."
-	@log=$$(mktemp); trap 'rm -f "$$log"' EXIT; for t in $$(find .github/scripts -maxdepth 1 -name '*_test.sh' | sort); do \
-		$(_RUNSH); \
-	done
+	@$(_HDR) "\n....Running the *_test.sh suites in parallel...."
+	@$(MAKE) --no-print-directory -k -j$$(nproc) -Otarget $(SHELL_SUITES)
 	$(_Q)shellcheck -S warning .github/scripts/hypervisor-disk-preflight.sh \
 		.github/scripts/wait-for-memory.sh \
 		.github/scripts/go-build-cache.sh \
-		.github/scripts/e2e-select.sh
+		.github/scripts/e2e-select.sh \
+		.github/scripts/e2e-shard.sh \
+		.github/scripts/hv-lock.sh
 	@$(_HDR) "\n....Running shellcheck over build/scripts/**/*.sh + the node scripts...."
 	$(_Q)shellcheck -S warning $$(find build/scripts -name '*.sh' | sort) $(NODE_SCRIPTS)
 	$(_Q)shellcheck -S warning -x scripts/build-oracle-appliance.sh $$(find scripts/appliance -name '*.sh' | sort)
@@ -515,9 +519,9 @@ install-system:
 	fi
 
 install-go:
-	@echo -e "\n....Installing Go 1.27.3 for $(ARCH) ($(GO_ARCH))...."
+	@echo -e "\n....Installing Go 1.27.4 for $(ARCH) ($(GO_ARCH))...."
 	@if [ ! -d "/usr/local/go" ]; then \
-		curl -L https://go.dev/dl/go1.27.3.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
+		curl -L https://go.dev/dl/go1.27.4.linux-$(GO_ARCH).tar.gz | tar -C /usr/local -xz; \
 	else \
 		echo "Go already installed in /usr/local/go"; \
 	fi

README.md

@@ -70,8 +70,8 @@ The platform serves **405 AWS API operations** across the services below. See ex
 | **RDS** | Databases | PostgreSQL and MariaDB, with automated backups | Available |
 | **Route 53** | DNS | [Northstar](https://github.com/mulgadc/northstar): authoritative DNS with hosted zones | Available |
 | **Bedrock** | AI deployment | vLLM for text generation and Hugging Face TEI for embeddings and reranking | Q4 2026 |
-| **Lambda** | Serverless functions | QEMU `microvm` execution environments | Q1 2027 |
-| **SQS** | Message queues | A dedicated NATS JetStream store in a per-Region QEMU `microvm` guest | Q1 2027 |
+| **Lambda** | Serverless functions | Firecracker microVM execution environments | Q1 2027 |
+| **SQS** | Message queues | A dedicated NATS JetStream store in a per-Region microVM guest | Q1 2027 |
 
 Roadmap services ship under the same AWS API surface, so code written for AWS today works the moment they land. [Track what's shipped in the release notes](https://github.com/mulgadc/spinifex/releases).
 

build/Dockerfile.distro

@@ -1,6 +1,6 @@
 ARG VERSION=dev
 
-FROM golang:1.27.3-bookworm@sha256:d9b6a1beb9474adc2a1ce842573639b44b2a0dd49b6669f51d3821baf9ec37c9 AS builder
+FROM golang:1.27.4-bookworm@sha256:4fa7c968ba93f1635670522dbb01a2c78150443e1385746924566079f36570ea AS builder
 
 ARG VERSION
 

build/microvm/sqs-worker/sqs-worker-lib.sh

@@ -15,6 +15,7 @@ SQSW_LIB=${SQSW_LIB:-/usr/local/lib/sqs-worker/sqs-worker-lib.sh}
 SQSW_NATS_BIN=${SQSW_NATS_BIN:-/usr/local/bin/nats-server}
 SQSW_STORE_PORT=${SQSW_STORE_PORT:-4222}
 SQSW_STOP_BUDGET_TENTHS=${SQSW_STOP_BUDGET_TENTHS:-100}
+SQSW_KILL_WAIT_TENTHS=${SQSW_KILL_WAIT_TENTHS:-30}
 SQSW_RESPAWN_GAP=${SQSW_RESPAWN_GAP:-5}
 SQSW_PARK=${SQSW_PARK:-sleep 2147483647}
 SQSW_BOOT_RETRY=${SQSW_BOOT_RETRY:-30}
@@ -158,7 +159,7 @@ sqsw_end() {
         sqsw_log "stop: $1 pid=$pid still running after $((SQSW_STOP_BUDGET_TENTHS / 10))s, SIGKILL"
         kill -KILL "$pid" 2>/dev/null || true
         j=0
-        while sqsw_same "$pid" "$st" && [ "$j" -lt 30 ]; do sleep 0.1; j=$((j + 1)); done
+        while sqsw_same "$pid" "$st" && [ "$j" -lt "$SQSW_KILL_WAIT_TENTHS" ]; do sleep 0.1; j=$((j + 1)); done
     fi
     if sqsw_same "$pid" "$st"; then
         sqsw_log "stop: $1 pid=$pid STILL ALIVE"

contracts/ec2/v1/commands.go

@@ -84,6 +84,9 @@ type DetachVolumeData struct {
 type AttachENIData struct {
 	NetworkInterfaceID string `json:"network_interface_id"`
 	DeviceIndex        int64  `json:"device_index"`
+	// AssignFreeIndex skips the occupied-index refusal for platform callers that
+	// accept whichever hot-plug slot is free. The customer API never sets it.
+	AssignFreeIndex bool `json:"assign_free_index,omitempty"`
 }
 
 // DetachENIData carries parameters for a detach-network-interface command.

contracts/viperblockd/legacy/v1/README.md

@@ -1,92 +0,0 @@
-# Viperblockd legacy contracts, version 1
-
-This package owns the deployed, transitional configuration-update and
-state-reload request/reply protocols between Spinifex's legacy control plane
-and `viperblockd`. It makes those existing compatibility boundaries explicit
-while the generic `providers/ebs` contract is adopted; it is not the EBS
-provider abstraction and new provider integrations must not depend on it.
-
-## Configuration update routes
-
-`ConfigUpdateSubject` is `ebs.config`. Every `viperblockd` daemon subscribes
-to it in the `spinifex-workers` queue group. It is the fallback path for an
-encrypted volume with no live mount: the selected daemon opens the volume
-exclusively and reseals its state.
-
-`VolumeConfigUpdateSubject(volume)` is `ebs.config.<volume>`. The daemon with
-the live mounted volume subscribes directly so the sole holder of the volume's
-state sequence number applies the update.
-
-Both routes carry `EBSConfigUpdateRequest` and respond with
-`EBSConfigUpdateResponse`. The JSON field spelling, route shapes, and
-request/reply semantics are deployed compatibility behaviour. Incompatible
-changes require a new contract version or an explicit migration plan.
-
-## State reload route
-
-`SyncSubject` is `ebs.sync`. Every `viperblockd` worker subscribes to it in the `spinifex-workers` queue group, and the selected worker reloads the volume's state from its backend and responds with `EBSSyncResponse`.
-It returns an error when it does not own a live mount of the volume.
-No control-plane code sends this request: `ExpandVolume` persists the new size before it returns and refuses a mounted volume, so every later mount loads the new size at open.
-The subscriber stays until the legacy subjects are retired.
-
-The route carries `EBSSyncRequest` and `EBSSyncResponse`. Their JSON shape and
-the queue-group route are compatibility behaviour.
-
-## Volume lifecycle routes
-
-`MountSubject(node)` and `UnmountSubject(node)` retain the deployed mount and
-unmount routes. With a node name they address that node's Viperblockd process
-(`ebs.<node>.mount` or `ebs.<node>.unmount`); without one they retain the
-single-node `spinifex-workers` queue-group routes. Both carry `EBSRequest`.
-`EBSMountResponse` supplies the exported NBD URI, while
-`EBSUnMountResponse` reports the seal result. The response broadcasts remain
-`ebs.mount.response` and `ebs.unmount.response` alongside ordinary
-request/reply replies.
-
-`DeleteSubject` retains the `ebs.delete` queue-group route. It carries
-`EBSDeleteRequest` and responds with `EBSDeleteResponse` after Viperblockd has
-removed its local state and export. These routes and JSON shapes are deployed
-legacy compatibility behaviour, not a generic EBS-provider interface.
-
-## Abandon route
-
-`VolumeAbandonSubject(node)` is `ebs.<node>.abandon`. The daemon uses
-request/reply to ask the Viperblock adapter on that node to tear an export
-down without sealing, because the instance using it belongs to another node
-now. The route carries `VolumeAbandonRequest` and responds with
-`VolumeAbandonResponse`. `Abandoned` is false with no error when the volume
-was not exported there, which is the ordinary case: the request is
-idempotent by design, since the caller is reconciling rather than
-transacting.
-
-## Dirty-marker bucket
-
-`DirtyBucket` is the JetStream KV bucket `VIPERBLOCK_VOLUME_DIRTY`, keyed by
-volume name under `VolumeKeyPattern`, storing `DirtyRecord` JSON. It records
-which node holds writes for a volume that the backend may not have.
-
-Owner: `providers/ebs/viperblock` owns the dirty-volume state and its
-transitions. `services/viperblockd` was only its transitional custodian.
-
-Producers: only the Viperblock adapter writes, clears or changes a dirty
-record.
-
-Consumers: Viperblock recovery and mount logic on any node. `spx admin` may
-obtain a read-only operational view.
-
-Storage contract: the bucket name, `VolumeKeyPattern` and the `DirtyRecord`
-JSON shape are this package's legacy v1 contract. There is no TTL; a record
-persists until an explicit adapter-owned recovery or unseal transition
-removes it.
-
-Compatibility: this is a deployed legacy v1 contract, and coordinated
-writers and readers support N/N-1. No renames, type changes or semantic
-reinterpretation of existing fields; a breaking change needs a successor
-contract and a migration.
-
-CLI authority: the CLI is never a writer and must not perform arbitrary
-bucket operations.
-
-This package does not authorize requests, validate a volume ID, store volume
-state, or define the generic storage-provider API. Those concerns remain with
-the consuming daemon, the EC2 domain, and `providers/ebs` respectively.

contracts/viperblockd/legacy/v1/abandon.go

@@ -1,36 +0,0 @@
-package viperblocklegacyv1
-
-// VolumeAbandonSubject is where the local daemon asks for an export to be torn
-// down because the instance using it belongs to another node now.
-//
-// It exists because an unmount is the wrong request and killing the guest is not
-// enough. An unmount seals, and this node's copy is the stale one, so sealing
-// would publish an older state over the winner's. Meanwhile the export holds the
-// volume lease, and a lease this node keeps renewing is one the winner can never
-// acquire — the guest would be stopped here and unable to start anywhere.
-//
-// Node-addressed, like mount and unmount: the export is local, and no other
-// node's viperblockd has anything it could tear down.
-func VolumeAbandonSubject(node string) string {
-	if node == "" {
-		return "ebs.abandon"
-	}
-	return "ebs." + node + ".abandon"
-}
-
-// VolumeAbandonRequest asks for one volume's export to go away without sealing.
-// Reason is logged, so a node that dropped an export can be told apart from one
-// that was fenced.
-type VolumeAbandonRequest struct {
-	Volume string `json:"volume"`
-	Reason string `json:"reason"`
-}
-
-// VolumeAbandonResponse reports what happened. Abandoned is false with no error
-// when the volume was not exported here, which is the ordinary case and not a
-// failure: the request is idempotent by design, since the caller is reconciling
-// rather than transacting.
-type VolumeAbandonResponse struct {
-	Abandoned bool   `json:"abandoned"`
-	Error     string `json:"error,omitempty"`
-}

contracts/viperblockd/legacy/v1/abandon_test.go

@@ -1,63 +0,0 @@
-package viperblocklegacyv1_test
-
-import (
-	"encoding/json"
-	"testing"
-
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// TestVolumeAbandonSubject_BothForms pins the abandon route: node-addressed,
-// with an empty-node fallback for a single-node daemon.
-func TestVolumeAbandonSubject_BothForms(t *testing.T) {
-	assert.Equal(t, "ebs.node-a.abandon", viperblocklegacyv1.VolumeAbandonSubject("node-a"))
-	assert.Equal(t, "ebs.abandon", viperblocklegacyv1.VolumeAbandonSubject(""))
-}
-
-// TestVolumeAbandonRequest_JSON pins the request runtime/daemon/vm_adapters.go sends
-// on the abandon route.
-func TestVolumeAbandonRequest_JSON(t *testing.T) {
-	golden := `{"volume":"vol-1","reason":"instance owned by another node"}`
-	req := viperblocklegacyv1.VolumeAbandonRequest{Volume: "vol-1", Reason: "instance owned by another node"}
-
-	marshaled, err := json.Marshal(req)
-	require.NoError(t, err)
-	assert.JSONEq(t, golden, string(marshaled))
-
-	var decoded viperblocklegacyv1.VolumeAbandonRequest
-	require.NoError(t, json.Unmarshal([]byte(golden), &decoded))
-	assert.Equal(t, req, decoded)
-}
-
-// TestVolumeAbandonResponse_JSON pins both response shapes: the ordinary
-// reconciling case with no error, and the failure case where Error is set.
-// Error is omitempty, so the two must not be mistaken for the same shape.
-func TestVolumeAbandonResponse_JSON(t *testing.T) {
-	t.Run("abandoned, no error", func(t *testing.T) {
-		golden := `{"abandoned":true}`
-		resp := viperblocklegacyv1.VolumeAbandonResponse{Abandoned: true}
-
-		marshaled, err := json.Marshal(resp)
-		require.NoError(t, err)
-		assert.JSONEq(t, golden, string(marshaled))
-
-		var decoded viperblocklegacyv1.VolumeAbandonResponse
-		require.NoError(t, json.Unmarshal([]byte(golden), &decoded))
-		assert.Equal(t, resp, decoded)
-	})
-
-	t.Run("not abandoned, with error", func(t *testing.T) {
-		golden := `{"abandoned":false,"error":"nbdkit did not exit"}`
-		resp := viperblocklegacyv1.VolumeAbandonResponse{Abandoned: false, Error: "nbdkit did not exit"}
-
-		marshaled, err := json.Marshal(resp)
-		require.NoError(t, err)
-		assert.JSONEq(t, golden, string(marshaled))
-
-		var decoded viperblocklegacyv1.VolumeAbandonResponse
-		require.NoError(t, json.Unmarshal([]byte(golden), &decoded))
-		assert.Equal(t, resp, decoded)
-	})
-}

contracts/viperblockd/legacy/v1/config_update.go

@@ -1,37 +0,0 @@
-// Package viperblocklegacyv1 contains the versioned, transitional wire
-// contracts between Spinifex's legacy control-plane routes and viperblockd.
-package viperblocklegacyv1
-
-import "encoding/json"
-
-const (
-	// ConfigUpdateSubject is the legacy queue-group route for an encrypted
-	// volume configuration update when no mounted volume owns the request.
-	ConfigUpdateSubject = "ebs.config"
-)
-
-// VolumeConfigUpdateSubject returns the legacy route for an update handled by
-// the node with a live mount of volume. Volume remains unvalidated here: route
-// authorization and volume-name validation are responsibilities of the
-// consumer, as they were before this contract was extracted.
-func VolumeConfigUpdateSubject(volume string) string {
-	return ConfigUpdateSubject + "." + volume
-}
-
-// EBSConfigUpdateRequest carries a control-plane VolumeConfig update for an
-// encrypted volume. config.json is a sealed VBState; only the master-key holder
-// (viperblockd) can reseal it, so the EC2 edge ships the new config here instead
-// of rewriting the object directly. VolumeConfig is a marshaled
-// viperblock.VolumeConfig (RawMessage keeps this contract dependency-free).
-type EBSConfigUpdateRequest struct {
-	Volume       string          `json:"Volume"`
-	VolumeConfig json.RawMessage `json:"VolumeConfig"`
-}
-
-// EBSConfigUpdateResponse is the request/reply response for either legacy
-// configuration-update route.
-type EBSConfigUpdateResponse struct {
-	Volume  string `json:"Volume"`
-	Success bool   `json:"Success"`
-	Error   string `json:"Error"`
-}

contracts/viperblockd/legacy/v1/config_update_test.go

@@ -1,35 +0,0 @@
-package viperblocklegacyv1
-
-import (
-	"encoding/json"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestConfigUpdateSubjects(t *testing.T) {
-	require.Equal(t, "ebs.config", ConfigUpdateSubject)
-	require.Equal(t, "ebs.config.vol-123", VolumeConfigUpdateSubject("vol-123"))
-	// The deployed route builder did not validate its final token. Validation
-	// remains with the handler and is not silently introduced by this move.
-	require.Equal(t, "ebs.config.", VolumeConfigUpdateSubject(""))
-}
-
-func TestConfigUpdateRequestJSONContract(t *testing.T) {
-	data, err := json.Marshal(EBSConfigUpdateRequest{
-		Volume:       "vol-123",
-		VolumeConfig: json.RawMessage(`{"size_gib":8}`),
-	})
-	require.NoError(t, err)
-	require.JSONEq(t, `{"Volume":"vol-123","VolumeConfig":{"size_gib":8}}`, string(data))
-}
-
-func TestConfigUpdateResponseJSONContract(t *testing.T) {
-	data, err := json.Marshal(EBSConfigUpdateResponse{
-		Volume:  "vol-123",
-		Success: true,
-		Error:   "",
-	})
-	require.NoError(t, err)
-	require.JSONEq(t, `{"Volume":"vol-123","Success":true,"Error":""}`, string(data))
-}

contracts/viperblockd/legacy/v1/sync.go

@@ -1,22 +0,0 @@
-package viperblocklegacyv1
-
-const (
-	// SyncSubject is the legacy queue-group route that asks a viperblockd
-	// worker to reload a mounted volume's state from its backend.
-	SyncSubject = "ebs.sync"
-)
-
-// EBSSyncRequest identifies the volume whose mounted Viperblock state should
-// be reloaded by the selected legacy worker.
-type EBSSyncRequest struct {
-	Volume string `json:"Volume"`
-}
-
-// EBSSyncResponse reports the selected worker's reload result. A worker that
-// does not own the mounted volume returns an error rather than forwarding the
-// request.
-type EBSSyncResponse struct {
-	Volume string `json:"Volume"`
-	Synced bool   `json:"Synced"`
-	Error  string `json:"Error"`
-}

contracts/viperblockd/legacy/v1/sync_test.go

@@ -1,28 +0,0 @@
-package viperblocklegacyv1
-
-import (
-	"encoding/json"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestSyncSubject(t *testing.T) {
-	require.Equal(t, "ebs.sync", SyncSubject)
-}
-
-func TestSyncRequestJSONContract(t *testing.T) {
-	data, err := json.Marshal(EBSSyncRequest{Volume: "vol-123"})
-	require.NoError(t, err)
-	require.JSONEq(t, `{"Volume":"vol-123"}`, string(data))
-}
-
-func TestSyncResponseJSONContract(t *testing.T) {
-	data, err := json.Marshal(EBSSyncResponse{
-		Volume: "vol-123",
-		Synced: true,
-		Error:  "",
-	})
-	require.NoError(t, err)
-	require.JSONEq(t, `{"Volume":"vol-123","Synced":true,"Error":""}`, string(data))
-}

contracts/viperblockd/legacy/v1/volume.go

@@ -1,87 +0,0 @@
-package viperblocklegacyv1
-
-import "fmt"
-
-const (
-	// DeleteSubject is the legacy queue-group route that deletes a volume's
-	// Viperblockd-local state and any active export.
-	DeleteSubject = "ebs.delete"
-
-	// MountResponseSubject is the legacy broadcast emitted after an ebs.mount
-	// request is handled. Request/reply callers receive the same payload.
-	MountResponseSubject = "ebs.mount.response"
-
-	// UnmountResponseSubject is the legacy broadcast emitted after an
-	// ebs.unmount request is handled. Request/reply callers receive the same
-	// payload.
-	UnmountResponseSubject = "ebs.unmount.response"
-)
-
-// MountSubject returns the deployed mount route. An empty node retains the
-// single-node queue-group route; a named node selects the Viperblockd worker
-// colocated with the instance runtime.
-func MountSubject(node string) string {
-	if node == "" {
-		return "ebs.mount"
-	}
-	return fmt.Sprintf("ebs.%s.mount", node)
-}
-
-// UnmountSubject returns the deployed unmount route. Its empty-node behaviour
-// matches MountSubject for single-node compatibility.
-func UnmountSubject(node string) string {
-	if node == "" {
-		return "ebs.unmount"
-	}
-	return fmt.Sprintf("ebs.%s.unmount", node)
-}
-
-// EBSRequest is the deployed request payload shared by the legacy mount and
-// unmount routes. It is intentionally separate from the generic EBS provider
-// contract: this is Viperblockd's transitional NATS protocol.
-type EBSRequest struct {
-	Name                string `json:"Name"`
-	VolType             string `json:"VolType"`
-	Boot                bool   `json:"Boot"`
-	EFI                 bool   `json:"EFI"`
-	DeleteOnTermination bool   `json:"DeleteOnTermination"`
-	NBDURI              string `json:"NBDURI"`
-	DeviceName          string `json:"DeviceName"`
-	HotplugPort         int    `json:"HotplugPort,omitempty"`
-	// BlockDevice and BlockSizeBytes describe a host block device export, the
-	// alternative to NBDURI; the guest sees BlockSizeBytes of it. Exactly one
-	// of NBDURI and BlockDevice may be set.
-	BlockDevice    string `json:"BlockDevice,omitempty"`
-	BlockSizeBytes uint64 `json:"BlockSizeBytes,omitempty"`
-}
-
-// EBSMountResponse is the response to an EBSRequest on MountSubject.
-type EBSMountResponse struct {
-	URI       string `json:"URI"`
-	Mounted   bool   `json:"Mounted"`
-	Error     string `json:"Error"`
-	Retryable bool   `json:"Retryable"`
-}
-
-// EBSUnMountResponse is the response to an EBSRequest on UnmountSubject.
-// NotFound reports an idempotent retry after an already-complete seal; Reaped
-// reports a leaked export that was cleaned up without a registry entry.
-type EBSUnMountResponse struct {
-	Volume   string `json:"Volume"`
-	Mounted  bool   `json:"Mounted"`
-	Error    string `json:"Error"`
-	NotFound bool   `json:"NotFound"`
-	Reaped   bool   `json:"Reaped,omitempty"`
-}
-
-// EBSDeleteRequest identifies the volume to delete on DeleteSubject.
-type EBSDeleteRequest struct {
-	Volume string `json:"Volume"`
-}
-
-// EBSDeleteResponse is the response to EBSDeleteRequest on DeleteSubject.
-type EBSDeleteResponse struct {
-	Volume  string `json:"Volume"`
-	Success bool   `json:"Success"`
-	Error   string `json:"Error"`
-}

contracts/viperblockd/legacy/v1/volume_test.go

@@ -1,78 +0,0 @@
-package viperblocklegacyv1
-
-import (
-	"encoding/json"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestVolumeSubjects(t *testing.T) {
-	require.Equal(t, "ebs.delete", DeleteSubject)
-	require.Equal(t, "ebs.mount", MountSubject(""))
-	require.Equal(t, "ebs.unmount", UnmountSubject(""))
-	require.Equal(t, "ebs.node-a.mount", MountSubject("node-a"))
-	require.Equal(t, "ebs.node-a.unmount", UnmountSubject("node-a"))
-	require.Equal(t, "ebs.mount.response", MountResponseSubject)
-	require.Equal(t, "ebs.unmount.response", UnmountResponseSubject)
-}
-
-func TestEBSRequestJSONContract(t *testing.T) {
-	data, err := json.Marshal(EBSRequest{
-		Name:                "vol-123",
-		VolType:             "gp3",
-		Boot:                true,
-		EFI:                 true,
-		DeleteOnTermination: true,
-		NBDURI:              "nbd:unix:/run/spinifex/vol-123.sock",
-		DeviceName:          "/dev/sdf",
-		HotplugPort:         3,
-	})
-	require.NoError(t, err)
-	require.JSONEq(t, `{
-		"Name":"vol-123", "VolType":"gp3", "Boot":true, "EFI":true,
-		"DeleteOnTermination":true,
-		"NBDURI":"nbd:unix:/run/spinifex/vol-123.sock",
-		"DeviceName":"/dev/sdf", "HotplugPort":3
-	}`, string(data))
-
-	block, err := json.Marshal(EBSRequest{
-		Name:           "vol-456",
-		BlockDevice:    "/dev/sdx",
-		BlockSizeBytes: 10 << 30,
-	})
-	require.NoError(t, err)
-	require.JSONEq(t, `{
-		"Name":"vol-456", "VolType":"", "Boot":false, "EFI":false,
-		"DeleteOnTermination":false, "NBDURI":"", "DeviceName":"",
-		"BlockDevice":"/dev/sdx", "BlockSizeBytes":10737418240
-	}`, string(block))
-}
-
-func TestVolumeResponseJSONContracts(t *testing.T) {
-	mount, err := json.Marshal(EBSMountResponse{
-		URI: "nbd:unix:/run/spinifex/vol-123.sock", Mounted: true, Retryable: true,
-	})
-	require.NoError(t, err)
-	require.JSONEq(t, `{
-		"URI":"nbd:unix:/run/spinifex/vol-123.sock", "Mounted":true,
-		"Error":"", "Retryable":true
-	}`, string(mount))
-
-	unmount, err := json.Marshal(EBSUnMountResponse{
-		Volume: "vol-123", NotFound: true, Reaped: true,
-	})
-	require.NoError(t, err)
-	require.JSONEq(t, `{
-		"Volume":"vol-123", "Mounted":false, "Error":"",
-		"NotFound":true, "Reaped":true
-	}`, string(unmount))
-
-	deleteRequest, err := json.Marshal(EBSDeleteRequest{Volume: "vol-123"})
-	require.NoError(t, err)
-	require.JSONEq(t, `{"Volume":"vol-123"}`, string(deleteRequest))
-
-	deleteResponse, err := json.Marshal(EBSDeleteResponse{Volume: "vol-123", Success: true})
-	require.NoError(t, err)
-	require.JSONEq(t, `{"Volume":"vol-123", "Success":true, "Error":""}`, string(deleteResponse))
-}

docs/PACKAGE_BOUNDARY_MIGRATION.md

@@ -695,7 +695,7 @@ series above; "needs decision" is open.
 | `gateway/{ecs,elbv2}` | — | Done. The action adapters moved to `domains/{ecs,elbv2}/awsapi` in `spinifex#1273` with no API, NATS subject, storage or behaviour change; the root `gateway` dispatch that still imports them is recorded `gateway` (service registration) debt. |
 | `gateway/spx` | — | Done. The gather helpers, storage-status config and image-promote request moved to `operator/admin` with no behaviour change and the package is removed. The `spinifex` Query dispatch and the `/admin/{method}` handlers that call them now live in `operator/admin` as one seam registration. |
 | `gateway/tagging` | — | Done. `domains/tagging` owns the read-only cross-resource `GetResources` aggregation and its API semantics (`0d830e901`, `aedd0b2c4`); AWSGW composes the owner listers, the EC2 tag index and ELBv2 (`f3545e3ea`); the dispatch moved to `domains/tagging/awsapi` and the root `gateway` no longer serves Tagging. The package is removed, and its two allowlist rows and frozen-legacy entry are gone. Owner tag mutation capabilities remain future direction, so the RDS and ECS tag-writing edges above stay open. |
-| `services/viperblockd` | — | Done. The adapter is `providers/ebs/viperblock` (with the read-only `dirty` reader), the shared vocabulary is `contracts/viperblockd/legacy/v1`, and the role is `runtime/roles/viperblockd`. `services/` is empty. |
+| `services/viperblockd` | — | Done. The adapter is `providers/ebs/viperblock` (with the `dirty` package, which owns the dirty-marker bucket and its read-only reader), it serves only the `providers/ebs` contract, and the role is `runtime/roles/viperblockd`. `services/` is empty. |
 | `utils` | — | Done and deleted. `ec2_subject.go` went to `contracts/ec2/v1`, the vpcd protocol to `contracts/network/v1` with the typed client in `domains/network/projection`, and the master-key loader to `domains/ec2/ebs/encryption`. |
 | `vpcd` | — | Done. The IMDS local-file reader and record-space opener moved to `domains/ec2/instance/state` and are injected through `guestmetadata.LocalStateReader`/`RecordLoader` (recorded above), so `vpcd` no longer imports `daemon` directly; its inter-process boundary now exists (`contracts/network/v1`, with `domains/network/subscribers` as the consumer and `domains/network/projection` as the producer client). The package moved unchanged to `runtime/roles/vpcd` (recorded below), and the legacy directory is removed. |
 | `daemon` | `runtime/daemon` (whole-package move, above) | Done. Q-28's ownership and deployment refactor remains deferred debt; the amended ADR-0001 row makes it not a relocation prerequisite. |

docs/containers/eks/README.md

@@ -59,7 +59,7 @@ Authentication uses **EKS access entries** (the API authentication mode) — IAM
 
 - **Access entries are the only auth path.** `authentication_mode` accepts `API` (the default when unset) and `API_AND_CONFIG_MAP`; bare `CONFIG_MAP` is rejected. Spinifex has no `aws-auth` ConfigMap path, so `API_AND_CONFIG_MAP` behaves as `API`: grant access with access entries.
 - **The cluster security group is auto-managed.** As on EKS, Spinifex creates one `eks-cluster-sg-<name>-<id>` in your VPC, tagged `kubernetes.io/cluster/<name>=owned` and `aws:eks:cluster-name`, allowing all traffic from itself. It is attached to the workers and the private-endpoint network interface, and `DescribeCluster` reports it as `clusterSecurityGroupId`. A launch template that names security groups replaces it on the workers, as on AWS. Clusters created before this change keep their `-control-plane-sg` and `-nodegroup-sg` until you recreate them. Deleting a cluster deletes the cluster security group and removes only the rules on your other security groups that reference it. `vpc_config.security_group_ids` is ignored.
-- **Node group deletes are asynchronous.** `DeleteNodegroup` returns at once with the group `DELETING`, and `DescribeNodegroup` answers `ResourceNotFoundException` once the workers and the group's instance profile are gone. Deleting a cluster or creating a group of the same name while a group is `DELETING` returns `ResourceInUseException`, as does a second delete of a `DELETING` group.
+- **Node group deletes are asynchronous.** `DeleteNodegroup` returns at once with the group `DELETING`, and `DescribeNodegroup` answers `ResourceNotFoundException` once the workers and the group's instance profile are gone. Deleting a cluster or creating a group of the same name while a group is `DELETING` returns `ResourceInUseException`. A second delete of a `DELETING` group returns the group as it stands and starts nothing new, as on EKS.
 - **If a node group delete cannot finish,** the group moves to `DELETE_FAILED` after 30 minutes, with a `NODE_TERMINATION_FAILURE` health issue naming the workers that are still running. Run `DeleteNodegroup` again to resume it.
 - **Each node group owns an EKS-managed instance profile.** Spinifex creates it, tags it `spinifex:eks-cluster`, `spinifex:eks-nodegroup` and `spinifex:eks-profile-id`, and puts the node role in it. It is removed with the group, so you can delete the node role afterwards with no extra steps. Spinifex never deletes an instance profile it did not create, or one whose tags, name or role no longer match what it recorded; it leaves the profile and logs a warning with its name.
 - **The worker image is fixed.** Node groups always boot Spinifex's `eks-node` image. `ami_type` is recorded but does not select the image, and the image must be registered on the cluster before you create a node group.

docs/coverage/ec2/intro.md

@@ -20,3 +20,18 @@ The same form is used for a bad `Architecture`, `BootMode` or `Name` length.
 A missing `Name`, `BlockDeviceMappings` or root snapshot answers `MissingParameter` and names what is missing.
 Registering with `--ena-support`, `--no-ena-support` and neither, copying the image, and three refusals were run on a three-node cluster with the AWS CLI.
 Launching an instance from an `EnaSupport` image was not run, so the launch path is unchanged by construction and not by test.
+
+### CreateSnapshot Error Mapping
+
+`CreateSnapshot` drains and snapshots the volume inside the EBS provider.
+A provider that cannot snapshot a volume because it is in use, as the `qemunbd` provider cannot for an attached volume, answers `ErrVolumeInUse`, which becomes `IncorrectState` with the message `The volume 'vol-...' is in use. Detach it before creating a snapshot.`
+The `viperblockd` provider drains a mounted export and snapshots it, so an attached volume succeeds there.
+A provider that cannot answer now, for example because another node holds the volume, answers `ServiceUnavailable` so the caller retries.
+Any other provider failure is `ServerInternal`.
+The snapshot taken for `CreateImage` maps these errors the same way.
+
+### EBS Encryption by Default Follows the Platform
+
+Whether new volumes are encrypted is decided by the platform's encryption key, not by the account.
+`GetEbsEncryptionByDefault` and `EnableEbsEncryptionByDefault` report `true` when the platform has an encryption key configured and `false` otherwise, and `EnableEbsEncryptionByDefault` does not change that.
+`DisableEbsEncryptionByDefault` is refused with `OperationNotPermitted` and the message `EBS encryption by default cannot be disabled: this platform encrypts every volume` while the platform encrypts, and answers `false` where it does not.

docs/terraform-workbooks/demo-app/Dockerfile

@@ -4,7 +4,7 @@
 # distroless base (no shell, no package manager) for a small, production-shaped
 # image suitable for pushing to ECR and pulling onto EKS workers.
 
-FROM golang:1.27.3-alpine@sha256:d200d2dd76b03a99b52651fd1325aff7dcd35422c35c01bac56571b0507f3deb AS build
+FROM golang:1.27.4-alpine@sha256:0a5f7fe796599ff487605987432e7619ae45beeb2fba4c2ce0b907879932aebf AS build
 WORKDIR /src
 COPY go.mod ./
 COPY main.go ./

docs/terraform-workbooks/demo-app/go.mod

@@ -1,3 +1,3 @@
 module github.com/mulgadc/spinifex-demo
 
-go 1.27.3
+go 1.27.4

docs/testing/e2e-coverage.md

@@ -13,6 +13,12 @@ Suites live in `tests/e2e/<name>/` and build to `tests/e2e/_bin/<name>.test`. Ev
 
 The nightly permutation sets are deliberately narrower than the full ones: those cells have a ~35 minute budget and exist to prove every install / network / host-OS combination boots and serves. `eks`, `ecs`, `rds`, `storagefault`, `instancerecovery`, `lbrecovery` and `kvquorum` each get a dedicated cell instead.
 
+**The push E2E runs `single` as three shards** on separate `ci-single` envs (`E2E_SINGLE_SHARDS`, default 3; a `test_run` dispatch runs it whole).
+Each leg lists `single.test -test.list` itself and `.github/scripts/e2e-shard.sh` assigns the names longest-first by the seconds in `.github/e2e-weights/single.txt`, so every leg computes the same partition and a test missing from the weights still lands in exactly one shard.
+A leg passes the other shards' names to `-test.skip` and never names a test in `-test.run`, so opt-in tests such as `TestRebootLoop` stay skipped as in a whole-suite run.
+A failed or empty listing fails the leg, and any failed leg fails the `e2e` status.
+The rest of the push base set (`iam`, `cert`, `partialblock`) runs whole on a `ci-pseudo` env.
+
 **`multinode` carries the KV replication sweep as well**, which is why the thing `kvquorum` proves is still checked in every multi-node permutation cell even though the suite itself is not. The read-only half costs a NATS connection and a stream listing; only the node-loss half needs a cell of its own.
 
 ## The suites
@@ -28,7 +34,7 @@ The largest suite, 27 top-level tests. Notable entry points:
 - `TestWANEgressSMTPBlock` — egress port blocking.
 - `TestInstanceMetadata` — IMDS from inside a guest.
 - **`TestIMDSDatapathSurvivesVswitchdRestart`** restarts `ovs-vswitchd` under a guest with an Elastic IP, once through systemd and once with SIGKILL. After each, the guest must answer over its EIP and fetch its instance ID through IMDSv2 within 60 seconds, and `br-imds` must hold all 7 of the guest's own flows. It is sequential because it takes the whole node's datapath down.
-- `TestStopStart`, `TestEarlyRebootLiveness`, `TestGuestChurnDurability`, `TestSnapshotLifecycle`, `TestSnapshotBackedLaunch`, `TestCreateImage`, `TestVolumeLifecycle`, `TestSpotInstanceLifecycle`, `TestLaunchTemplateBoot`.
+- `TestStopStart`, `TestEarlyRebootLiveness`, `TestGuestChurnDurability`, `TestAttachedENISurvivesTerminate` (an interface attached with `AttachNetworkInterface` defaults to `DeleteOnTermination=false`, toggles through `ModifyNetworkInterfaceAttribute`, and outlives `TerminateInstances`), `TestSnapshotLifecycle`, `TestSnapshotBackedLaunch`, `TestCreateImage`, `TestVolumeLifecycle`, `TestSpotInstanceLifecycle`, `TestLaunchTemplateBoot`.
 - **`TestRebootLoop`** is opt-in and skipped in a whole-suite run. Name it in `-test.run`, or set `SPINIFEX_REBOOT_LOOP_ROUNDS` (default 10), and it reboots one fresh guest through the API that many times and then once from inside, requiring a new boot ID every time.
 - `TestNegativeErrorPaths` — the `8a`–`8t` AWS error-shape cases (malformed AMI ID, invalid instance type, volume in use, detach-root forbidden, unknown or malformed instance IDs on the lifecycle verbs, and so on).
 - **`TestKVReplication`** — the single-node half of the KV replication policy, and it is not the multi-node assertion with a smaller number in it. On one node one replica per bucket is the correct answer rather than the defect, so what it proves is the opposite: that nothing declared a replica count this deployment cannot place, that `spx admin kv replicas` exits 0 and reports `WANT 1`, and that `--repair` run twice against a cluster with nothing to repair changes no bucket's replica count and leaves every bucket serving. The multi-node suites cannot cover any of that, because on three nodes the unplaceable count is the right one. It lives here rather than in its own suite so it runs in every single-node nightly cell, across every host and network permutation.
@@ -125,13 +131,16 @@ Internal and internet-facing ALB and NLB, HTTPS listeners, listener rules, `Modi
 Unhealthy-target routing: an NLB with an HTTP health check on a TCP target group stops sending traffic to a target that fails it while still accepting TCP, sends traffic again once it recovers, and fails open when every target is unhealthy.
 An ALB whose targets all fail their health check keeps serving from them instead of answering 503.
 
+`Internal_NLB_DeletionProtection` enables `deletion_protection.enabled` on an internal NLB.
+`DeleteLoadBalancer` must then fail with `OperationNotPermitted` and AWS's message, leave the listener and ENI in place and keep serving traffic, and succeed once protection is turned off.
+
 `TestInternalALBPrivateSubnet` launches an internal ALB in a private subnet of a non-default VPC with no internet gateway and no public route.
 The load balancer must reach active with a healthy target, which needs the owner stamp on its ENI to resolve, and a client inside the VPC must be served by it on every request.
 The client reports on its serial console because the runner has no route into the VPC.
 
 ### `eks`, `ecs`, `ecr` — container services
 
-`eks`: cluster create/delete, access entries, addon delivery, `GetToken`, kubeconfig artifacts, the EBS CSI volume path, an IRSA web-identity exchange under the stock AWS trust policy (`StringEquals` on the issuer host-path `:sub`/`:aud`, a mismatched `:sub` denied with `AccessDenied`, a `StringLike` `:sub` granted), and an IRSA pod whose spec names only a ServiceAccount with the role-arn annotation, where the pod-identity webhook injects the token volume, `AWS_*` environment, STS endpoint and CA bundle and the pod assumes the role, while a pod of an unannotated ServiceAccount is left untouched. The unmodified aws-load-balancer-controller chart 1.11.0 installs with the values an AWS user passes plus the gateway endpoint override, the Shield and WAF switches and the single-subnet feature gate, and reconciles an Ingress into an active ALB. Worker IMDS is covered too: a managed nodegroup worker reports IMDSv2 required with hop limit 2, a pod with no IRSA resolves to the node role through IMDS, and a nodegroup built from a launch template takes the template's metadata options. The private-endpoint NIC of the cluster's endpoint NLB is disabled through QMP and every worker must be Ready again within ten minutes while the cluster stays `ACTIVE` and the NLB keeps its guest instance. Control-plane isolation is covered as well: no control-plane host registers as a Node, a DaemonSet tolerating every taint runs on workers only, metrics-server is Ready, and logs work for every kube-system pod. `ecs`: clusters, container instances, the three network modes (`awsvpc`, `bridge`, `host`), task-role credentials fetched from the container credentials endpoint, a service behind an ELB, and ENI release on terminate. `ecr`: control-plane CRUD plus a real OCI data plane — docker login/build/push/pull round trips, multi-layer images, and raw blob HTTP.
+`eks`: cluster create/delete, access entries, addon delivery, `GetToken`, kubeconfig artifacts, the EBS CSI volume path, an IRSA web-identity exchange under the stock AWS trust policy (`StringEquals` on the issuer host-path `:sub`/`:aud`, a mismatched `:sub` denied with `AccessDenied`, a `StringLike` `:sub` granted), and an IRSA pod whose spec names only a ServiceAccount with the role-arn annotation, where the pod-identity webhook injects the token volume, `AWS_*` environment, STS endpoint and CA bundle and the pod assumes the role, while a pod of an unannotated ServiceAccount is left untouched. The unmodified aws-load-balancer-controller chart 1.11.0 installs with the values an AWS user passes plus the gateway endpoint override, the Shield and WAF switches and the single-subnet feature gate, and reconciles an Ingress into an active ALB. Worker IMDS is covered too: a managed nodegroup worker reports IMDSv2 required with hop limit 2, a pod with no IRSA resolves to the node role through IMDS, and a nodegroup built from a launch template takes the template's metadata options. The private-endpoint NIC of the cluster's endpoint NLB is disabled through QMP and every worker must be Ready again within ten minutes while the cluster stays `ACTIVE` and the NLB keeps its guest instance. Control-plane isolation is covered as well: no control-plane host registers as a Node, a DaemonSet tolerating every taint runs on workers only, metrics-server is Ready, and logs work for every kube-system pod. `ecs`: clusters, container instances, the three network modes (`awsvpc`, `bridge`, `host`), task-role credentials fetched from the container credentials endpoint, a service behind an ELB, an unforced `DeleteService` of that service refused while it is scaled above 0 with its task left running, and ENI release on terminate. `ecr`: control-plane CRUD plus a real OCI data plane — docker login/build/push/pull round trips, multi-layer images, and raw blob HTTP.
 
 ### `rds` — managed databases
 
@@ -140,6 +149,7 @@ The create/describe/connect path against a live cluster: a DB VM that actually b
 ### Storage and durability
 
 - **`storagefault`** — what a guest does when its storage backend stops answering, and what stop/start does when the seal failed. **The claim is about corruption, not availability.** SIGSTOPs a cluster-wide service, so it always runs last and in its own cell. Three-node, because the RS(2,1) shard floor and cross-node takeover after a failed seal do not exist on one node.
+- **No suite reaches a `pending` snapshot.** Every e2e cell runs the default `viperblockd` provider, which finishes a snapshot before `CreateSnapshot` answers, so `single`'s `TestSnapshotLifecycle`, `TestSnapshotBackedLaunch` and `TestCreateImage`, and `rds`'s `TestSnapshotRestore` and `TestDelete`, run the create that writes the snapshot document `pending` before the provider call and settle it `completed` within the request, and the pending-snapshot reaper finds nothing to settle. The `pending` paths are unit-tested against the memory provider answering `pending`: the reaper settling snapshots and AMIs and carrying out deferred deletes, the request's rollback and its ambiguous calls, and `DeleteSnapshot` of a pending snapshot in `domains/ec2/snapshot`; `CreateImage` and the readiness gates in `domains/ec2/image`, `domains/ec2/instance` and `domains/ec2/volume`; DB snapshots and the final snapshot waiting in `creating` in `domains/rds/instance`; and the import's wait in `operator/admin`. The contract's `pending` answer, its filtered `ListSnapshots` and the `state` it reports are in the conformance suite (`providers/ebs/conformance`), which every provider runs.
 - **`partialblock`** — concurrent sub-block writes to the same block surviving a cold reread through NBD from a real guest.
 - **`storagegrowth`** — volume growth and image pull. Requires `external_mode=pool`; skips on a nat env, which has no EIP and therefore no guest SSH.
 - **`reboot`** — a slow guest waited for and keeping what it wrote, and a wedged guest forced off and relaunched once its budget expires. Proves a real reboot by requiring a *new* boot ID.

go.mod

@@ -1,30 +1,31 @@
 module github.com/mulgadc/spinifex
 
-go 1.27.3
+go 1.27.4
 
 require (
 	github.com/aws/aws-sdk-go v1.55.8
-	github.com/aws/aws-sdk-go-v2 v1.47.5
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25
-	github.com/aws/aws-sdk-go-v2/config v1.33.12
-	github.com/aws/aws-sdk-go-v2/credentials v1.20.12
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5
-	github.com/aws/aws-sdk-go-v2/service/lambda v1.113.4
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6
-	github.com/aws/aws-sdk-go-v2/service/sqs v1.52.7
-	github.com/aws/smithy-go v1.28.7
+	github.com/aws/aws-sdk-go-v2 v1.47.7
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.26
+	github.com/aws/aws-sdk-go-v2/config v1.33.15
+	github.com/aws/aws-sdk-go-v2/credentials v1.20.15
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.7
+	github.com/aws/aws-sdk-go-v2/service/lambda v1.114.2
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.114.8
+	github.com/aws/aws-sdk-go-v2/service/sqs v1.52.10
+	github.com/aws/smithy-go v1.28.8
 	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/charmbracelet/bubbles v1.0.0
 	github.com/charmbracelet/bubbletea v1.3.10
 	github.com/charmbracelet/lipgloss v1.1.0
 	github.com/containerd/containerd/v2 v2.4.2
 	github.com/distribution/reference v0.6.0
+	github.com/fsnotify/fsnotify v1.10.1
 	github.com/go-chi/chi/v5 v5.5.0
 	github.com/golang-jwt/jwt/v5 v5.3.1
 	github.com/insomniacslk/dhcp v0.0.0-20260719225207-c76316d4aa82
-	github.com/jackc/pgx/v5 v5.11.0
+	github.com/jackc/pgx/v5 v5.12.0
 	github.com/klauspost/compress v1.20.3
-	github.com/klauspost/cpuid/v2 v2.4.1
+	github.com/klauspost/cpuid/v2 v2.4.2
 	github.com/miekg/dns v1.1.73
 	github.com/minio/highwayhash v1.0.4
 	github.com/mulgadc/bluebottle v1.24.0
@@ -33,14 +34,14 @@ require (
 	github.com/mulgadc/viperblock v1.24.0
 	github.com/nats-io/nats-server/v2 v2.15.2
 	github.com/nats-io/nats.go v1.55.0
-	github.com/nats-io/nkeys v0.4.16
+	github.com/nats-io/nkeys v0.4.17
 	github.com/opencontainers/runtime-spec v1.3.0
-	github.com/oracle/oci-go-sdk/v65 v65.128.0
-	github.com/ovn-kubernetes/libovsdb v0.9.0
+	github.com/oracle/oci-go-sdk/v65 v65.129.0
+	github.com/ovn-kubernetes/libovsdb v0.9.1
 	github.com/pelletier/go-toml/v2 v2.4.3
 	github.com/pterm/pterm v0.12.84
 	github.com/spf13/cobra v1.10.2
-	github.com/spf13/pflag v1.0.10
+	github.com/spf13/pflag v1.1.0
 	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.12.1
 	go.etcd.io/etcd/api/v3 v3.7.3
@@ -55,7 +56,7 @@ require (
 	go.uber.org/goleak v1.3.1-0.20260602155401-5f791e3ef83b
 	go.yaml.in/yaml/v3 v3.0.6
 	golang.org/x/crypto v0.58.0
-	golang.org/x/sync v0.24.0
+	golang.org/x/sync v0.25.0
 	golang.org/x/sys v0.49.0
 	golang.org/x/term v0.47.0
 	golang.org/x/time v0.16.0
@@ -72,17 +73,17 @@ require (
 	github.com/Microsoft/hcsshim v0.15.0-rc.4 // indirect
 	github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect
 	github.com/atotto/clipboard v0.1.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/signin v1.12.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.39.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.24 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.11 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.12.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.39.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.51.8 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
@@ -119,7 +120,6 @@ require (
 	github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
 	github.com/fatih/color v1.19.0 // indirect
 	github.com/felixge/httpsnoop v1.1.0 // indirect
-	github.com/fsnotify/fsnotify v1.10.1 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.15 // indirect
 	github.com/go-logr/logr v1.4.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
@@ -144,7 +144,7 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
-	github.com/jackc/puddle/v2 v2.2.2 // indirect
+	github.com/jackc/puddle/v2 v2.2.3 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/native v1.1.0 // indirect
 	github.com/jstemmer/go-junit-report/v2 v2.1.0 // indirect

go.sum

@@ -22,46 +22,46 @@ github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z
 github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.47.5 h1:5yA+A+BiXbDq1ZLbYwPViwhEGDMdR0MEhL8Jh2LnLS4=
-github.com/aws/aws-sdk-go-v2 v1.47.5/go.mod h1:iNIkogrRvFwm+1OQIUOP2lPBtlXZboOIUWiCa+pHEMw=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 h1:MW0Iql1SOTgQPj8mCxrU/eH/K/AIDWi08ItOsXaWRLU=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25/go.mod h1:+ucVYmHYB3JqVueY0E1RHahdXAnKQbIHqybSbvOUbOM=
-github.com/aws/aws-sdk-go-v2/config v1.33.12 h1:IV1YARaXahwCqUIyP8IlWsP85Lil+5ozTozDQXPumRk=
-github.com/aws/aws-sdk-go-v2/config v1.33.12/go.mod h1:h51R06bzMVwNBRZOX5ylHhNDnkB/azZeq/LvxM6q2pQ=
-github.com/aws/aws-sdk-go-v2/credentials v1.20.12 h1:6P7Q7yH/GIwHg6MLKv3ySsBqaCoE1z+C0OTzYMenhEY=
-github.com/aws/aws-sdk-go-v2/credentials v1.20.12/go.mod h1:Kw/SaoW/wc7IqJErqXJIzKo8b7Aelxfbuc6XlbrRnsc=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 h1:iXjy2SFhGYNB1jWZBhQqq8HbhsbPF23w+/zbMvPXsTg=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5/go.mod h1:k0RUiBnyXU0AXRJ9tJmY2jQke6XUIX2PNVnWtvgaPtc=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 h1:gzIWc3DTPfPbWKrQnmL6BV6WD7NPD1htxzgdk/bQ/Z4=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8/go.mod h1:vvUA5cDAZWGo4R14FZPZfXfJCmgsafOWEq78Kg0D+ug=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 h1:PWv+vG8aCLS8qd+HPdYhK7Vees90aKHWFFQ1AGYD/8E=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8/go.mod h1:xxPZEpflYHJA12366ZGyq66z9dHPCoeCIu99Jck+MPw=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 h1:WOp5De9iWdEHDPh2Kb0tBP3vGBm/dVAuRGh4a4tqduo=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8/go.mod h1:z8W8YgfhbWl62XqcTtK+CxPrE+iHGRVUOtrrbIOp7ZY=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 h1:MCIR6iZaCKO4UE0hUvoWWLI6JVjppopV64JfAJaMjng=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23/go.mod h1:pphWFS+z0MmIPivj8c/6bATd1Z+Av7Ust75DPdEFsPA=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 h1:GACgECI422Xt5n4Pojr3O0aocPv8lfH4PJe2Aon5Wv8=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9/go.mod h1:lPfnRkHpn1qWZJLbVEw10rIqSuCee1pNQTyGkUjAYh8=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 h1:+PMpWlJYOn2tm7sm6OZ/ZLd4jStl17T7wjNapd1Rczs=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8/go.mod h1:cnG3xZLGQL4c/bt8pjdzlFtZsW7epBBuNS04a5xMjdo=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 h1:RS1Wt+K0r4X0ci9Vf4JYTUfHzsQihYA2LgGJeEneKpk=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8/go.mod h1:adMnTy5kyoUW86jhXnGSVc/mvMQI4GZd1qYWMeaS8M8=
-github.com/aws/aws-sdk-go-v2/service/lambda v1.113.4 h1:wvAdq1sNfwuedO1BX8GjY+oZWzoyeD4hhdjJI+4EPnI=
-github.com/aws/aws-sdk-go-v2/service/lambda v1.113.4/go.mod h1:HgkYN4d72Fawg3MAJQepmNnROFoN11XYu14hgFfChTY=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6 h1:to7+Vr3rLkpYRwVKJ5QRriI0eE306vrVN/6TnVHgxvY=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6/go.mod h1:Jc89ASa2k7HMAskj3bSnt8bf+k+p2IApzkoXnEjkgUw=
-github.com/aws/aws-sdk-go-v2/service/signin v1.12.0 h1:3vu/eUqh0rjh7Zy4N4YKcpZX4Aud+zinB3VsLhhlQFw=
-github.com/aws/aws-sdk-go-v2/service/signin v1.12.0/go.mod h1:wxAk0D2oVBSjuXEIzvcKxTZfM90zticdAbh/VD42L0Y=
-github.com/aws/aws-sdk-go-v2/service/sqs v1.52.7 h1:Eip+mZ0wFZY/10e5w89lWhF09Pa8tbbGwxePP3T2iMc=
-github.com/aws/aws-sdk-go-v2/service/sqs v1.52.7/go.mod h1:bXp38eBNyvSevFIao60mYcC3pQIAkPYWOddhFVv9z8E=
-github.com/aws/aws-sdk-go-v2/service/sso v1.39.0 h1:kIEpD5dUry7VET3Ybl7zj7PPHRB4lb52G9iTz+a1Tqk=
-github.com/aws/aws-sdk-go-v2/service/sso v1.39.0/go.mod h1:UGsEYgWz4Gi+46vDQ5zkFLqUgJziyOE2SXjZjgmfmHc=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0 h1:WfFev+HgTodggcNon5HieiLxxwWwb8Br8R2+nDkixvk=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0/go.mod h1:gJSdIY6YUzDvqQLcMtPVZgBRHrFwzHryvpE8Igzbxps=
-github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 h1:BVjKrtRUtSYKztBCCfE1oAB+d3ViKcW2cKZfserOqMQ=
-github.com/aws/aws-sdk-go-v2/service/sts v1.51.6/go.mod h1:eSukFLeLkPMm+1OxDR+F0joauZdyBkF47Ow7MqjZiJw=
-github.com/aws/smithy-go v1.28.7 h1:h/n1SHrwVtaDYLz8mR1/Kq4Xh4RaFbFW7bOYckg9mfE=
-github.com/aws/smithy-go v1.28.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
+github.com/aws/aws-sdk-go-v2 v1.47.7 h1:jBmtzRtGWVAzbfc4RBWo1dTte0ZZH8p9U42YJIw2+Bw=
+github.com/aws/aws-sdk-go-v2 v1.47.7/go.mod h1:I8lFSH4MGrZiw2m7miCrRYHsgpC1+HW8uiVGOiPYPcw=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.26 h1:SJNA4XaM8Z09adeTHf/vME93jzl44lJV70pSawIBXac=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.26/go.mod h1:l0ME5pd2QytH8UZzETcrt90x9Bm+3TyBVeWfVEcZ82c=
+github.com/aws/aws-sdk-go-v2/config v1.33.15 h1:eSO7lvznhY5nMN1K9xK+pPTwNNH0chVmCIzH3C5as+E=
+github.com/aws/aws-sdk-go-v2/config v1.33.15/go.mod h1:QLBxqEJFcgbrsw2dHaWMDW6DcIFWkHplXJMZGB/s2Ew=
+github.com/aws/aws-sdk-go-v2/credentials v1.20.15 h1:LUFG3364w8wKBhGxGQ2XUxCCs+yOUrSg0dWUtBJskp8=
+github.com/aws/aws-sdk-go-v2/credentials v1.20.15/go.mod h1:Q+tKQpy3sU1MjBqXzhZauN7ffcNRF9p7Rvw293sYwE4=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.7 h1:5GGOwjXU738VrsBT0EDFtP5JtLPmJQILp8RS6P+KqLM=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.7/go.mod h1:jY/VeijtHNuTiy5XI9HjpLiamtaaLw+KInZULfookJ8=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.10 h1:DKtPO+uoekvZX1tIFGVlWypYGSYkHPb7rliUI1wZtEw=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.10/go.mod h1:uAi0RDdZZz8IRpXERptWALeCgQG9Ye/HiMLxkxlt8IU=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.10 h1:r8p95YELmfmuV9VvAaFcLesVBZduCCCccxpzQKkNThw=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.10/go.mod h1:W3M52Mqai5TjgzyL/dEG+UwE+Xz9PykyjXelvbAzWIA=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.10 h1:iytL6kFchhbCWWEfLzMGq1+yRecF7+BOXnp70J9zNRM=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.10/go.mod h1:OaBcfeWMHPSyFqiHL/KN1zVIricTvYDNiO4XhR781eM=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.24 h1:jGzHjc2jDx08Rji1JTBtMxuXUDG7muTEw+ExYXqFzPg=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.24/go.mod h1:uJt75pK2cEyY80F5C+DK9YkCkJWsMkJViHxLYZRHl8Y=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.11 h1:+YhA5h9QMjVsOHfkfY/iVU7FfbT25LcGttfVzzed6v8=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.11/go.mod h1:Xn22cWokj55agZo3I0w1JFQUiHhA3qLmHi/F5t1COi0=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.10 h1:3nCD0vZOTTMypcaEEJW0rAWqEgUXVzfmNn3dDI1IQ7I=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.10/go.mod h1:KJbydA7vvlrQSceENaM/C//yI18Rt36x//hG0Bw3uW0=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.10 h1:bCz6PRKhO7ErM3F0XqLl8pbQq+r0Fra6R8PpfPnjgbo=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.10/go.mod h1:N8UvNaHipKSUnUFR07OpGdjU8B5aS/XBxgrcKURb2DQ=
+github.com/aws/aws-sdk-go-v2/service/lambda v1.114.2 h1:SbfR8t3crViVjf8bCh9ohXwQYDbPJV/6P6CE3d9iKXc=
+github.com/aws/aws-sdk-go-v2/service/lambda v1.114.2/go.mod h1:is4sHkblHDcfDUxxZVrDR9hSzf5I4id6qgX6hH0tUDA=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.114.8 h1:0g/xXBWQEyR6Mcxo1ukBYqzeRX0bQ/ZdXrtKkIo+yT0=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.114.8/go.mod h1:g6EsKHPrO5GpWYKwBfenLLtjuoPRE+WDIlLPQOIixLI=
+github.com/aws/aws-sdk-go-v2/service/signin v1.12.3 h1:8cQxxirJ0d0tvmS89hYUpnVArXIKlzwRyiaSfdaokvQ=
+github.com/aws/aws-sdk-go-v2/service/signin v1.12.3/go.mod h1:cou3YaR7daAzApgGTG9il1c3iCh/bs72w6p4WFvJnaY=
+github.com/aws/aws-sdk-go-v2/service/sqs v1.52.10 h1:ojdbdSOuwvVqcY+yfFO7Kz9Qow7brLZCALVBhorjUv8=
+github.com/aws/aws-sdk-go-v2/service/sqs v1.52.10/go.mod h1:zPf8oFenYL6wvZ8izCh+fzFsSwpNNIBQP965awjBaXQ=
+github.com/aws/aws-sdk-go-v2/service/sso v1.39.3 h1:r5DHs3x9vC1p3VQh+V9EFfHIN5QmWDdS0yKITZM2kBI=
+github.com/aws/aws-sdk-go-v2/service/sso v1.39.3/go.mod h1:u/xiahjixIqyBFiYNeQkmuimZycycBUJEomieSVsrXk=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.3 h1:ENjUto22h2MiHmv8WehZ2874SSCh5knHKsNxeyjzKpY=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.3/go.mod h1:jmVZvJzjoaStVpr4Nd11mMYCnyrq+sWT6UiDXVnHyco=
+github.com/aws/aws-sdk-go-v2/service/sts v1.51.8 h1:lervRo8V4a9tticNT+nPjqDq9olQPDmgqYczjvJbNio=
+github.com/aws/aws-sdk-go-v2/service/sts v1.51.8/go.mod h1:NBCcbmgAjvOsYwA6CPgMmipPUxsEep1JxKb3RWBr9rA=
+github.com/aws/smithy-go v1.28.8 h1:aEmdAq7gVCoGCnvf1NnELlu3I6D/nhqrC321FyFiQkQ=
+github.com/aws/smithy-go v1.28.8/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -234,10 +234,10 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
 github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
 github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
-github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg=
-github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
-github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
-github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
+github.com/jackc/pgx/v5 v5.12.0 h1:hWwax8YiGT0YiN50U1oIDn5kgR7KC2tG3rmavwpPkxY=
+github.com/jackc/pgx/v5 v5.12.0/go.mod h1:u2Fi6no03HCSBYE+ZVdLxbwdu3xyi2vU+K/YNJioSds=
+github.com/jackc/puddle/v2 v2.2.3 h1:f8HoREISb3l5eO9coPgK7U3hKLhWGokOFVcThl5jrao=
+github.com/jackc/puddle/v2 v2.2.3/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
@@ -249,8 +249,8 @@ github.com/jstemmer/go-junit-report/v2 v2.1.0 h1:X3+hPYlSczH9IMIpSC9CQSZA0L+BipY
 github.com/jstemmer/go-junit-report/v2 v2.1.0/go.mod h1:mgHVr7VUo5Tn8OLVr1cKnLuEy0M92wdRntM99h7RkgQ=
 github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4=
 github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
-github.com/klauspost/cpuid/v2 v2.4.1 h1:I9KxadAkvo+QjtcEE2f2tyfEvmIla/T0m+14PrZNl0U=
-github.com/klauspost/cpuid/v2 v2.4.1/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
+github.com/klauspost/cpuid/v2 v2.4.2 h1:xRw7IZEsAgVCp6trS/DgWbTAqpixhmywgVgBjYGg1bQ=
+github.com/klauspost/cpuid/v2 v2.4.2/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
 github.com/klauspost/reedsolomon v1.14.2 h1:SafJYwpBBQBI6amHUygcjxZjXeN2HpiENHQDwuPWCCQ=
 github.com/klauspost/reedsolomon v1.14.2/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@@ -317,8 +317,8 @@ github.com/nats-io/nats-server/v2 v2.15.2 h1:yLsgt2kslgymvxvjFq1WGWck5hkat2YYfbX
 github.com/nats-io/nats-server/v2 v2.15.2/go.mod h1:VXHCZgDglV5wO2B8ZE58yG2HzqxQxYUbQqibUh5Xcr8=
 github.com/nats-io/nats.go v1.55.0 h1:Ng8QKHdvdHIWoaiq0+d/aH1BWkxvcJxWJ0Y/9BPKx3M=
 github.com/nats-io/nats.go v1.55.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
-github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
-github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
+github.com/nats-io/nkeys v0.4.17 h1:JAJrlDhOyQqOt0a1eg4cbIZ1Kc7+vmkCigAltpwQpVA=
+github.com/nats-io/nkeys v0.4.17/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
@@ -327,10 +327,10 @@ github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJw
 github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
 github.com/opencontainers/runtime-spec v1.3.0 h1:YZupQUdctfhpZy3TM39nN9Ika5CBWT5diQ8ibYCRkxg=
 github.com/opencontainers/runtime-spec v1.3.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/oracle/oci-go-sdk/v65 v65.128.0 h1:QWh0D4VsQ+94XMFy4sapwHVuW5+VOF4dom+h8/cyJsw=
-github.com/oracle/oci-go-sdk/v65 v65.128.0/go.mod h1:87yiCsoPSaOZkpt5KIEEThoSirgT8WolahE3I5/qz78=
-github.com/ovn-kubernetes/libovsdb v0.9.0 h1:WNQM8oCmh793mKa7PIIsF5JYoCa91lgXxG1nEEDStFU=
-github.com/ovn-kubernetes/libovsdb v0.9.0/go.mod h1:mbCZzSSa9FQ0YEFgeO2rXW/OYoMh6vqNrPGpsS2XYAo=
+github.com/oracle/oci-go-sdk/v65 v65.129.0 h1:keTL99Tc9APm7Cm3QQLtiHZ3KU9ZRkiXJzzgjzjpKRk=
+github.com/oracle/oci-go-sdk/v65 v65.129.0/go.mod h1:87yiCsoPSaOZkpt5KIEEThoSirgT8WolahE3I5/qz78=
+github.com/ovn-kubernetes/libovsdb v0.9.1 h1:+ltjpC2PEObBWqg2UuG83fH+qdnTuAibXqGrWdUJ1Iw=
+github.com/ovn-kubernetes/libovsdb v0.9.1/go.mod h1:mbCZzSSa9FQ0YEFgeO2rXW/OYoMh6vqNrPGpsS2XYAo=
 github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY=
 github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
 github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
@@ -375,8 +375,8 @@ github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qq
 github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
 github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
 github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
-github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/pflag v1.1.0 h1:6W6MbVUrnMb8PEhjkqwsVLY7ryuzRfFHF4mQd7L94ag=
+github.com/spf13/pflag v1.1.0/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
 github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -485,8 +485,8 @@ golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
-golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
+golang.org/x/sync v0.25.0 h1:dJnifr0nMV8/SBnBvdo4eSGGnrc3O4FFx6IqvtLcmRs=
+golang.org/x/sync v0.25.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

scripts/build-microvm-image.sh

@@ -321,12 +321,19 @@ echo "[build-microvm-image] building initramfs: $INITRAMFS_OUT"
 # Create the static device nodes and pack the archive inside a single fakeroot
 # session: mknod/cpio see faked device entries and write real char-device
 # records into the cpio, while the host filesystem never gets an actual node.
-fakeroot sh -c '
+# Pack to a .partial and list it before renaming, so a pack cut short (fakeroot
+# losing its daemon, a full disk) fails here and never leaves a stale archive.
+fakeroot bash -c '
+    set -euo pipefail
     cd "$1" || exit 1
     mknod -m 600 dev/console c 5 1
     mknod -m 666 dev/null    c 1 3
     find . | cpio --quiet -o -H newc | gzip -9 > "$2"
-' _ "$CHROOT_DIR" "$INITRAMFS_OUT"
+' _ "$CHROOT_DIR" "$INITRAMFS_OUT.partial" \
+    && gzip -t "$INITRAMFS_OUT.partial" \
+    && zcat "$INITRAMFS_OUT.partial" | cpio -t --quiet >/dev/null \
+    || { rm -f "$INITRAMFS_OUT.partial" "$INITRAMFS_OUT"; echo "ERROR: initramfs $INITRAMFS_OUT is incomplete or unreadable" >&2; exit 1; }
+mv -f "$INITRAMFS_OUT.partial" "$INITRAMFS_OUT"
 
 # --- Log artifact sizes ---
 echo ""

scripts/images/eks-node/k3s-agent.initd

@@ -11,13 +11,16 @@
 description="K3s Kubernetes agent (Mulga EKS worker)"
 command="/usr/local/bin/k3s"
 command_args="agent"
-command_background="yes"
 pidfile="/run/k3s-agent.pid"
+
+# Supervise the agent as k3s.initd supervises the server: k3s exits when flannel
+# cannot reach the API, and without a supervisor the node then stays NotReady.
+supervisor="supervise-daemon"
+respawn_delay=5
+respawn_max=0
 output_log="/var/log/k3s-agent.log"
 error_log="/var/log/k3s-agent.log"
 
-start_stop_daemon_args="--wait 5000"
-
 depend() {
     need net cgroups
     after firewall

scripts/images/eks-node/mulga-eks-log-forward.sh

@@ -42,7 +42,8 @@ select_lines() {
         line = $0
         gsub(/\r/, "", line)
         keep = 0
-        if (line ~ /level=(warning|error)/) keep = 1
+        if (line ~ /level=(warning|error|fatal)/) keep = 1
+        else if (line ~ /^(panic|fatal error):/) keep = 1
         else if (line ~ /^[WE][0-9][0-9][0-9][0-9] /) keep = 1
         else if (line ~ /NodeNotReady|PLEG|[Ll]ease|[Nn]ode status/) keep = 1
         if (!keep) next

scripts/images/eks-node/mulga-eks-log-forward_test.sh

@@ -60,6 +60,16 @@ I1103 03:00:03.123456    1234 kubelet.go:1000] SyncLoop (UPDATE, "api")
 EOF
 check "filter: info and debug lines are dropped" "0" "$(forwarded)"
 
+# --- filter: the lines an agent exit leaves behind must pass, redacted ---
+forward filter_exit <<'EOF'
+time="2026-11-03T03:00:08Z" level=fatal msg="flannel exited: failed to register flannel network: token K1009f3c2d4e5::server:s3cr3tExit"
+panic: runtime error: invalid memory address or nil pointer dereference
+fatal error: concurrent map writes
+goroutine 1 [running]:
+EOF
+check "filter: fatal, panic and fatal error lines pass" "3" "$(forwarded)"
+check "filter: a fatal line is still redacted" "0" "$(grep -c 's3cr3tExit' "${OUT}" || true)"
+
 # --- redaction: one case per secret shape ---
 redact() { # redact <name> <line> <secret-that-must-not-appear>
     forward "redact_$1" <<EOF

scripts/lib/microvm-image.sh

@@ -382,10 +382,17 @@ microvm_pack_initramfs() {
     # Create the static device nodes and pack the archive inside a single fakeroot
     # session: mknod/cpio see faked device entries and write real char-device
     # records into the cpio, while the host filesystem never gets an actual node.
-    fakeroot sh -c '
+    # Pack to a .partial and list it before renaming, so a pack cut short fails
+    # here and never leaves a stale archive.
+    fakeroot bash -c '
+        set -euo pipefail
         cd "$1" || exit 1
         mknod -m 600 dev/console c 5 1
         mknod -m 666 dev/null    c 1 3
         find . | cpio --quiet -o -H newc | gzip -9 > "$2"
-    ' _ "$CHROOT_DIR" "$INITRAMFS_OUT"
+    ' _ "$CHROOT_DIR" "$INITRAMFS_OUT.partial" \
+        && gzip -t "$INITRAMFS_OUT.partial" \
+        && zcat "$INITRAMFS_OUT.partial" | cpio -t --quiet >/dev/null \
+        || { rm -f "$INITRAMFS_OUT.partial" "$INITRAMFS_OUT"; echo "ERROR: initramfs $INITRAMFS_OUT is incomplete or unreadable" >&2; exit 1; }
+    mv -f "$INITRAMFS_OUT.partial" "$INITRAMFS_OUT"
 }

scripts/lib/microvm-image_test.sh

@@ -0,0 +1,77 @@
+#!/bin/sh
+# microvm-image_test.sh — microvm_pack_initramfs publishes only an archive that
+# lists completely, removes a rejected one, and succeeds again on a retry.
+#
+# The real pack step runs against a small chroot; fakeroot and mknod are
+# replaced so it needs no root, and cpio is wrapped to simulate a cut pack.
+set -eu
+
+SCRIPTS_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT
+
+fails=0
+pass() { echo "  ok   - $1"; }
+fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
+
+echo "-- microvm_pack_initramfs"
+
+if ! command -v cpio >/dev/null 2>&1 || ! command -v bash >/dev/null 2>&1; then
+    echo "  skip - cpio or bash not installed"
+    exit 0
+fi
+REAL_CPIO=$(command -v cpio)
+
+mkdir -p "$TMP/bin" "$TMP/chroot/dev" "$TMP/chroot/etc" "$TMP/stage"
+printf '#!/bin/sh\nexec "$@"\n' > "$TMP/bin/fakeroot"
+printf '#!/bin/sh\nexit 0\n' > "$TMP/bin/mknod"
+chmod +x "$TMP/bin/fakeroot" "$TMP/bin/mknod"
+# Large enough that a cut at 64 KiB loses whole entries and the trailer.
+head -c 400000 /dev/urandom > "$TMP/chroot/etc/blob"
+printf 'init\n' > "$TMP/chroot/init"
+
+# pack <cpio mode>: runs the real step; mode "cut" truncates the archive stream
+# and exits 0, mode "fail" writes it whole and exits 1, mode "ok" is untouched.
+pack() {
+    cat > "$TMP/bin/cpio" <<EOF
+#!/bin/bash
+case " \$* " in *" -o "*) ;; *) exec "$REAL_CPIO" "\$@" ;; esac
+case "$1" in
+    cut)  "$REAL_CPIO" "\$@" | head -c 65536; exit 0 ;;
+    fail) "$REAL_CPIO" "\$@"; exit 1 ;;
+    *)    exec "$REAL_CPIO" "\$@" ;;
+esac
+EOF
+    chmod +x "$TMP/bin/cpio"
+    PATH="$TMP/bin:$PATH" bash -c '
+        set -euo pipefail
+        . "$1/lib/microvm-image.sh"
+        TAG="[test]" CHROOT_DIR="$2" STAGE_DIR="$3"
+        microvm_pack_initramfs
+    ' _ "$SCRIPTS_DIR" "$TMP/chroot" "$TMP/stage" > "$TMP/log" 2>&1
+}
+
+OUT="$TMP/stage/initramfs.cpio.gz"
+
+for mode in cut fail; do
+    printf 'stale\n' > "$OUT"
+    if pack "$mode"; then
+        fail "$mode: pack succeeded, want failure"
+    elif [ -e "$OUT" ] || [ -e "$OUT.partial" ]; then
+        fail "$mode: rejected archive left behind"
+    else
+        pass "$mode: pack fails and leaves no archive"
+    fi
+done
+
+if ! pack ok; then
+    fail "retry after failure: $(tr '\n' ' ' < "$TMP/log")"
+elif [ -e "$OUT.partial" ]; then
+    fail "retry after failure: partial file left behind"
+elif ! zcat "$OUT" | "$REAL_CPIO" -t --quiet 2>/dev/null | grep -qx 'etc/blob'; then
+    fail "retry after failure: archive does not list etc/blob"
+else
+    pass "retry after failure publishes a complete archive"
+fi
+
+[ "$fails" -eq 0 ] || exit 1

scripts/lib/sqs-worker-guest_test.sh

@@ -98,7 +98,7 @@ record() { echo "$1 $( . "$LIB"; sqsw_starttime "$1" )" > "$T/run/${2:-ready}.pi
 shutdown_step() {
     (
         SQSW_RUN="$T/run" SQSW_DATA="$T/data" SQSW_CONSOLE="$T/console" SQSW_MOUNTS="$T/mounts"
-        UMOUNT="$TMP/bin/umount" SQSW_STOP_BUDGET_TENTHS=5
+        UMOUNT="$TMP/bin/umount" SQSW_STOP_BUDGET_TENTHS=5 SQSW_KILL_WAIT_TENTHS=5
         # shellcheck source=../../build/microvm/sqs-worker/sqs-worker-lib.sh
         . "$LIB"
         if [ -n "${1:-}" ]; then sqsw_same() { [ "$1" = "$survivor" ]; }; survivor=$1; fi

scripts/setup_firewall_conntrack_test.sh

@@ -62,6 +62,10 @@ cat > "$TMP/flow.py" << 'PY'
 import os, socket, subprocess, sys
 
 srv_ns, cli_ns, script = sys.argv[1], sys.argv[2], sys.argv[3]
+# An optional wait applies only to the last step, the one expected to be lost;
+# a loss there exits 3, and a loss anywhere else exits 1.
+last_wait = float(sys.argv[4]) if len(sys.argv) > 4 else 2
+steps = script.split("@@")
 
 def enter(ns):
     fd = os.open("/run/netns/" + ns, os.O_RDONLY)
@@ -76,7 +80,7 @@ listener.listen(1)
 enter(cli_ns)
 
 cli = srv = None
-for step in script.split("@@"):
+for i, step in enumerate(steps):
     if step == "connect":
         cli = socket.create_connection(("10.99.0.1", 6644), timeout=3)
         srv, _ = listener.accept()
@@ -84,6 +88,8 @@ for step in script.split("@@"):
         srv.settimeout(2)
     elif step in ("s2c", "c2s"):
         tx, rx = (srv, cli) if step == "s2c" else (cli, srv)
+        last = i == len(steps) - 1
+        rx.settimeout(last_wait if last else 2)
         tx.sendall(b"x" * 300)
         got = b""
         try:
@@ -91,17 +97,18 @@ for step in script.split("@@"):
                 got += rx.recv(300 - len(got))
         except socket.timeout:
             print("lost " + step)
-            sys.exit(1)
+            sys.exit(3 if last else 1)
     else:
         subprocess.run(step, shell=True, check=True)
 PY
 
-# dial <client source> <server addr> <port> — a new connection from the server
-# namespace to the client, which listens on the port. 0 when it connects.
+# dial <client source> <server addr> <port> [wait] — a new connection from the
+# server namespace to the client, which listens on the port. 0 when it connects.
 cat > "$TMP/dial.py" << 'PY'
 import os, socket, sys
 
 srv_ns, cli_ns, src, port = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4])
+wait = float(sys.argv[5]) if len(sys.argv) > 5 else 2
 fd = os.open("/run/netns/" + cli_ns, os.O_RDONLY)
 os.setns(fd, os.CLONE_NEWNET)
 listener = socket.socket()
@@ -111,7 +118,7 @@ fd = os.open("/run/netns/" + srv_ns, os.O_RDONLY)
 os.setns(fd, os.CLONE_NEWNET)
 dialer = socket.socket()
 dialer.bind((src, 0))
-dialer.settimeout(2)
+dialer.settimeout(wait)
 try:
     dialer.connect(("10.99.0.3", port))
 except OSError:
@@ -148,14 +155,21 @@ flow() {
     if out=$($SUDO python3 "$TMP/flow.py" "$S" "$C" "$2" 2>&1); then pass "$1"; else fail "$1: $out"; fi
 }
 
+# Steps expected to be dropped wait only this long, since a segment on a local
+# veth arrives in microseconds; steps expected to arrive keep the 2s default.
+DROP_WAIT=0.5
+
 # Control first: with nothing tracking, arming under an open flow must drop the
 # server's next segment, or this kernel cannot show the failure and the cases
 # below prove nothing.
 fresh
-if $SUDO python3 "$TMP/flow.py" "$S" "$C" "connect@@s2c@@c2s@@$arm@@s2c" > /dev/null 2>&1; then
+rc=0; $SUDO python3 "$TMP/flow.py" "$S" "$C" "connect@@s2c@@c2s@@$arm@@s2c" "$DROP_WAIT" > /dev/null 2>&1 || rc=$?
+if [ "$rc" = 3 ]; then
+    pass "control: a flow opened with no conntrack user loses the server's next segment once armed"
+elif [ "$rc" = 0 ]; then
     fail "control: a flow opened with no conntrack user survived the arm, so this kernel cannot reproduce the drop"
 else
-    pass "control: a flow opened with no conntrack user loses the server's next segment once armed"
+    fail "control: the flow failed before the arm (exit $rc), so the control proves nothing"
 fi
 
 fresh
@@ -182,12 +196,12 @@ if $SUDO python3 "$TMP/dial.py" "$S" "$C" 10.99.0.1 6644; then
 else
     fail "armed: a peer cannot reach a cluster port"
 fi
-if $SUDO python3 "$TMP/dial.py" "$S" "$C" 10.99.0.9 6644; then
+if $SUDO python3 "$TMP/dial.py" "$S" "$C" 10.99.0.9 6644 "$DROP_WAIT"; then
     fail "armed: a non-peer reaches a cluster port"
 else
     pass "armed: a non-peer is dropped on a cluster port"
 fi
-if $SUDO python3 "$TMP/dial.py" "$S" "$C" 10.99.0.1 7777; then
+if $SUDO python3 "$TMP/dial.py" "$S" "$C" 10.99.0.1 7777 "$DROP_WAIT"; then
     fail "armed: a peer reaches a port the policy does not open"
 else
     pass "armed: a peer is dropped on a port the policy does not open"

spec/EBSProvider/README.md

@@ -17,6 +17,7 @@ The contract that every implementation of the `EBSProvider` interface must meet.
   A failed call leaves the state unchanged, but a timeout can arrive after the provider acted, and a crash can leave work half done.
 - Is the schema version part of the contract?
   Every method refuses an unsupported version with `unsupported_version`, and the conformance suite tests it, but the specification does not model versions.
+  A snapshot's `state` in a ListSnapshots ref, and the `snapshot_id` filter, arrive at contract version 4: a ref carries no state below it, and a filtered call below it is refused with `unsupported_version` rather than answered with every snapshot.
 - Are `internal` and `unavailable` the only errors that a failed provider may cause?
 - Must a present but malformed ID be refused with `invalid_argument`?
   IDs are opaque here, so only an omitted one is refused.
@@ -100,13 +101,10 @@ The contract that every implementation of the `EBSProvider` interface must meet.
 
 ### Snapshots
 
-- Should a pending snapshot's move to `completed` or `error` be modelled, as a step that happens without a call?
-  Without it a pending snapshot never completes, and no snapshot is ever in `error`.
-
 #### CreateSnapshot
 
-- May success return a `pending` snapshot?
-  AWS starts a snapshot as `pending`, while the memory implementation returns `completed` and the NATS implementation waits for completion and refuses anything else.
+- May a provider forget a snapshot's source?
+  The control plane mints every snapshot ID and records it before the first call, so the ID alone names the request. A provider that no longer knows a snapshot's source, such as qemu-nbd after a restart, answers a retry with the snapshot, and refuses with `already_exists` only when it knows the source differs. The specification models a provider that always knows the source.
 - Must a retry compare the volume handle?
   It is ignored, as in the memory implementation, so a volume deleted and recreated under the same ID returns the old snapshot.
 - Should a copy's ID count as a retry?
@@ -133,7 +131,7 @@ The contract that every implementation of the `EBSProvider` interface must meet.
 - Is the handle check part of the contract?
   The Viperblock implementation ignores the handle and deletes anyway.
 - May a pending snapshot be deleted?
-  AWS allows it.
+  AWS allows it, while OCI refuses to delete a backup that is still being created, so the control plane never asks: it defers a customer's delete of a pending snapshot until the snapshot settles.
 - What does success promise?
   The Go docs say the snapshot is gone, but a snapshot whose handle did not match is left in place.
 - Should a snapshot that does not exist be refused with `not_found`?

spec/EBSProvider/Snapshots.tla

@@ -13,8 +13,9 @@ CreateSnapshotRequests == [snapshot_id   : Optional(SnapshotIDs),
 
 CreateSnapshot(req, val, err) ==
   (*******************************************************************************************************************)
-  (* A volume's data is frozen as a snapshot, from which new volumes can be restored. A retry of a call that         *)
-  (* already succeeded returns the same snapshot.                                                                    *)
+  (* A volume's data is frozen as a snapshot, from which new volumes can be restored. The snapshot holds the         *)
+  (* volume's data as it was when the call returned, even while it is pending, and a pending snapshot settles later  *)
+  (* without a call (SettleSnapshot). A retry of a call that already succeeded returns the same snapshot.            *)
   (*******************************************************************************************************************)
   LET snapID == req.snapshot_id
       volID  == req.volume_id
@@ -97,27 +98,45 @@ CopySnapshot(req, val, err) ==
 
 
 ListSnapshotsRequests == [max_results    : -1..(MaxListResults + 1),
-                          starting_token : Optional(ListTokens)]
+                          starting_token : Optional(ListTokens),
+                          snapshot_id    : Optional(SnapshotIDs)]
+
+\* The ref that ListSnapshots returns for a snapshot, carrying its current state.
+SnapshotRef(id) == [id               |-> id,
+                    source_volume_id |-> snapshots[id].source_volume_id,
+                    handle           |-> snapshots[id].handle,
+                    state            |-> snapshots[id].state]
 
 ListSnapshots(req, val, err) ==
   (*******************************************************************************************************************)
   (* Some of the snapshots are listed, a page at a time, and nothing changes. A call with no starting token that     *)
-  (* returns no next token lists every snapshot.                                                                     *)
+  (* returns no next token lists every snapshot. A call that names a snapshot returns that snapshot in its current   *)
+  (* state, or nothing when it does not exist, so an accepted snapshot is never reported absent while it is pending. *)
   (*******************************************************************************************************************)
-  LET Errors ==
-    LET checks == [unsupported_capability |-> ~Capabilities.snapshot_enumeration]
-    IN {e \in DOMAIN checks : checks[e]}
+  LET snapID == req.snapshot_id
+
+      Errors ==
+        \* A call that names a snapshot reads only that one, so it needs no enumeration.
+        LET checks == [unsupported_capability |-> snapID = None /\ ~Capabilities.snapshot_enumeration]
+        IN {e \in DOMAIN checks : checks[e]}
 
      \* The request was rejected, or the provider failed to answer.
   IN \/ /\ val = None
         /\ err \in Errors \union ProviderErrors
         /\ UNCHANGED vars
 
+     \* The request named one snapshot, so paging is ignored.
+     \/ /\ Errors = {}
+        /\ snapID /= None
+        /\ val = [snapshots  |-> IF KeyExists(snapshots, snapID) THEN {SnapshotRef(snapID)} ELSE {},
+                  next_token |-> None]
+        /\ err = None
+        /\ UNCHANGED vars
+
      \* The request was accepted.
      \/ /\ Errors = {}
-        /\ LET refs == {[id               |-> id,
-                         source_volume_id |-> snapshots[id].source_volume_id,
-                         handle           |-> snapshots[id].handle] : id \in DOMAIN snapshots}
+        /\ snapID = None
+        /\ LET refs == {SnapshotRef(id) : id \in DOMAIN snapshots}
            \* The page and the next token are the implementation's choice.
            IN \E page \in SUBSET refs, next \in Optional(ListTokens) :
                 /\ Cardinality(page) <= PageSize(req.max_results)
@@ -156,4 +175,18 @@ DeleteSnapshot(req, err) ==
         /\ err = None
         /\ UNCHANGED <<volumes, publications>>
 
+-----------------------------------------------------------------------------------------------------------------------
+\* Steps without a call
+
+SettleSnapshot(snapID) ==
+  (*******************************************************************************************************************)
+  (* A pending snapshot settles as completed or error, as the provider finishes it. Its data is unchanged, since it  *)
+  (* was fixed when CreateSnapshot returned, and a snapshot that has settled never changes state again.              *)
+  (*******************************************************************************************************************)
+  /\ KeyExists(snapshots, snapID)
+  /\ snapshots[snapID].state = "pending"
+  /\ \E s \in {"completed", "error"} :
+       snapshots' = [snapshots EXCEPT ![snapID].state = s]
+  /\ UNCHANGED <<volumes, publications>>
+
 =======================================================================================================================

spinifex/domains/ec2/awsapi/actions.go

@@ -113,6 +113,7 @@ var ec2ListsAsSet = map[reflect.Type][]string{
 	reflect.TypeFor[ec2.NetworkInterface]():                 {"Ipv4Prefixes", "Ipv6Prefixes"},
 	reflect.TypeFor[ec2.Vpc]():                              {"Ipv6CidrBlockAssociationSet"},
 	reflect.TypeFor[ec2.Subnet]():                           {"Tags"},
+	reflect.TypeFor[ec2.InstanceStatusSummary]():            {"Details"},
 }
 
 // marshalEC2Response renders an EC2 handler's output into the action's XML

spinifex/domains/ec2/awsapi/emptyresult_test.go

@@ -186,3 +186,16 @@ func TestEC2_NetworkListsAWSOmitsWhenEmpty(t *testing.T) {
 	assert.NotContains(t, subnet, "tagSet")
 	assert.Contains(t, subnet, "<ipv6CidrBlockAssociationSet></ipv6CidrBlockAssociationSet>")
 }
+
+// A stopped instance's not-applicable checks carry no details on the wire.
+func TestEC2_InstanceStatusDetailsOmittedWhenNotApplicable(t *testing.T) {
+	out := render[ec2.DescribeInstanceStatusInput](t, "DescribeInstanceStatus", ec2.DescribeInstanceStatusOutput{
+		InstanceStatuses: []*ec2.InstanceStatus{{
+			InstanceId:     aws.String("i-1"),
+			InstanceStatus: &ec2.InstanceStatusSummary{Status: aws.String("not-applicable")},
+			SystemStatus:   &ec2.InstanceStatusSummary{Status: aws.String("not-applicable")},
+		}},
+	})
+	assert.Contains(t, out, "<instanceStatus><status>not-applicable</status></instanceStatus>")
+	assert.NotContains(t, out, "details")
+}

spinifex/domains/ec2/awsapi/image/DescribeImageAttribute.go

@@ -12,11 +12,14 @@ import (
 )
 
 // supportedImageAttributes lists the AMI attributes spinifex exposes.
-// description is modifiable; blockDeviceMapping is synthesised read-only from
-// AMIMetadata. Any other attribute is rejected rather than returning empty.
+// Any other attribute is rejected rather than returning empty.
 var supportedImageAttributes = map[string]bool{
 	ec2.ImageAttributeNameDescription:        true,
 	ec2.ImageAttributeNameBlockDeviceMapping: true,
+	ec2.ImageAttributeNameLaunchPermission:   true,
+	ec2.ImageAttributeNameBootMode:           true,
+	ec2.ImageAttributeNameImdsSupport:        true,
+	ec2.ImageAttributeNameLastLaunchedTime:   true,
 }
 
 // ValidateDescribeImageAttributeInput requires an ami- prefixed ImageId and an Attribute.

spinifex/domains/ec2/awsapi/image/DescribeImageAttribute_test.go

@@ -46,24 +46,6 @@ func TestValidateDescribeImageAttributeInput(t *testing.T) {
 			wantErr: true,
 			errMsg:  awserrors.ErrorInvalidAMIIDMalformed,
 		},
-		{
-			name: "UnsupportedAttribute_LaunchPermission",
-			input: &ec2.DescribeImageAttributeInput{
-				ImageId:   aws.String("ami-1234567890abcdef0"),
-				Attribute: aws.String("launchPermission"),
-			},
-			wantErr: true,
-			errMsg:  awserrors.ErrorInvalidParameterValue,
-		},
-		{
-			name: "UnsupportedAttribute_BootMode",
-			input: &ec2.DescribeImageAttributeInput{
-				ImageId:   aws.String("ami-1234567890abcdef0"),
-				Attribute: aws.String("bootMode"),
-			},
-			wantErr: true,
-			errMsg:  awserrors.ErrorInvalidParameterValue,
-		},
 		{
 			name: "UnsupportedAttribute_Kernel",
 			input: &ec2.DescribeImageAttributeInput{
@@ -89,6 +71,38 @@ func TestValidateDescribeImageAttributeInput(t *testing.T) {
 			},
 			wantErr: false,
 		},
+		{
+			name: "ValidLaunchPermission",
+			input: &ec2.DescribeImageAttributeInput{
+				ImageId:   aws.String("ami-1234567890abcdef0"),
+				Attribute: aws.String("launchPermission"),
+			},
+			wantErr: false,
+		},
+		{
+			name: "ValidBootMode",
+			input: &ec2.DescribeImageAttributeInput{
+				ImageId:   aws.String("ami-1234567890abcdef0"),
+				Attribute: aws.String("bootMode"),
+			},
+			wantErr: false,
+		},
+		{
+			name: "ValidImdsSupport",
+			input: &ec2.DescribeImageAttributeInput{
+				ImageId:   aws.String("ami-1234567890abcdef0"),
+				Attribute: aws.String("imdsSupport"),
+			},
+			wantErr: false,
+		},
+		{
+			name: "ValidLastLaunchedTime",
+			input: &ec2.DescribeImageAttributeInput{
+				ImageId:   aws.String("ami-1234567890abcdef0"),
+				Attribute: aws.String("lastLaunchedTime"),
+			},
+			wantErr: false,
+		},
 	}
 
 	for _, tt := range tests {

spinifex/domains/ec2/awsapi/instance/DescribeInstanceStatus.go

@@ -183,20 +183,8 @@ func buildInstanceStatusFromInstance(inst *ec2.Instance, az string) *ec2.Instanc
 		AvailabilityZone: aws.String(az),
 		InstanceId:       inst.InstanceId,
 		InstanceState:    state,
-		InstanceStatus: &ec2.InstanceStatusSummary{
-			Status: aws.String(notApplicable),
-			Details: []*ec2.InstanceStatusDetails{{
-				Name:   aws.String(reachabilityDetailName),
-				Status: aws.String(notApplicable),
-			}},
-		},
-		SystemStatus: &ec2.InstanceStatusSummary{
-			Status: aws.String(notApplicable),
-			Details: []*ec2.InstanceStatusDetails{{
-				Name:   aws.String(reachabilityDetailName),
-				Status: aws.String(notApplicable),
-			}},
-		},
+		InstanceStatus:   &ec2.InstanceStatusSummary{Status: aws.String(notApplicable)},
+		SystemStatus:     &ec2.InstanceStatusSummary{Status: aws.String(notApplicable)},
 	}
 }
 

spinifex/domains/ec2/awsapi/instance/DescribeInstanceStatus_test.go

@@ -162,7 +162,7 @@ func TestDescribeInstanceStatus_IncludeAllAddsStoppedAsNotApplicable(t *testing.
 	assert.Equal(t, "stopped", *stopped.InstanceState.Name)
 	assert.Equal(t, "not-applicable", *stopped.InstanceStatus.Status)
 	assert.Equal(t, "not-applicable", *stopped.SystemStatus.Status)
-	assert.Equal(t, "not-applicable", *stopped.InstanceStatus.Details[0].Status)
+	assert.Nil(t, stopped.InstanceStatus.Details)
 	assert.Equal(t, "az-a", *stopped.AvailabilityZone)
 }
 
@@ -233,6 +233,8 @@ func TestBuildInstanceStatusFromInstance_PopulatesState(t *testing.T) {
 	assert.Equal(t, "az-a", *got.AvailabilityZone)
 	assert.Equal(t, "not-applicable", *got.InstanceStatus.Status)
 	assert.Equal(t, "not-applicable", *got.SystemStatus.Status)
+	assert.Nil(t, got.InstanceStatus.Details, "AWS reports a stopped instance's checks without details")
+	assert.Nil(t, got.SystemStatus.Details, "AWS reports a stopped instance's checks without details")
 }
 
 func TestBuildInstanceStatusFromInstance_FallbackStateName(t *testing.T) {

spinifex/domains/ec2/awsapi/vpc/AttachNetworkInterface.go

@@ -76,6 +76,9 @@ func AttachNetworkInterface(ctx context.Context, input *ec2.AttachNetworkInterfa
 
 	responseError, err := awserrors.ValidateErrorPayload(msg.Data)
 	if err != nil {
+		if text := aws.StringValue(responseError.Message); text != "" {
+			return output, awserrors.Errorf(*responseError.Code, "%s", text)
+		}
 		return output, errors.New(*responseError.Code)
 	}
 

spinifex/domains/ec2/awsapi/vpc/network_interface_hotplug_test.go

@@ -145,6 +145,28 @@ func TestAttachNetworkInterface_DaemonErrorResponse(t *testing.T) {
 	assert.EqualError(t, err, awserrors.ErrorIncorrectInstanceState)
 }
 
+// The daemon's AWS error text must reach the client alongside the code.
+func TestAttachNetworkInterface_DaemonErrorKeepsMessage(t *testing.T) {
+	nc := newTestNATS(t)
+	const text = "Instance 'i-msg' already has an interface attached at device index '1'."
+
+	sub, err := nc.Subscribe("ec2.cmd.i-msg", func(msg *nats.Msg) {
+		_ = msg.Respond(awserrors.GenerateErrorPayloadWithMessage(awserrors.ErrorInvalidParameterValue, text))
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+
+	_, err = AttachNetworkInterface(context.Background(), &ec2.AttachNetworkInterfaceInput{
+		NetworkInterfaceId: aws.String("eni-msg"),
+		InstanceId:         aws.String("i-msg"),
+		DeviceIndex:        aws.Int64(1),
+	}, nc, testAccountID)
+	code, msg, ok := awserrors.ResolveErrorDetail(err)
+	require.True(t, ok)
+	assert.Equal(t, awserrors.ErrorInvalidParameterValue, code)
+	assert.Equal(t, text, msg)
+}
+
 func TestAttachNetworkInterface_MalformedDaemonResponse(t *testing.T) {
 	nc := newTestNATS(t)
 

spinifex/domains/ec2/ebs/metadata/metadata.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"strings"
 	"time"
@@ -82,6 +83,48 @@ type Snapshot struct {
 	AvailabilityZone string            `json:"availability_zone"`
 	Tags             map[string]string `json:"tags,omitempty"`
 	ProviderHandle   string            `json:"provider_handle,omitempty"`
+	// DeleteRequested is a DeleteSnapshot accepted before the snapshot could be
+	// deleted; the snapshot is hidden from then on and the reaper deletes it.
+	DeleteRequested bool `json:"delete_requested,omitempty"`
+	// AmbiguousCreate is set before every provider CreateSnapshot call and kept
+	// while any call on the document went unanswered, as one may still be accepted.
+	AmbiguousCreate bool `json:"ambiguous_create,omitempty"`
+	// TombstonedAt is when a deleted AmbiguousCreate snapshot's document became
+	// a tombstone, under which a late-accepted snapshot of its ID is deleted.
+	TombstonedAt time.Time `json:"tombstoned_at,omitzero"`
+}
+
+// Hidden reports a snapshot a customer has deleted: it is absent to every
+// request, and only the reaper still acts on its document.
+func (s Snapshot) Hidden() bool {
+	return s.DeleteRequested || !s.TombstonedAt.IsZero()
+}
+
+// Tombstoned reports a document kept only to catch a late-accepted create.
+func (s Snapshot) Tombstoned() bool {
+	return !s.TombstonedAt.IsZero()
+}
+
+// SnapshotStateCompleted is the one snapshot state a consumer may use.
+const SnapshotStateCompleted = "completed"
+
+// The states of an AMI document. A CreateImage's AMI is pending until its
+// snapshot settles; available is the one state an instance may launch from.
+const (
+	AMIStatePending   = "pending"
+	AMIStateAvailable = "available"
+	AMIStateFailed    = "failed"
+)
+
+// Ready reports a snapshot whose contents a consumer may read.
+func (s Snapshot) Ready() bool {
+	return s.State == SnapshotStateCompleted
+}
+
+// NotReadyError is the IncorrectState a consumer answers for a snapshot that
+// is not Ready.
+func (s Snapshot) NotReadyError() error {
+	return awserrors.Errorf(awserrors.ErrorIncorrectState, "Snapshot is in invalid state - %s", s.State)
 }
 
 // RootDeviceName is the device every image this platform serves reports as its
@@ -115,6 +158,18 @@ type AMI struct {
 	UEFIData *UEFIDataRef `json:"uefi_data,omitempty"`
 }
 
+// Available reports an AMI an instance may launch from or a copy may read. An
+// empty State predates the field, and every such image is complete.
+func (a AMI) Available() bool {
+	return a.State == "" || a.State == AMIStateAvailable
+}
+
+// UnavailableError is the InvalidAMIID.Unavailable answer for imageID, whose
+// document is a, when a is not Available.
+func (a AMI) UnavailableError(imageID string) error {
+	return awserrors.Errorf(awserrors.ErrorInvalidAMIIDUnavailable, "The image '%s' is %s, not available.", imageID, a.State)
+}
+
 // UEFIDataRef names an image's UEFI variable store object and the firmware it
 // was written under, so a launch can check the firmware without reading it.
 type UEFIDataRef struct {

spinifex/domains/ec2/guestmetadata/dns_forwarder.go

@@ -7,6 +7,7 @@ import (
 	"log/slog"
 	"net"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"golang.org/x/time/rate"
@@ -16,10 +17,14 @@ import (
 // advertise up to 4096, so the relay buffer must not truncate below that.
 const maxDNSUDPSize = 4096
 
-// dnsExchangeTimeout bounds one backend attempt (UDP round-trip or TCP dial),
-// so failover across every backend stays inside a guest resolver's ~5s retry.
+// dnsExchangeTimeout is how long a UDP attempt runs before the next backend is
+// also tried, and bounds a TCP dial, so failover stays inside a guest's ~5s retry.
 const dnsExchangeTimeout = 2 * time.Second
 
+// dnsExchangeTotalTimeout bounds a UDP query across all attempts: above northstar's
+// 5s upstream read, so a slow answer is relayed, and below glibc's 10s total.
+const dnsExchangeTotalTimeout = 8 * time.Second
+
 // dnsTCPSessionTimeout bounds a proxied TCP DNS session end-to-end.
 const dnsTCPSessionTimeout = 30 * time.Second
 
@@ -84,10 +89,13 @@ func (b *dnsBackend) markUp() {
 type dnsForwarder struct {
 	backends    []*dnsBackend // northstar targets, health-tried healthy-first
 	timeout     time.Duration
+	total       time.Duration // whole-query bound; every attempt still gets timeout
 	cooldown    time.Duration // backend down-marking window (H3)
 	maxInFlight int           // per-tap concurrency cap (H2)
 	rate, burst int           // per-tap token-bucket query rate (H2)
 	now         func() time.Time
+
+	inFlight atomic.Int32 // open UDP attempts, for tests
 }
 
 func newDNSForwarder(targets []string) *dnsForwarder {
@@ -98,6 +106,7 @@ func newDNSForwarder(targets []string) *dnsForwarder {
 	return &dnsForwarder{
 		backends:    backends,
 		timeout:     dnsExchangeTimeout,
+		total:       dnsExchangeTotalTimeout,
 		cooldown:    dnsBackendCooldown,
 		maxInFlight: dnsMaxInFlightPerTap,
 		rate:        dnsQueryRatePerTap,
@@ -129,6 +138,7 @@ func (f *dnsForwarder) ordered(now time.Time) []*dnsBackend {
 func (f *dnsForwarder) serveUDP(pc net.PacketConn) {
 	sem := make(chan struct{}, f.maxInFlight)
 	limiter := newTokenBucket(f.rate, f.burst, f.now)
+	warns := &warnThrottle{every: f.cooldown, now: f.now}
 	buf := make([]byte, maxDNSUDPSize)
 	for {
 		n, addr, err := pc.ReadFrom(buf)
@@ -154,7 +164,7 @@ func (f *dnsForwarder) serveUDP(pc net.PacketConn) {
 			defer func() { <-sem }()
 			resp, err := f.exchangeUDP(query)
 			if err != nil {
-				slog.Debug("IMDS: DNS shim exchange failed on all backends", "err", err)
+				f.logAllBackendsFailed(warns, err)
 				resp = servfail(query)
 				if resp == nil {
 					return
@@ -167,34 +177,91 @@ func (f *dnsForwarder) serveUDP(pc net.PacketConn) {
 	}
 }
 
-// exchangeUDP relays one query to the backends healthy-first, returning the first
-// answer received within the per-backend timeout. A failing backend is marked
-// down for the cooldown window; a responding one is marked healthy again.
+// exchangeUDP relays one query to the backends healthy-first. A backend that has
+// not answered within the per-backend timeout stays open while the next is tried,
+// and the first answer wins, so a slow but valid answer is relayed, not SERVFAILed.
 func (f *dnsForwarder) exchangeUDP(query []byte) ([]byte, error) {
-	var lastErr error
-	for _, b := range f.ordered(f.now()) {
-		resp, err := f.exchangeUDPOnce(b.addr, query)
-		if err != nil {
-			b.markDown(f.now(), f.cooldown)
-			lastErr = err
-			continue
-		}
-		b.markUp()
-		return resp, nil
+	backends := f.ordered(f.now())
+	if len(backends) == 0 {
+		return nil, errors.New("no DNS backends configured")
 	}
-	if lastErr == nil {
-		lastErr = errors.New("no DNS backends configured")
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	type result struct {
+		i    int
+		resp []byte
+		err  error
+	}
+	results := make(chan result, len(backends))
+	overall := time.Now().Add(f.total)
+	started := make([]time.Time, len(backends))
+	done := make([]bool, len(backends))
+	next, open := 0, 0
+	launch := func() {
+		i := next
+		started[i] = time.Now()
+		deadline := started[i].Add(f.timeout)
+		if overall.After(deadline) {
+			deadline = overall
+		}
+		next++
+		open++
+		f.inFlight.Add(1)
+		go func() {
+			defer f.inFlight.Add(-1)
+			resp, err := f.exchangeUDPOnce(ctx, backends[i].addr, query, deadline)
+			results <- result{i: i, resp: resp, err: err}
+		}()
+	}
+
+	launch()
+	hop := time.NewTimer(f.timeout)
+	defer hop.Stop()
+	var lastErr error
+	for open > 0 {
+		select {
+		case r := <-results:
+			open--
+			done[r.i] = true
+			if r.err == nil {
+				backends[r.i].markUp()
+				for j := range next {
+					if !done[j] && time.Since(started[j]) >= f.timeout {
+						backends[j].markDown(f.now(), f.cooldown)
+					}
+				}
+				return r.resp, nil
+			}
+			backends[r.i].markDown(f.now(), f.cooldown)
+			lastErr = r.err
+			if next < len(backends) {
+				launch()
+				hop.Reset(f.timeout)
+			}
+		case <-hop.C:
+			if next < len(backends) {
+				launch()
+				hop.Reset(f.timeout)
+			}
+		}
 	}
 	return nil, lastErr
 }
 
-func (f *dnsForwarder) exchangeUDPOnce(target string, query []byte) ([]byte, error) {
+// exchangeUDPOnce runs one attempt until deadline; cancelling ctx closes its
+// socket, so an attempt that lost the race does not outlive the query.
+func (f *dnsForwarder) exchangeUDPOnce(ctx context.Context, target string, query []byte, deadline time.Time) ([]byte, error) {
 	conn, err := net.DialTimeout("udp", target, f.timeout)
 	if err != nil {
 		return nil, err
 	}
-	defer func() { _ = conn.Close() }()
-	if err := conn.SetDeadline(time.Now().Add(f.timeout)); err != nil {
+	stop := context.AfterFunc(ctx, func() { _ = conn.Close() })
+	defer func() {
+		stop()
+		_ = conn.Close()
+	}()
+	if err := conn.SetDeadline(deadline); err != nil {
 		return nil, err
 	}
 	if _, err := conn.Write(query); err != nil {
@@ -208,6 +275,36 @@ func (f *dnsForwarder) exchangeUDPOnce(target string, query []byte) ([]byte, err
 	return buf[:n], nil
 }
 
+// warnThrottle admits one Warn per window. Each serveUDP owns one, so one tap's
+// failing guest cannot hide another tap's failures.
+type warnThrottle struct {
+	mu    sync.Mutex
+	next  time.Time
+	every time.Duration
+	now   func() time.Time
+}
+
+func (w *warnThrottle) allow() bool {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	now := w.now()
+	if now.Before(w.next) {
+		return false
+	}
+	w.next = now.Add(w.every)
+	return true
+}
+
+// logAllBackendsFailed reports a SERVFAIL at Warn at most once per window per tap,
+// so a guest flooding a dead backend cannot flood the log; the rest stay at Debug.
+func (f *dnsForwarder) logAllBackendsFailed(warns *warnThrottle, err error) {
+	if warns.allow() {
+		slog.Warn("IMDS: DNS shim exchange failed on all backends", "backends", len(f.backends), "err", err)
+		return
+	}
+	slog.Debug("IMDS: DNS shim exchange failed on all backends", "backends", len(f.backends), "err", err)
+}
+
 // serveTCP accepts guest TCP DNS sessions (truncated-response retries) and
 // proxies each to the first dialable backend. A per-tap token bucket and
 // in-flight semaphore bound accepts the same way serveUDP bounds datagrams (H2).

spinifex/domains/ec2/guestmetadata/dns_forwarder_test.go

@@ -88,19 +88,129 @@ func TestDNSForwarder_UDPServfailWhenAllBackendsDead(t *testing.T) {
 	dead := fakeUDPUpstream(t, silent)
 	f := newDNSForwarder([]string{dead})
 	f.timeout = 100 * time.Millisecond
+	f.total = 300 * time.Millisecond
 
 	pc, err := net.ListenPacket("udp", "127.0.0.1:0")
 	require.NoError(t, err)
 	go f.serveUDP(pc)
 	defer func() { _ = pc.Close() }()
 
+	start := time.Now()
 	resp := udpQuery(t, pc.LocalAddr().String(), testDNSQuery)
+	elapsed := time.Since(start)
+	assert.GreaterOrEqual(t, elapsed, f.total, "SERVFAIL must wait for the overall deadline, not one backend's timeout")
+	assert.Less(t, elapsed, f.total+500*time.Millisecond)
 	require.GreaterOrEqual(t, len(resp), 12)
 	assert.Equal(t, testDNSQuery[0:2], resp[0:2], "SERVFAIL must echo the query ID")
 	assert.NotZero(t, resp[2]&0x80, "QR bit must be set")
 	assert.Equal(t, byte(2), resp[3]&0x0F, "RCODE must be SERVFAIL")
 }
 
+// delayed answers like respond, but only after d, as a slow upstream would.
+func delayed(d time.Duration, respond func(q []byte) []byte) func(q []byte) []byte {
+	return func(q []byte) []byte {
+		time.Sleep(d)
+		return respond(q)
+	}
+}
+
+// A lone backend that answers after its per-backend timeout is relayed, not
+// replaced with SERVFAIL, as long as it answers inside the overall bound.
+func TestDNSForwarder_UDPRelaysSlowAnswerFromOnlyBackend(t *testing.T) {
+	slow := fakeUDPUpstream(t, delayed(300*time.Millisecond, markedResponse('S')))
+	f := newDNSForwarder([]string{slow})
+	f.timeout = 100 * time.Millisecond
+	f.total = 600 * time.Millisecond
+
+	resp, err := f.exchangeUDP(testDNSQuery)
+	require.NoError(t, err, "a slow answer inside the overall bound must be relayed")
+	assert.Equal(t, byte('S'), resp[len(resp)-1])
+}
+
+// A slow first backend stays open while the next is tried, and its answer wins
+// when the next one never answers.
+func TestDNSForwarder_UDPKeepsSlowBackendOpenDuringFailover(t *testing.T) {
+	slow := fakeUDPUpstream(t, delayed(300*time.Millisecond, markedResponse('S')))
+	dead := fakeUDPUpstream(t, silent)
+	f := newDNSForwarder([]string{slow, dead})
+	f.timeout = 100 * time.Millisecond
+	f.total = 600 * time.Millisecond
+
+	resp, err := f.exchangeUDP(testDNSQuery)
+	require.NoError(t, err)
+	assert.Equal(t, byte('S'), resp[len(resp)-1], "the first backend's late answer must be returned")
+}
+
+// The overall bound is a boundary: just inside it is relayed, just past it fails.
+func TestDNSForwarder_UDPOverallDeadlineBoundary(t *testing.T) {
+	inside := fakeUDPUpstream(t, delayed(250*time.Millisecond, markedResponse('I')))
+	f := newDNSForwarder([]string{inside})
+	f.timeout = 100 * time.Millisecond
+	f.total = 400 * time.Millisecond
+	_, err := f.exchangeUDP(testDNSQuery)
+	require.NoError(t, err, "an answer before the overall deadline must be relayed")
+
+	outside := fakeUDPUpstream(t, delayed(600*time.Millisecond, markedResponse('O')))
+	f = newDNSForwarder([]string{outside})
+	f.timeout = 100 * time.Millisecond
+	f.total = 400 * time.Millisecond
+	start := time.Now()
+	_, err = f.exchangeUDP(testDNSQuery)
+	require.Error(t, err, "an answer after the overall deadline must not be waited for")
+	elapsed := time.Since(start)
+	assert.GreaterOrEqual(t, elapsed, f.total, "the exchange must wait for the overall deadline")
+	assert.Less(t, elapsed, 550*time.Millisecond, "the exchange must end at the overall deadline")
+}
+
+// With more silent backends than the overall bound has hops for, the last one
+// still gets its own per-backend timeout, as sequential failover gave it.
+func TestDNSForwarder_UDPLastBackendReachableBehindSilentOnes(t *testing.T) {
+	targets := make([]string, 0, 5)
+	for range 4 {
+		targets = append(targets, fakeUDPUpstream(t, silent))
+	}
+	targets = append(targets, fakeUDPUpstream(t, markedResponse('L')))
+	f := newDNSForwarder(targets)
+	f.timeout = 100 * time.Millisecond
+	f.total = 250 * time.Millisecond
+
+	resp, err := f.exchangeUDP(testDNSQuery)
+	require.NoError(t, err, "the fifth backend must still be tried and answer")
+	assert.Equal(t, byte('L'), resp[len(resp)-1])
+}
+
+// Attempts that lost the race are closed when the exchange returns, so they do
+// not hold sockets past the query and escape the per-tap in-flight cap.
+func TestDNSForwarder_UDPClosesLosingAttempts(t *testing.T) {
+	dead := fakeUDPUpstream(t, silent)
+	live := fakeUDPUpstream(t, markedResponse('Y'))
+	f := newDNSForwarder([]string{dead, live})
+	f.timeout = 100 * time.Millisecond
+	f.total = 5 * time.Second
+
+	_, err := f.exchangeUDP(testDNSQuery)
+	require.NoError(t, err)
+	assert.Eventually(t, func() bool { return f.inFlight.Load() == 0 },
+		time.Second, 10*time.Millisecond, "the silent backend's attempt must not run to the 5s deadline")
+}
+
+// Each tap's throttle is its own: one tap's suppressed failures leave another
+// tap's first Warn through, and each suppresses its own repeats in the window.
+func TestWarnThrottle_PerTapWindows(t *testing.T) {
+	base := time.Now()
+	now := func() time.Time { return base }
+	a := &warnThrottle{every: 10 * time.Second, now: now}
+	b := &warnThrottle{every: 10 * time.Second, now: now}
+
+	assert.True(t, a.allow(), "tap A's first failure warns")
+	assert.False(t, a.allow(), "tap A's repeat inside the window is suppressed")
+	assert.True(t, b.allow(), "tap B's first failure warns despite tap A's window")
+	assert.False(t, b.allow())
+
+	base = base.Add(11 * time.Second)
+	assert.True(t, a.allow(), "tap A warns again once its window has passed")
+}
+
 func TestDNSForwarder_TCPProxiesFramedSession(t *testing.T) {
 	// Echo backend: framed request bytes come back verbatim, proving the proxy
 	// relays the length-prefixed stream untouched in both directions.

spinifex/domains/ec2/image/account_termination_current_behaviour_test.go

@@ -57,6 +57,11 @@ func newStorageServices(t *testing.T) storageServices {
 	snapshots.SetEBSProvider(provider)
 	images := ec2image.NewImageServiceImplWithConfig(cfg, store, nil)
 	images.SetEBSProvider(provider)
+	// CreateImage snapshots through a snapshot service with no index handle.
+	imageSnapshots := ec2snapshot.NewSnapshotServiceImplWithStore(cfg, store, nc)
+	imageSnapshots.SetEBSProvider(provider)
+	images.SetSnapshotCreator(imageSnapshots)
+	images.SetImageClaims(imageSnapshots)
 
 	return storageServices{
 		volumes: volumes, snapshots: snapshots, images: images,
@@ -230,9 +235,9 @@ func TestCurrentBehaviour_AccountTermination_DeleteSnapshotDuringCloneCreationSt
 	assert.EqualError(t, err, awserrors.ErrorInvalidSnapshotNotFound, "a clone started after the delete is refused")
 }
 
-// CreateImage on an instance re-reads the image the instance launched from,
-// after the provider has taken the snapshot, so once that image is
-// deregistered the call fails and leaves a provider snapshot with no document.
+// CreateImage on an instance reads the image the instance launched from before
+// it snapshots the root volume, so once that image is deregistered the call
+// fails and the provider holds no snapshot for it.
 func TestCurrentBehaviour_AccountTermination_CreateImageOfRetainedInstanceNeedsItsLaunchImage(t *testing.T) {
 	s := newStorageServices(t)
 	ctx := context.Background()
@@ -252,17 +257,7 @@ func TestCurrentBehaviour_AccountTermination_CreateImageOfRetainedInstanceNeedsI
 
 	listed, err := s.provider.ListSnapshots(ctx, ebsprovider.ListSnapshotsRequest{Versioned: ebsprovider.NewVersioned()})
 	require.NoError(t, err)
-	var orphans []string
 	for _, ref := range listed.Snapshots {
-		if ref.SourceVolumeID == "vol-retained-root" {
-			orphans = append(orphans, ref.ID)
-		}
-	}
-	require.Len(t, orphans, 1, "the provider kept the snapshot it took before the image read failed")
-
-	docs, err := s.metadata.ListSnapshotsStrict(ctx, ctAccountID)
-	require.NoError(t, err)
-	for _, doc := range docs {
-		assert.NotEqual(t, orphans[0], doc.SnapshotID, "no document names that snapshot")
+		assert.NotEqual(t, "vol-retained-root", ref.SourceVolumeID, "the provider took a snapshot for an image that failed")
 	}
 }

spinifex/domains/ec2/image/create_image_pending_test.go

@@ -0,0 +1,308 @@
+package image
+
+import (
+	"context"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/aws/aws-sdk-go/service/s3"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	ec2snapshot "github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// pendingImageFixture is an image service wired to a snapshot service over the
+// same store and memory provider, whose reaper settles what CreateImage leaves.
+type pendingImageFixture struct {
+	svc       *ImageServiceImpl
+	snapshots *ec2snapshot.SnapshotServiceImpl
+	mem       *ebsprovider.MemoryProvider
+	volume    ebsmetadata.Volume
+}
+
+func newPendingImageFixture(t *testing.T) *pendingImageFixture {
+	t.Helper()
+	svc, _, provider := setupProviderSnapshotImageService(t)
+	mem, ok := provider.(*ebsprovider.MemoryProvider)
+	require.True(t, ok)
+	snapshots := useProvider(svc, mem)
+	return &pendingImageFixture{
+		svc:       svc,
+		snapshots: snapshots,
+		mem:       mem,
+		volume:    seedProviderVolume(t, svc, provider, "vol-createimage01", 8),
+	}
+}
+
+func (f *pendingImageFixture) createImage(t *testing.T) (*ec2.CreateImageOutput, error) {
+	t.Helper()
+	return f.svc.CreateImageFromInstance(CreateImageParams{
+		Input:        &ec2.CreateImageInput{InstanceId: aws.String("i-createimage01"), Name: aws.String("pending-image")},
+		RootVolumeID: f.volume.VolumeID,
+	}, testAccountID)
+}
+
+// onlyAMI is the one AMI document in the store, and its snapshot's document.
+func (f *pendingImageFixture) onlyAMI(t *testing.T) (ebsmetadata.AMI, ebsmetadata.Snapshot) {
+	t.Helper()
+	amis, err := f.svc.MetadataStore().ListAMIs(context.Background())
+	require.NoError(t, err)
+	require.Len(t, amis, 1)
+	snap, err := f.svc.MetadataStore().GetSnapshot(context.Background(), testAccountID, amis[0].SnapshotID)
+	require.NoError(t, err)
+	return amis[0], snap
+}
+
+func (f *pendingImageFixture) sweep(t *testing.T) {
+	t.Helper()
+	_, err := f.snapshots.NewPendingSnapshotReaper().Sweep(context.Background())
+	require.NoError(t, err)
+}
+
+// recordingCreateProvider reads the image's documents when the provider is called.
+type recordingCreateProvider struct {
+	ebsprovider.EBSProvider
+
+	store     *ebsmetadata.Store
+	amis      []ebsmetadata.AMI
+	snapshot  ebsmetadata.Snapshot
+	readError error
+}
+
+func (p *recordingCreateProvider) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
+	p.amis, p.readError = p.store.ListAMIs(ctx)
+	if p.readError == nil {
+		p.snapshot, p.readError = p.store.GetSnapshot(ctx, testAccountID, req.SnapshotID)
+	}
+	return p.EBSProvider.CreateSnapshot(ctx, req)
+}
+
+func TestCreateImage_WritesBothDocumentsPendingBeforeTheProviderCall(t *testing.T) {
+	f := newPendingImageFixture(t)
+	recording := &recordingCreateProvider{EBSProvider: f.mem, store: f.svc.MetadataStore()}
+	f.snapshots.SetEBSProvider(recording)
+
+	_, err := f.createImage(t)
+	require.NoError(t, err)
+
+	require.NoError(t, recording.readError)
+	require.Len(t, recording.amis, 1, "the AMI document exists when the provider is called")
+	assert.Equal(t, ebsmetadata.AMIStatePending, recording.amis[0].State)
+	assert.Equal(t, recording.snapshot.SnapshotID, recording.amis[0].SnapshotID)
+	assert.Equal(t, "pending", recording.snapshot.State, "the snapshot document is pending when the provider is called")
+}
+
+func TestCreateImage_DefiniteRefusalRemovesBothDocuments(t *testing.T) {
+	f := newPendingImageFixture(t)
+	f.snapshots.SetEBSProvider(&createInUseProvider{EBSProvider: f.mem})
+
+	_, err := f.createImage(t)
+	code, _, ok := awserrors.ResolveErrorDetail(err)
+	require.True(t, ok, "unregistered error: %v", err)
+	assert.Equal(t, awserrors.ErrorIncorrectState, code)
+
+	amis, err := f.svc.MetadataStore().ListAMIs(context.Background())
+	require.NoError(t, err)
+	assert.Empty(t, amis, "a refused image leaves no AMI document")
+	snaps, err := f.svc.MetadataStore().ListAllSnapshots(context.Background())
+	require.NoError(t, err)
+	assert.Empty(t, snaps, "a refused image leaves no snapshot document")
+}
+
+func TestCreateImage_SettlesTheSnapshotAndTheAMI(t *testing.T) {
+	tests := []struct {
+		name    string
+		arrange func(f *pendingImageFixture)
+		wantErr string
+		// settle is how the provider finishes a snapshot it answered pending; empty: it did not.
+		settle      ebsprovider.SnapshotState
+		wantPending bool
+		wantSnap    string
+		wantAMI     string
+	}{
+		{name: "completed in the request", arrange: func(*pendingImageFixture) {}, wantSnap: "completed", wantAMI: ebsmetadata.AMIStateAvailable},
+		{
+			name: "answered pending, then completed", arrange: func(f *pendingImageFixture) { f.mem.AnswerSnapshotsPending(true) },
+			wantPending: true, settle: ebsprovider.SnapshotStateCompleted, wantSnap: "completed", wantAMI: ebsmetadata.AMIStateAvailable,
+		},
+		{
+			name: "answered pending, then error", arrange: func(f *pendingImageFixture) { f.mem.AnswerSnapshotsPending(true) },
+			wantPending: true, settle: ebsprovider.SnapshotStateError, wantSnap: "error", wantAMI: ebsmetadata.AMIStateFailed,
+		},
+		{
+			name: "both answers lost", arrange: func(f *pendingImageFixture) { f.mem.FailCreateSnapshotsAfterAccept(2, nil) },
+			wantErr: awserrors.ErrorServerInternal, wantPending: true, wantSnap: "completed", wantAMI: ebsmetadata.AMIStateAvailable,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newPendingImageFixture(t)
+			tc.arrange(f)
+
+			_, err := f.createImage(t)
+			if tc.wantErr != "" {
+				require.Error(t, err)
+				assert.Equal(t, tc.wantErr, err.Error())
+			} else {
+				require.NoError(t, err)
+			}
+
+			ami, snap := f.onlyAMI(t)
+			if tc.wantPending {
+				assert.Equal(t, ebsmetadata.AMIStatePending, ami.State)
+				assert.Equal(t, "pending", snap.State)
+				if tc.settle != "" {
+					require.NoError(t, f.mem.SettleSnapshot(snap.SnapshotID, tc.settle))
+				}
+				f.sweep(t)
+				ami, snap = f.onlyAMI(t)
+			}
+			assert.Equal(t, tc.wantSnap, snap.State)
+			assert.Equal(t, tc.wantAMI, ami.State)
+		})
+	}
+}
+
+// DeregisterImage is allowed on an image whose snapshot has not completed.
+func TestDeregisterImage_AllowsAPendingAMI(t *testing.T) {
+	f := newPendingImageFixture(t)
+	f.mem.AnswerSnapshotsPending(true)
+	out, err := f.createImage(t)
+	require.NoError(t, err)
+
+	_, err = f.svc.DeregisterImage(context.Background(), &ec2.DeregisterImageInput{ImageId: out.ImageId}, testAccountID)
+	require.NoError(t, err)
+	_, err = f.svc.MetadataStore().GetAMI(context.Background(), aws.StringValue(out.ImageId))
+	assert.Error(t, err, "the pending AMI is deregistered")
+}
+
+// putHookStore runs onPut once, before the write to a matching key that
+// follows skip earlier matching writes reaches the store.
+type putHookStore struct {
+	objectstore.ObjectStore
+
+	match func(key string) bool
+	skip  int
+	onPut func()
+
+	mu   sync.Mutex
+	seen int
+}
+
+func (s *putHookStore) PutObject(ctx context.Context, input *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
+	if s.match(aws.StringValue(input.Key)) {
+		s.mu.Lock()
+		fire := s.seen == s.skip
+		s.seen++
+		s.mu.Unlock()
+		if fire {
+			s.onPut()
+		}
+	}
+	return s.ObjectStore.PutObject(ctx, input)
+}
+
+// A DeregisterImage that lands between the reaper's reread of a pending AMI
+// and its write either waits the write out or is refused retryably; one that
+// succeeds is never undone by the reaper writing the settled AMI back.
+func TestDeregisterImage_DuringTheReapersAMIWriteStaysDeleted(t *testing.T) {
+	f := newPendingImageFixture(t)
+	f.mem.AnswerSnapshotsPending(true)
+	out, err := f.createImage(t)
+	require.NoError(t, err)
+	imageID := aws.StringValue(out.ImageId)
+	ami, snap := f.onlyAMI(t)
+	require.Equal(t, ebsmetadata.AMIStatePending, ami.State)
+	require.NoError(t, f.mem.SettleSnapshot(snap.SnapshotID, ebsprovider.SnapshotStateCompleted))
+
+	key, err := ebsmetadata.AMIKey(imageID)
+	require.NoError(t, err)
+	var deregErr error
+	deregistered := false
+	hooked := &putHookStore{ObjectStore: f.svc.store, match: func(k string) bool { return k == key }}
+	hooked.onPut = func() {
+		deregistered = true
+		_, deregErr = f.svc.DeregisterImage(context.Background(), &ec2.DeregisterImageInput{ImageId: out.ImageId}, testAccountID)
+	}
+	reaping := ec2snapshot.NewSnapshotServiceImplWithStore(f.svc.config, hooked, nil)
+	reaping.SetEBSProvider(f.mem)
+	f.svc.SetImageClaims(reaping)
+
+	_, err = reaping.NewPendingSnapshotReaper().Sweep(context.Background())
+	require.NoError(t, err)
+	require.True(t, deregistered, "the reaper wrote the settled AMI")
+
+	assertDeregistrationStaysDeleted(t, f, imageID, deregErr, func() {
+		_, err := reaping.NewPendingSnapshotReaper().Sweep(context.Background())
+		require.NoError(t, err)
+	})
+}
+
+// assertDeregistrationStaysDeleted checks a DeregisterImage of imageID that
+// raced a settling write, and answered deregErr: one that succeeded left the
+// AMI gone, and one refused retryably succeeds on retry and stays gone after sweep.
+func assertDeregistrationStaysDeleted(t *testing.T, f *pendingImageFixture, imageID string, deregErr error, sweep func()) {
+	t.Helper()
+	_, getErr := f.svc.MetadataStore().GetAMI(context.Background(), imageID)
+	if deregErr == nil {
+		require.True(t, objectstore.IsNoSuchKeyError(getErr), "a deregistered AMI was written back: %v", getErr)
+		return
+	}
+	code, _, ok := awserrors.ResolveErrorDetail(deregErr)
+	require.True(t, ok, "unregistered error: %v", deregErr)
+	assert.Equal(t, awserrors.ErrorIncorrectState, code, "a refused deregistration is retryable")
+	require.NoError(t, getErr)
+
+	_, err := f.svc.DeregisterImage(context.Background(), &ec2.DeregisterImageInput{ImageId: aws.String(imageID)}, testAccountID)
+	require.NoError(t, err, "the retry succeeds once the settling write is done")
+	sweep()
+	_, getErr = f.svc.MetadataStore().GetAMI(context.Background(), imageID)
+	assert.True(t, objectstore.IsNoSuchKeyError(getErr), "the retried deregistration stays deleted: %v", getErr)
+}
+
+// A DeregisterImage that lands between CreateImage's reread of its pending AMI
+// and the write making it available is refused retryably, never undone.
+func TestDeregisterImage_DuringCreateImagesAMIWriteStaysDeleted(t *testing.T) {
+	f := newPendingImageFixture(t)
+	sample, err := ebsmetadata.AMIKey("ami-0")
+	require.NoError(t, err)
+	amiKeyPrefix := sample[:strings.LastIndex(sample, "ami-0")]
+	var imageID string
+	var deregErr error
+	deregistered := false
+	// The Recorded hook's pending write is skipped; the Completed hook's is the one.
+	hooked := &putHookStore{
+		ObjectStore: f.svc.store,
+		skip:        1,
+		match:       func(k string) bool { return strings.HasPrefix(k, amiKeyPrefix) },
+	}
+	hooked.onPut = func() {
+		amis, err := f.svc.MetadataStore().ListAMIs(context.Background())
+		if err != nil || len(amis) != 1 {
+			deregErr = err
+			return
+		}
+		imageID = amis[0].ImageID
+		deregistered = true
+		_, deregErr = f.svc.DeregisterImage(context.Background(), &ec2.DeregisterImageInput{ImageId: aws.String(imageID)}, testAccountID)
+	}
+	f.svc.metadata = ebsmetadata.NewStore(hooked, testBucket)
+	settling := ec2snapshot.NewSnapshotServiceImplWithStore(f.svc.config, hooked, nil)
+	settling.SetEBSProvider(f.mem)
+	f.svc.SetSnapshotCreator(settling)
+	f.svc.SetImageClaims(settling)
+
+	out, err := f.createImage(t)
+	require.NoError(t, err)
+	require.True(t, deregistered, "CreateImage wrote the available AMI")
+	require.Equal(t, imageID, aws.StringValue(out.ImageId))
+
+	assertDeregistrationStaysDeleted(t, f, imageID, deregErr, func() { f.sweep(t) })
+}

spinifex/domains/ec2/image/provider_snapshot_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	ec2snapshot "github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
@@ -40,10 +41,42 @@ func setupProviderSnapshotImageService(t *testing.T) (*ImageServiceImpl, *object
 		},
 	}
 	svc := NewImageServiceImplWithConfig(cfg, store, nil)
-	svc.SetEBSProvider(provider)
+	useProvider(svc, provider)
 	return svc, store, provider
 }
 
+// useProvider makes p svc's provider and wires a snapshot service over svc's
+// store and p, as the daemon wires its own, for CreateImage to snapshot through.
+func useProvider(svc *ImageServiceImpl, p ebsprovider.EBSProvider) *ec2snapshot.SnapshotServiceImpl {
+	svc.SetEBSProvider(p)
+	snapshots := ec2snapshot.NewSnapshotServiceImplWithStore(svc.config, svc.store, nil)
+	snapshots.SetEBSProvider(p)
+	svc.SetSnapshotCreator(snapshots)
+	svc.SetImageClaims(snapshots)
+	return snapshots
+}
+
+// createImageOf runs CreateImage of a stopped instance whose root volume is
+// volumeID, and returns its AMI's snapshot ID when one was recorded.
+func createImageOf(t *testing.T, svc *ImageServiceImpl, volumeID, name string) (string, error) {
+	t.Helper()
+	out, err := svc.CreateImageFromInstance(CreateImageParams{
+		Input:        &ec2.CreateImageInput{InstanceId: aws.String("i-" + name), Name: aws.String(name)},
+		RootVolumeID: volumeID,
+	}, testAccountID)
+	amis, listErr := svc.MetadataStore().ListAMIs(context.Background())
+	require.NoError(t, listErr)
+	for _, ami := range amis {
+		if ami.Name == name {
+			return ami.SnapshotID, err
+		}
+	}
+	if err == nil {
+		t.Fatalf("CreateImage %s returned %s but recorded no AMI", name, aws.StringValue(out.ImageId))
+	}
+	return "", err
+}
+
 // seedProviderVolume creates a provider-backed volume plus its ebsmetadata
 // document, wiring the document's ProviderHandle to the volume's real handle
 // so the fixture cannot drift from what the provider actually issued.
@@ -100,15 +133,15 @@ func TestGetVolumeMetadata_Provider_UnknownVolumeIsNotFound(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorInvalidVolumeNotFound, err.Error())
 }
 
-// TestSnapshotVolume_Provider_CreatesThroughProvider locks that a
+// TestCreateImageFromInstance_Provider_CreatesThroughProvider locks that a
 // root-volume snapshot under a provider is created via provider.CreateSnapshot
 // and never falls through to building an embedded engine — the Predastore host
 // is a closed port, so any fallback would fail with connection refused.
-func TestSnapshotVolume_Provider_CreatesThroughProvider(t *testing.T) {
+func TestCreateImageFromInstance_Provider_CreatesThroughProvider(t *testing.T) {
 	svc, _, provider := setupProviderSnapshotImageService(t)
 	doc := seedProviderVolume(t, svc, provider, "vol-stopped001", 8)
 
-	err := svc.snapshotVolume(doc.VolumeID, "snap-stopped001", testAccountID)
+	snapshotID, err := createImageOf(t, svc, doc.VolumeID, "stopped001")
 	require.NoError(t, err, "the provider path must not fall through to the embedded engine against the closed Predastore port")
 
 	// CreateSnapshot is idempotent on (SnapshotID, VolumeID): re-issuing it
@@ -116,28 +149,28 @@ func TestSnapshotVolume_Provider_CreatesThroughProvider(t *testing.T) {
 	// confirms the snapshot genuinely exists in the provider (sourced from
 	// the right volume) instead of the call having silently no-opped.
 	snap, err := provider.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{
-		Versioned: ebsprovider.NewVersioned(), SnapshotID: "snap-stopped001", VolumeID: doc.VolumeID,
+		Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: doc.VolumeID,
 	})
 	require.NoError(t, err)
 	assert.Equal(t, doc.VolumeID, snap.SourceVolumeID)
 }
 
-// TestSnapshotVolume_Provider_PassesProviderHandleAsVolumeHandle locks
+// TestCreateImageFromInstance_Provider_PassesProviderHandleAsVolumeHandle locks
 // that the ebsmetadata document's ProviderHandle is threaded through as
 // CreateSnapshotRequest.VolumeHandle. A wrong handle must be rejected by the
 // provider (MemoryProvider treats an empty VolumeHandle as "skip the check",
 // so only a mismatched non-empty value proves the real value is forwarded).
-func TestSnapshotVolume_Provider_PassesProviderHandleAsVolumeHandle(t *testing.T) {
+func TestCreateImageFromInstance_Provider_PassesProviderHandleAsVolumeHandle(t *testing.T) {
 	svc, _, provider := setupProviderSnapshotImageService(t)
 	doc := seedProviderVolume(t, svc, provider, "vol-running001", 8)
 
-	err := svc.snapshotVolume(doc.VolumeID, "snap-running001", testAccountID)
+	_, err := createImageOf(t, svc, doc.VolumeID, "running001")
 	require.NoError(t, err, "the correct provider handle must let CreateSnapshot succeed")
 
 	doc.ProviderHandle = "wrong-handle"
 	require.NoError(t, svc.MetadataStore().PutVolume(context.Background(), doc))
 
-	err = svc.snapshotVolume(doc.VolumeID, "snap-running002", testAccountID)
+	_, err = createImageOf(t, svc, doc.VolumeID, "running002")
 	require.Error(t, err, "a mismatched provider handle must be rejected, proving the value came from the document")
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
@@ -236,16 +269,21 @@ func (p *createUnavailableProvider) CreateSnapshot(context.Context, ebsprovider.
 	return nil, &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnavailable, Message: "held by node-2"}
 }
 
-// TestSnapshotVolume_Provider_UnavailableIsRetryLater locks that the
-// CreateImage snapshot surfaces a provider's retry-later as ServiceUnavailable.
-func TestSnapshotVolume_Provider_UnavailableIsRetryLater(t *testing.T) {
+// TestCreateImageFromInstance_Provider_UnavailableLeavesThePendingSnapshot
+// locks that a CreateImage snapshot whose call and repeat both go unanswered
+// returns InternalError and leaves its document pending for the reaper.
+func TestCreateImageFromInstance_Provider_UnavailableLeavesThePendingSnapshot(t *testing.T) {
 	svc, _, provider := setupProviderSnapshotImageService(t)
 	doc := seedProviderVolume(t, svc, provider, "vol-stopped-busy", 8)
-	svc.SetEBSProvider(&createUnavailableProvider{EBSProvider: provider})
+	useProvider(svc, &createUnavailableProvider{EBSProvider: provider})
 
-	err := svc.snapshotVolume(doc.VolumeID, "snap-stopped-busy", testAccountID)
+	snapshotID, err := createImageOf(t, svc, doc.VolumeID, "stopped-busy")
 	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServiceUnavailable, err.Error())
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+	snap, err := svc.metadata.GetSnapshot(context.Background(), testAccountID, snapshotID)
+	require.NoError(t, err)
+	assert.Equal(t, "pending", snap.State)
+	assert.True(t, snap.AmbiguousCreate)
 }
 
 // createInUseProvider refuses every CreateSnapshot as volume in use.
@@ -257,14 +295,14 @@ func (p *createInUseProvider) CreateSnapshot(context.Context, ebsprovider.Create
 	return nil, &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeVolumeInUse, Message: "published"}
 }
 
-// TestSnapshotVolume_Provider_VolumeInUseIsIncorrectState locks that the
-// CreateImage snapshot tells the caller to detach a volume the provider refuses as in use.
-func TestSnapshotVolume_Provider_VolumeInUseIsIncorrectState(t *testing.T) {
+// TestCreateImageFromInstance_Provider_VolumeInUseIsIncorrectState locks that
+// the CreateImage snapshot tells the caller to detach a volume the provider refuses as in use.
+func TestCreateImageFromInstance_Provider_VolumeInUseIsIncorrectState(t *testing.T) {
 	svc, _, provider := setupProviderSnapshotImageService(t)
 	doc := seedProviderVolume(t, svc, provider, "vol-attached-inuse", 8)
-	svc.SetEBSProvider(&createInUseProvider{EBSProvider: provider})
+	useProvider(svc, &createInUseProvider{EBSProvider: provider})
 
-	err := svc.snapshotVolume(doc.VolumeID, "snap-attached-inuse", testAccountID)
+	_, err := createImageOf(t, svc, doc.VolumeID, "attached-inuse")
 
 	code, message, ok := awserrors.ResolveErrorDetail(err)
 	require.True(t, ok, "unregistered error: %v", err)
@@ -295,7 +333,7 @@ func TestCreateImageFromInstance_RunningInstanceLeavesTheDrainToTheProvider(t *t
 	svc.natsConn = nc
 	svc.config.DataDir = testutil.SocketTempDir(t)
 	counting := &countingSnapshotProvider{EBSProvider: provider}
-	svc.SetEBSProvider(counting)
+	useProvider(svc, counting)
 
 	doc := seedProviderVolume(t, svc, provider, "vol-running-drain", 8)
 	doc.State, doc.AttachedInstance = "in-use", instanceID

spinifex/domains/ec2/image/readiness_gate_test.go

@@ -0,0 +1,78 @@
+package image
+
+import (
+	"context"
+	"testing"
+
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRegisterImage_RefusesASnapshotThatIsNotCompleted(t *testing.T) {
+	tests := []struct {
+		name     string
+		snapshot ebsmetadata.Snapshot
+		wantCode string
+	}{
+		{name: "pending", snapshot: ebsmetadata.Snapshot{State: "pending"}, wantCode: awserrors.ErrorIncorrectState},
+		{name: "error", snapshot: ebsmetadata.Snapshot{State: "error"}, wantCode: awserrors.ErrorIncorrectState},
+		{name: "deleted while pending", snapshot: ebsmetadata.Snapshot{State: "pending", DeleteRequested: true}, wantCode: awserrors.ErrorInvalidSnapshotNotFound},
+		{name: "deleted after completing", snapshot: ebsmetadata.Snapshot{State: "completed", DeleteRequested: true, AmbiguousCreate: true}, wantCode: awserrors.ErrorInvalidSnapshotNotFound},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			svc, store := setupTestImageService(t)
+			snap := tc.snapshot
+			snap.SnapshotID, snap.VolumeID, snap.VolumeSize, snap.OwnerID = "snap-gate000001", "vol-gate000001", 8, testAccountID
+			putTestSnapshotDoc(t, store, snap)
+
+			_, err := svc.RegisterImage(context.Background(), validRegisterImageServiceInput(snap.SnapshotID), testAccountID)
+			code, _, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok, "unregistered error: %v", err)
+			assert.Equal(t, tc.wantCode, code)
+			amis, err := svc.MetadataStore().ListAMIs(context.Background())
+			require.NoError(t, err)
+			assert.Empty(t, amis, "a refused registration writes no AMI")
+		})
+	}
+}
+
+// An AMI whose snapshot is gone reads as gone, as an AMI with no snapshot document does.
+func TestCopyImage_RefusesASourceThatIsNotAvailable(t *testing.T) {
+	tests := []struct {
+		name     string
+		amiState string
+		hideSnap bool
+		wantCode string
+	}{
+		{name: "pending", amiState: ebsmetadata.AMIStatePending, wantCode: awserrors.ErrorInvalidAMIIDUnavailable},
+		{name: "failed", amiState: ebsmetadata.AMIStateFailed, wantCode: awserrors.ErrorInvalidAMIIDUnavailable},
+		{name: "snapshot deleted", amiState: "available", hideSnap: true, wantCode: awserrors.ErrorInvalidAMIIDNotFound},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			svc, store := setupTestImageService(t)
+			seedCopyableAMI(t, store, "ami-gate000001", "gated", testAccountID, "snap-gate000001", "vol-gate000001", 8)
+			ctx := context.Background()
+			ami, err := svc.MetadataStore().GetAMI(ctx, "ami-gate000001")
+			require.NoError(t, err)
+			ami.State = tc.amiState
+			require.NoError(t, svc.MetadataStore().PutAMI(ctx, ami))
+			if tc.hideSnap {
+				snap := readTestSnapshotDoc(t, store, testAccountID, "snap-gate000001")
+				snap.DeleteRequested = true
+				putTestSnapshotDoc(t, store, snap)
+			}
+
+			_, err = svc.CopyImage(ctx, validCopyImageServiceInput("ami-gate000001", "copy"), testAccountID)
+			code, _, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok, "unregistered error: %v", err)
+			assert.Equal(t, tc.wantCode, code)
+			amis, err := svc.MetadataStore().ListAMIs(ctx)
+			require.NoError(t, err)
+			assert.Len(t, amis, 1, "a refused copy writes no AMI")
+		})
+	}
+}

spinifex/domains/ec2/image/service_impl.go

@@ -55,6 +55,24 @@ type ImageServiceImpl struct {
 
 	uefiRecords UEFIStoreRecords
 	nodeID      string
+
+	// snapshots takes an image's snapshot through the snapshot service, so the
+	// claim and reference are the ones CreateSnapshot uses.
+	snapshots ec2snapshot.ImageSnapshotCreator
+	// imageClaims is the claim DeregisterImage shares with whatever settles a
+	// pending AMI; nil excludes nothing.
+	imageClaims ec2snapshot.ImageClaims
+}
+
+// SetSnapshotCreator wires the snapshot service CreateImage snapshots through.
+func (s *ImageServiceImpl) SetSnapshotCreator(snapshots ec2snapshot.ImageSnapshotCreator) {
+	s.snapshots = snapshots
+}
+
+// SetImageClaims wires the image claims of the snapshot service whose reaper
+// settles this service's pending AMIs.
+func (s *ImageServiceImpl) SetImageClaims(claims ec2snapshot.ImageClaims) {
+	s.imageClaims = claims
 }
 
 // SetEBSProvider injects the provider boundary used by the control plane.
@@ -461,12 +479,14 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 		}
 	}
 
-	// Step 2: Snapshot root volume
-	if err := s.snapshotVolume(params.RootVolumeID, snapshotID, accountID); err != nil {
-		return nil, err
+	// Step 2: Read source volume config for size
+	if s.config == nil {
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if s.snapshots == nil || s.imageClaims == nil {
+		slog.Error("CreateImageFromInstance: no snapshot service wired", "amiId", amiID)
+		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-
-	// Step 3: Read source volume config for size
 	volMeta, err := s.getVolumeMetadata(context.Background(), accountID, params.RootVolumeID)
 	if err != nil {
 		slog.Error("CreateImageFromInstance: failed to read volume metadata", "volumeId", params.RootVolumeID, "err", err)
@@ -474,7 +494,7 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 	}
 	volumeSizeGiB := volMeta.CapacityGiB
 
-	// Step 4: Read source AMI config for architecture, platform, etc.
+	// Step 3: Read source AMI config for architecture, platform, etc.
 	sourceAMI := ebsmetadata.AMI{
 		Architecture:    "x86_64",
 		PlatformDetails: "Linux/UNIX",
@@ -489,13 +509,6 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 		sourceAMI = srcCfg
 	}
 
-	// Step 5: Store snapshot metadata
-	if err := s.putSnapshotMetadata(context.Background(), snapshotID, params.RootVolumeID, volumeSizeGiB, accountID); err != nil {
-		slog.Error("CreateImageFromInstance: failed to write snapshot metadata", "snapshotId", snapshotID, "err", err)
-		return nil, errors.New(awserrors.ErrorServerInternal)
-	}
-
-	// Step 6: Build and store AMI config
 	meta := ebsmetadata.AMI{
 		ImageID:         amiID,
 		Name:            name,
@@ -513,57 +526,41 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 		EnaSupport:   aws.Bool(true),
 		Distro:       sourceAMI.Distro,
 		DistroFamily: sourceAMI.DistroFamily,
-		// Snapshot succeeded before this point, so the image is complete.
-		State:    "available",
+		// Pending until its snapshot completes, here or in the snapshot reaper.
+		State:    ebsmetadata.AMIStatePending,
 		UEFIData: uefiData,
 	}
 
-	if err := s.putAMIConfig(context.Background(), amiID, meta); err != nil {
+	// Step 4: Snapshot the root volume. Its document and reference, then the
+	// AMI's document, are written pending before the provider is called, so an
+	// accepted snapshot whose answer is lost is still settled by the reaper.
+	ctx := context.Background()
+	snap, err := s.snapshots.CreateImageSnapshot(ctx, volMeta, snapshotID, accountID, "Created by CreateImage("+aws.StringValue(input.InstanceId)+") for "+amiID,
+		ec2snapshot.ImageSnapshotHooks{
+			Recorded:  func(ctx context.Context) error { return s.putAMIConfig(ctx, amiID, meta) },
+			Refused:   func(ctx context.Context) { s.removePendingAMI(ctx, amiID) },
+			Completed: func(ctx context.Context) { s.imageClaims.SettlePendingAMI(ctx, amiID, ebsmetadata.AMIStateAvailable) },
+		})
+	if err != nil {
 		// A UEFI object left under the image's prefix is in the orphan report.
-		slog.Error("CreateImageFromInstance: failed to store AMI config", "amiId", amiID, "err", err)
-		return nil, errors.New(awserrors.ErrorServerInternal)
+		slog.Error("CreateImageFromInstance: snapshot failed", "amiId", amiID, "snapshotId", snapshotID, "err", err)
+		return nil, err
 	}
 
-	slog.Info("CreateImageFromInstance completed", "amiId", amiID, "snapshotId", snapshotID)
+	slog.Info("CreateImageFromInstance completed", "amiId", amiID, "snapshotId", snapshotID, "snapshotState", aws.StringValue(snap.State))
 
 	return &ec2.CreateImageOutput{
 		ImageId: aws.String(amiID),
 	}, nil
 }
 
-// snapshotVolume snapshots the root volume through the EBS provider, whether
-// or not the instance is running. A provider that buffers writes drains the
-// volume inside its own CreateSnapshot, so no drain is issued here.
-func (s *ImageServiceImpl) snapshotVolume(volumeID, snapshotID, accountID string) error {
-	if s.config == nil {
-		return errors.New(awserrors.ErrorServerInternal)
+// removePendingAMI deletes the AMI document of a CreateImage whose snapshot
+// the provider refused, before the snapshot's own document goes.
+func (s *ImageServiceImpl) removePendingAMI(ctx context.Context, amiID string) {
+	if err := s.metadata.DeleteAMI(ctx, amiID); err != nil && !objectstore.IsNoSuchKeyError(err) {
+		slog.ErrorContext(ctx, "CreateImageFromInstance: failed to remove a refused image's AMI document; the reaper marks it failed",
+			"amiId", amiID, "err", err)
 	}
-
-	volMeta, err := s.getVolumeMetadata(context.Background(), accountID, volumeID)
-	if err != nil {
-		slog.Error("snapshotVolume: failed to read volume metadata", "volumeId", volumeID, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-
-	return s.createSnapshot(context.Background(), volMeta, snapshotID, "snapshotVolume")
-}
-
-// createSnapshot delegates snapshot creation for volMeta to the EBS provider.
-// op names the calling path so a failure is attributable in the log.
-func (s *ImageServiceImpl) createSnapshot(ctx context.Context, volMeta ebsmetadata.Volume, snapshotID, op string) error {
-	if s.provider == nil {
-		slog.ErrorContext(ctx, "no EBS provider configured", "op", op)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	created, err := s.provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{
-		Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: volMeta.VolumeID, VolumeHandle: volMeta.ProviderHandle,
-	})
-	if err != nil || created == nil {
-		slog.ErrorContext(ctx, "provider CreateSnapshot failed", "op", op, "volumeId", volMeta.VolumeID, "snapshotId", snapshotID, "err", err)
-		return ec2snapshot.ProviderCreateError(volMeta.VolumeID, err)
-	}
-	slog.InfoContext(ctx, "snapshot created", "op", op, "volumeId", volMeta.VolumeID, "snapshotId", snapshotID)
-	return nil
 }
 
 // getVolumeMetadata reads a volume's control-plane document, remapping a
@@ -648,6 +645,9 @@ func (s *ImageServiceImpl) GetAMISourceVolumeID(ctx context.Context, imageID str
 			"imageId", imageID, "snapshotId", meta.SnapshotID, "err", err)
 		return "", errors.New(awserrors.ErrorServerInternal)
 	}
+	if snapCfg.Hidden() {
+		return "", errors.New(awserrors.ErrorInvalidSnapshotNotFound)
+	}
 	if snapCfg.VolumeID == "" {
 		slog.ErrorContext(ctx, "GetAMISourceVolumeID: snapshot metadata has no source volume",
 			"imageId", imageID, "snapshotId", meta.SnapshotID)
@@ -815,6 +815,10 @@ func (s *ImageServiceImpl) CopyImage(ctx context.Context, input *ec2.CopyImageIn
 	if !callerCanReadAMI(srcMeta, accountID) {
 		return nil, errors.New(awserrors.ErrorInvalidAMIIDNotFound)
 	}
+	// The copy shares the source's snapshot, so it is no readier than it.
+	if !srcMeta.Available() {
+		return nil, srcMeta.UnavailableError(sourceImageID)
+	}
 
 	// Orphaned source (missing SnapshotID, or snapshot gone/corrupt) is
 	// reported as NotFound — don't leak the half-broken state.
@@ -842,6 +846,9 @@ func (s *ImageServiceImpl) CopyImage(ctx context.Context, input *ec2.CopyImageIn
 			return nil, errors.New(awserrors.ErrorServerInternal)
 		}
 	}
+	if srcSnap.Hidden() {
+		return nil, errors.New(awserrors.ErrorInvalidAMIIDNotFound)
+	}
 
 	if exists, err := s.amiNameExists(ctx, name); err != nil {
 		slog.ErrorContext(ctx, "CopyImage: failed to check AMI name uniqueness", "name", name, "err", err)
@@ -949,9 +956,9 @@ func mergeCopyImageTags(srcTags map[string]string, specs []*ec2.TagSpecification
 	return merged
 }
 
-// DescribeImageAttribute supports description and blockDeviceMapping only.
-// Cross-account reads return NotFound so the caller can't learn the ID exists
-// in another account.
+// DescribeImageAttribute leaves Value nil for an attribute the AMI never records,
+// as AWS does for an unset one. Cross-account reads return NotFound so the
+// caller can't learn the ID exists in another account.
 func (s *ImageServiceImpl) DescribeImageAttribute(ctx context.Context, input *ec2.DescribeImageAttributeInput, accountID string) (*ec2.DescribeImageAttributeOutput, error) {
 	if input == nil || input.ImageId == nil || *input.ImageId == "" ||
 		input.Attribute == nil || *input.Attribute == "" {
@@ -980,7 +987,15 @@ func (s *ImageServiceImpl) DescribeImageAttribute(ctx context.Context, input *ec
 
 	switch attribute {
 	case ec2.ImageAttributeNameDescription:
-		output.Description = &ec2.AttributeValue{Value: aws.String(meta.Description)}
+		output.Description = &ec2.AttributeValue{Value: nonEmptyString(meta.Description)}
+	case ec2.ImageAttributeNameLaunchPermission:
+		output.LaunchPermissions = []*ec2.LaunchPermission{}
+	case ec2.ImageAttributeNameBootMode:
+		output.BootMode = &ec2.AttributeValue{Value: nonEmptyString(meta.BootMode)}
+	case ec2.ImageAttributeNameImdsSupport:
+		output.ImdsSupport = &ec2.AttributeValue{}
+	case ec2.ImageAttributeNameLastLaunchedTime:
+		output.LastLaunchedTime = &ec2.AttributeValue{}
 	case ec2.ImageAttributeNameBlockDeviceMapping:
 		output.BlockDeviceMappings = synthesizeRootBlockDeviceMapping(meta, s.clusterEncryptionEnabled())
 	default:
@@ -991,6 +1006,14 @@ func (s *ImageServiceImpl) DescribeImageAttribute(ctx context.Context, input *ec
 	return output, nil
 }
 
+// nonEmptyString returns nil for "", so an unset attribute renders as {}.
+func nonEmptyString(v string) *string {
+	if v == "" {
+		return nil
+	}
+	return &v
+}
+
 // clusterEncryptionEnabled reports whether the daemon has a viperblock master
 // key configured. AMI metadata carries no per-image encryption flag, so block
 // device synthesis falls back to this cluster-level posture.
@@ -1063,6 +1086,14 @@ func (s *ImageServiceImpl) RegisterImage(ctx context.Context, input *ec2.Registe
 		slog.ErrorContext(ctx, "RegisterImage: failed to read snapshot metadata", "snapshotId", snapshotID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
+	if snapCfg.Hidden() {
+		return nil, errors.New(awserrors.ErrorInvalidSnapshotNotFound)
+	}
+	// An image registered on a snapshot that is not completed could never be
+	// launched from, and none is written pending outside CreateImage.
+	if !snapCfg.Ready() {
+		return nil, snapCfg.NotReadyError()
+	}
 
 	snapSizeGiB := uint64(0)
 	if snapCfg.VolumeSize > 0 {
@@ -1175,6 +1206,16 @@ func (s *ImageServiceImpl) DeregisterImage(ctx context.Context, input *ec2.Dereg
 		return nil, err
 	}
 
+	// Held across both deletes, so the reaper cannot write back a settled AMI
+	// it read before them.
+	if s.imageClaims != nil {
+		unlock, err := s.imageClaims.LockImage(ctx, imageID)
+		if err != nil {
+			return nil, err
+		}
+		defer unlock()
+	}
+
 	// Only the image's own prefix: a copy or a source instance has its own.
 	if n, err := uefistore.DeleteObjects(ctx, s.store, s.bucketName, imageID); err != nil {
 		slog.ErrorContext(ctx, "DeregisterImage: failed to delete the UEFI variable store", "imageId", imageID, "deleted", n, "err", err)

spinifex/domains/ec2/image/service_impl_test.go

@@ -6,8 +6,6 @@ import (
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"io"
 	"log/slog"
-	"net/http"
-	"net/http/httptest"
 	"strconv"
 	"strings"
 	"testing"
@@ -154,6 +152,24 @@ func TestCreateImageFromInstance_RunningInstance_NoNATS(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
 }
 
+// An image service the daemon gave no snapshot service fails CreateImage with
+// InternalError before it records anything.
+func TestCreateImageFromInstance_NoSnapshotServiceIsInternalError(t *testing.T) {
+	store := objectstore.NewMemoryObjectStore()
+	svc := NewImageServiceImplWithConfig(&config.Config{Predastore: config.PredastoreConfig{Bucket: testBucket}}, store, nil)
+	createTestVolumeConfig(t, store, "vol-root123", 10)
+
+	_, err := svc.CreateImageFromInstance(CreateImageParams{
+		Input:        &ec2.CreateImageInput{InstanceId: aws.String("i-test123"), Name: aws.String("my-image")},
+		RootVolumeID: "vol-root123",
+	}, testAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+	amis, err := svc.MetadataStore().ListAMIs(context.Background())
+	require.NoError(t, err)
+	assert.Empty(t, amis)
+}
+
 func TestCreateImageFromInstance_StoppedInstance_NoConfig(t *testing.T) {
 	svc, _ := setupTestImageService(t)
 
@@ -2298,7 +2314,7 @@ func TestDescribeImageAttribute_UnsupportedAttribute(t *testing.T) {
 
 	createTestAMIConfigWithOwner(t, store, "ami-unsup01", "u", testAccountID)
 
-	for _, attr := range []string{"launchPermission", "bootMode", "kernel", "ramdisk"} {
+	for _, attr := range []string{"kernel", "ramdisk"} {
 		t.Run(attr, func(t *testing.T) {
 			_, err := svc.DescribeImageAttribute(context.Background(), &ec2.DescribeImageAttributeInput{
 				ImageId:   aws.String("ami-unsup01"),
@@ -2310,6 +2326,66 @@ func TestDescribeImageAttribute_UnsupportedAttribute(t *testing.T) {
 	}
 }
 
+func describeImageAttr(t *testing.T, svc *ImageServiceImpl, imageID, attr string) *ec2.DescribeImageAttributeOutput {
+	t.Helper()
+	out, err := svc.DescribeImageAttribute(context.Background(), &ec2.DescribeImageAttributeInput{
+		ImageId:   aws.String(imageID),
+		Attribute: aws.String(attr),
+	}, testAccountID)
+	require.NoError(t, err)
+	return out
+}
+
+func TestDescribeImageAttribute_EmptyDescriptionHasNilValue(t *testing.T) {
+	svc, store := setupTestImageService(t)
+	createTestAMIConfigWithOwner(t, store, "ami-nodesc01", "u", testAccountID)
+
+	out := describeImageAttr(t, svc, "ami-nodesc01", "description")
+	require.NotNil(t, out.Description)
+	assert.Nil(t, out.Description.Value)
+}
+
+func TestDescribeImageAttribute_LaunchPermission(t *testing.T) {
+	svc, store := setupTestImageService(t)
+	createTestAMIConfigWithOwner(t, store, "ami-lp01", "u", testAccountID)
+
+	own := describeImageAttr(t, svc, "ami-lp01", "launchPermission")
+	require.NotNil(t, own.LaunchPermissions)
+	assert.Empty(t, own.LaunchPermissions)
+}
+
+func TestDescribeImageAttribute_BootMode(t *testing.T) {
+	svc, store := setupTestImageService(t)
+	createTestAMIConfigRich(t, store, ebsmetadata.AMI{ImageID: "ami-bm01", Name: "bm", RootDeviceType: "ebs", ImageOwnerAlias: testAccountID, BootMode: "uefi-preferred"})
+	createTestAMIConfigWithOwner(t, store, "ami-bm02", "u", testAccountID)
+
+	set := describeImageAttr(t, svc, "ami-bm01", "bootMode")
+	require.NotNil(t, set.BootMode)
+	assert.Equal(t, "uefi-preferred", aws.StringValue(set.BootMode.Value))
+
+	unset := describeImageAttr(t, svc, "ami-bm02", "bootMode")
+	require.NotNil(t, unset.BootMode)
+	assert.Nil(t, unset.BootMode.Value)
+}
+
+func TestDescribeImageAttribute_ImdsSupport(t *testing.T) {
+	svc, store := setupTestImageService(t)
+	createTestAMIConfigWithOwner(t, store, "ami-imds01", "u", testAccountID)
+
+	out := describeImageAttr(t, svc, "ami-imds01", "imdsSupport")
+	require.NotNil(t, out.ImdsSupport)
+	assert.Nil(t, out.ImdsSupport.Value)
+}
+
+func TestDescribeImageAttribute_LastLaunchedTime(t *testing.T) {
+	svc, store := setupTestImageService(t)
+	createTestAMIConfigWithOwner(t, store, "ami-llt01", "u", testAccountID)
+
+	out := describeImageAttr(t, svc, "ami-llt01", "lastLaunchedTime")
+	require.NotNil(t, out.LastLaunchedTime)
+	assert.Nil(t, out.LastLaunchedTime.Value)
+}
+
 func TestModifyImageAttribute_Description(t *testing.T) {
 	svc, store := setupTestImageService(t)
 
@@ -2482,8 +2558,7 @@ func TestResetImageAttribute_Description(t *testing.T) {
 	}, testAccountID)
 	require.NoError(t, err)
 	require.NotNil(t, out.Description)
-	require.NotNil(t, out.Description.Value)
-	assert.Empty(t, *out.Description.Value)
+	assert.Nil(t, out.Description.Value)
 }
 
 func TestResetImageAttribute_CrossAccount(t *testing.T) {
@@ -2550,114 +2625,3 @@ func TestResetImageAttribute_MissingParameters(t *testing.T) {
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorMissingParameter, err.Error())
 }
-
-func TestSnapshotVolume_VolumeConfigMissing(t *testing.T) {
-	store := objectstore.NewMemoryObjectStore()
-	svc := &ImageServiceImpl{
-		config:     &config.Config{},
-		store:      store,
-		bucketName: testBucket,
-	}
-	err := svc.snapshotVolume("vol-missing", "snap-s1", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-func TestSnapshotVolume_BackendInitFails(t *testing.T) {
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusForbidden)
-	}))
-	defer srv.Close()
-
-	store := objectstore.NewMemoryObjectStore()
-	svc := &ImageServiceImpl{
-		config: &config.Config{
-			Predastore: config.PredastoreConfig{
-				Host:   srv.URL,
-				Region: "us-east-1",
-				Bucket: testBucket,
-			},
-		},
-		store:      store,
-		bucketName: testBucket,
-	}
-	createTestVolumeConfig(t, store, "vol-stopped1", 10)
-
-	err := svc.snapshotVolume("vol-stopped1", "snap-s2", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-// imageS3MockServer passes ListObjectsV2 (Backend.Init) but returns 404 for everything else.
-func imageS3MockServer(t *testing.T) *httptest.Server {
-	t.Helper()
-	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if strings.Contains(r.URL.RawQuery, "list-type=2") {
-			w.Header().Set("Content-Type", "application/xml")
-			w.WriteHeader(http.StatusOK)
-			io.WriteString(w, `<?xml version="1.0" encoding="UTF-8"?>`+
-				`<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">`+
-				`<Name>test-bucket</Name><KeyCount>0</KeyCount>`+
-				`<MaxKeys>1000</MaxKeys><IsTruncated>false</IsTruncated></ListBucketResult>`)
-			return
-		}
-		w.Header().Set("Content-Type", "application/xml")
-		w.WriteHeader(http.StatusNotFound)
-		io.WriteString(w, `<?xml version="1.0" encoding="UTF-8"?>`+
-			`<Error><Code>NoSuchKey</Code>`+
-			`<Message>The specified key does not exist.</Message></Error>`)
-	}))
-}
-
-func TestSnapshotVolume_EncryptionKeyLoadError(t *testing.T) {
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusOK)
-	}))
-	defer srv.Close()
-
-	store := objectstore.NewMemoryObjectStore()
-	svc := &ImageServiceImpl{
-		config: &config.Config{
-			Predastore: config.PredastoreConfig{
-				Host:   srv.URL,
-				Region: "us-east-1",
-				Bucket: testBucket,
-			},
-			Viperblock: config.ViperblockConfig{
-				EncryptionKeyFile: "/nonexistent-image-stop-key.bin",
-			},
-		},
-		store:      store,
-		bucketName: testBucket,
-	}
-	createTestVolumeConfig(t, store, "vol-stopped-enckey", 10)
-
-	err := svc.snapshotVolume("vol-stopped-enckey", "snap-stop-enckey", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-func TestSnapshotVolume_LoadStateFails(t *testing.T) {
-	srv := imageS3MockServer(t)
-	defer srv.Close()
-
-	store := objectstore.NewMemoryObjectStore()
-	svc := &ImageServiceImpl{
-		config: &config.Config{
-			Predastore: config.PredastoreConfig{
-				Host:      srv.URL,
-				Region:    "us-east-1",
-				Bucket:    testBucket,
-				AccessKey: "test-key",
-				SecretKey: "test-secret",
-			},
-		},
-		store:      store,
-		bucketName: testBucket,
-	}
-	createTestVolumeConfig(t, store, "vol-stopped-lsf", 10)
-
-	err := svc.snapshotVolume("vol-stopped-lsf", "snap-stop-lsf", testAccountID)
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}

spinifex/domains/ec2/image/uefi_test.go

@@ -55,7 +55,7 @@ func newUEFIImageFixture(t *testing.T) *uefiImageFixture {
 	svc.SetUEFIStores(records, "node-a")
 	volume := seedProviderVolume(t, svc, provider, "vol-uefi001", 8)
 	snapshots := &countingSnapshots{EBSProvider: provider}
-	svc.SetEBSProvider(snapshots)
+	useProvider(svc, snapshots)
 	return &uefiImageFixture{svc: svc, snapshots: snapshots, objects: objects, records: records, volume: volume}
 }
 

spinifex/domains/ec2/instance/launch_claimed_unsealed_test.go

@@ -10,7 +10,6 @@ import (
 	"testing"
 	"time"
 
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
 	"github.com/stretchr/testify/assert"
@@ -37,7 +36,7 @@ func refusedLaunch(t *testing.T, checker *fakeUnsealedChecker) error {
 
 	// An instance type this node does not offer is the simplest capacity refusal.
 	instance := &vm.VM{ID: "i-held", AccountID: "acc", InstanceType: "no.such.type"}
-	instance.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-root", Boot: true}, {Name: "vol-data"}}
+	instance.EBSRequests.Requests = []vm.EBSRequest{{Name: "vol-root", Boot: true}, {Name: "vol-data"}}
 	return svc.launchClaimedInstance(t.Context(), instance, func() {})
 }
 
@@ -87,7 +86,7 @@ func TestLaunchClaimedInstance_UnansweredCheckIsBoundedAndFailsSafe(t *testing.T
 	svc, _ := claimedRelaunchService(t, func(*vm.VM) error { return nil })
 	svc.SetUnsealedWritesChecker(blockingUnsealedChecker{}, "node-a")
 	instance := &vm.VM{ID: "i-wedged", AccountID: "acc", InstanceType: "no.such.type"}
-	instance.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-root", Boot: true}}
+	instance.EBSRequests.Requests = []vm.EBSRequest{{Name: "vol-root", Boot: true}}
 
 	start := time.Now()
 	err := svc.launchClaimedInstance(t.Context(), instance, func() {})

spinifex/domains/ec2/instance/projection.go

@@ -201,6 +201,7 @@ func ProjectInstance(v *vm.VM, cfg InstanceProjection) (inst *ec2.Instance, stat
 				}
 			}
 
+			projectInterfaceFields(&nicCopy, v.AccountID)
 			nics = append(nics, &nicCopy)
 		}
 		instanceCopy.NetworkInterfaces = nics
@@ -209,6 +210,30 @@ func ProjectInstance(v *vm.VM, cfg InstanceProjection) (inst *ec2.Instance, stat
 	return &instanceCopy, stateMapped
 }
 
+// projectInterfaceFields stamps the fields AWS reports on every interface of an
+// instance, deriving PrivateIpAddresses from the primary address and its
+// association so the list cannot disagree with them. Mutates nic, not Attachment.
+func projectInterfaceFields(nic *ec2.InstanceNetworkInterface, ownerID string) {
+	nic.InterfaceType = aws.String(ec2.NetworkInterfaceTypeInterface)
+	nic.OwnerId = aws.String(ownerID)
+	if nic.Description == nil {
+		nic.Description = aws.String("")
+	}
+	if nic.Attachment != nil {
+		attachment := *nic.Attachment
+		attachment.NetworkCardIndex = aws.Int64(0)
+		nic.Attachment = &attachment
+	}
+	nic.PrivateIpAddresses = nil
+	if nic.PrivateIpAddress != nil {
+		nic.PrivateIpAddresses = []*ec2.InstancePrivateIpAddress{{
+			Primary:          aws.Bool(true),
+			PrivateIpAddress: nic.PrivateIpAddress,
+			Association:      nic.Association,
+		}}
+	}
+}
+
 // ParseInstanceIDFilter builds the set of instance IDs to filter on, rejecting
 // any ID without the "i-" prefix (including the empty string) as malformed. Nil
 // entries carry no constraint and are skipped. Empty input yields an empty set,

spinifex/domains/ec2/instance/projection_test.go

@@ -295,6 +295,40 @@ func TestProjectInstance_PrimaryNICAssociation(t *testing.T) {
 	})
 }
 
+// An interface stored without the launch-time fields still reports the ones
+// projected at describe time, and its PrivateIpAddresses entry carries the association only while the
+// public address is held.
+func TestProjectInstance_InterfaceFields(t *testing.T) {
+	v := fullVM(vm.StateRunning)
+	v.Instance.NetworkInterfaces = []*ec2.InstanceNetworkInterface{{
+		NetworkInterfaceId: aws.String("eni-primary"),
+		PrivateIpAddress:   aws.String("10.0.0.5"),
+		Attachment:         &ec2.InstanceNetworkInterfaceAttachment{DeviceIndex: aws.Int64(0)},
+	}}
+
+	running, _ := ProjectInstance(v, runningCfg())
+	stopped, _ := ProjectInstance(v, stoppedCfg(80, "stopped"))
+
+	nic := running.NetworkInterfaces[0]
+	assert.Equal(t, "interface", aws.StringValue(nic.InterfaceType))
+	assert.Equal(t, "123456789012", aws.StringValue(nic.OwnerId))
+	require.NotNil(t, nic.Description)
+	assert.Empty(t, *nic.Description)
+	require.NotNil(t, nic.Attachment.NetworkCardIndex)
+	assert.Equal(t, int64(0), *nic.Attachment.NetworkCardIndex)
+	require.Len(t, nic.PrivateIpAddresses, 1)
+	assert.True(t, aws.BoolValue(nic.PrivateIpAddresses[0].Primary))
+	assert.Equal(t, "10.0.0.5", aws.StringValue(nic.PrivateIpAddresses[0].PrivateIpAddress))
+	require.NotNil(t, nic.PrivateIpAddresses[0].Association)
+	assert.Equal(t, "203.0.113.7", aws.StringValue(nic.PrivateIpAddresses[0].Association.PublicIp))
+
+	require.Len(t, stopped.NetworkInterfaces[0].PrivateIpAddresses, 1)
+	assert.Nil(t, stopped.NetworkInterfaces[0].PrivateIpAddresses[0].Association)
+
+	assert.Nil(t, v.Instance.NetworkInterfaces[0].Attachment.NetworkCardIndex,
+		"the attachment is shared with the stored instance and must be copied before it is stamped")
+}
+
 // TestProjectInstance_EnclaveOptionsAlwaysDisabled covers the permanent
 // enclave_options {} Terraform diff: AWS always reports the block, and
 // Spinifex never enables it, on both the running and the stopped/KV paths.

spinifex/domains/ec2/instance/service.go

@@ -233,6 +233,7 @@ type ENIInfo struct {
 	VpcID              string
 	PrivateIpAddress   string
 	MacAddress         string
+	Description        string
 	Status             string
 	SecurityGroupIDs   []string
 	// PublicIpAddress / PublicIpPool are the address the interface is currently

spinifex/domains/ec2/instance/service_impl.go

@@ -4,7 +4,6 @@ import (
 	"cmp"
 	"context"
 	"encoding/base64"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/tagmirror"
@@ -20,7 +19,6 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/bluebottle/pkg/safecast"
 	"github.com/mulgadc/spinifex/contracts/ec2/v1"
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/dns"
 	ebsencryption "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/encryption"
@@ -699,6 +697,12 @@ func (s *InstanceServiceImpl) PrepareRunInstances(ctx context.Context, input *ec
 		slog.WarnContext(ctx, "PrepareRunInstances: AMI not owned by caller", "imageId", *input.ImageId, "amiOwner", amiOwner, "accountID", accountID)
 		return nil, nil, nil, errors.New(awserrors.ErrorInvalidAMIIDNotFound)
 	}
+	// A CreateImage AMI is pending until its snapshot completes, and failed if
+	// it errors; neither has a root volume to clone.
+	if !amiMeta.Available() {
+		slog.WarnContext(ctx, "PrepareRunInstances: AMI is not available", "imageId", *input.ImageId, "state", amiMeta.State)
+		return nil, nil, nil, amiMeta.UnavailableError(*input.ImageId)
+	}
 	if err := checkImageArchitecture(instanceType, amiMeta.Architecture); err != nil {
 		slog.WarnContext(ctx, "PrepareRunInstances: AMI architecture does not match instance type", "imageId", *input.ImageId, "InstanceType", *input.InstanceType, "amiArchitecture", amiMeta.Architecture)
 		return nil, nil, nil, err
@@ -901,7 +905,8 @@ func (s *InstanceServiceImpl) PrepareRunInstances(ctx context.Context, input *ec
 			instance.ENIMac = *eni.MacAddress
 			autoCreatedENI[instance.ID] = true
 
-			if _, attachErr := s.eniCreator.AttachENI(ctx, accountID, instance.ENIId, instance.ID, 0); attachErr != nil {
+			attachmentID, attachErr := s.eniCreator.AttachENI(ctx, accountID, instance.ENIId, instance.ID, 0)
+			if attachErr != nil {
 				// Without the attachment RegisterTargets silently drops the target;
 				// aborting prevents a leak of the auto-assigned EIP.
 				slog.ErrorContext(ctx, "PrepareRunInstances: AttachENI failed — aborting launch",
@@ -919,21 +924,24 @@ func (s *InstanceServiceImpl) PrepareRunInstances(ctx context.Context, input *ec
 			ec2Instance.SetSubnetId(*input.SubnetId)
 			ec2Instance.SetVpcId(*eni.VpcId)
 			ec2Instance.SecurityGroups = eni.Groups
-			ec2Instance.NetworkInterfaces = []*ec2.InstanceNetworkInterface{
-				{
-					NetworkInterfaceId: eni.NetworkInterfaceId,
-					PrivateIpAddress:   eni.PrivateIpAddress,
-					MacAddress:         eni.MacAddress,
-					SubnetId:           input.SubnetId,
-					VpcId:              eni.VpcId,
-					Status:             aws.String("in-use"),
-					Groups:             eni.Groups,
-					Attachment: &ec2.InstanceNetworkInterfaceAttachment{
-						DeviceIndex: aws.Int64(0),
-						Status:      aws.String("attached"),
-					},
+			primaryNIC := &ec2.InstanceNetworkInterface{
+				NetworkInterfaceId: eni.NetworkInterfaceId,
+				PrivateIpAddress:   eni.PrivateIpAddress,
+				MacAddress:         eni.MacAddress,
+				SubnetId:           input.SubnetId,
+				VpcId:              eni.VpcId,
+				Status:             aws.String("in-use"),
+				Groups:             eni.Groups,
+				Attachment: &ec2.InstanceNetworkInterfaceAttachment{
+					AttachmentId:        aws.String(attachmentID),
+					AttachTime:          ec2Instance.LaunchTime,
+					DeleteOnTermination: aws.Bool(true),
+					DeviceIndex:         aws.Int64(0),
+					Status:              aws.String("attached"),
 				},
 			}
+			projectInterfaceFields(primaryNIC, accountID)
+			ec2Instance.NetworkInterfaces = []*ec2.InstanceNetworkInterface{primaryNIC}
 
 			slog.InfoContext(ctx, "PrepareRunInstances: auto-created ENI for VPC instance",
 				"instanceId", instance.ID,
@@ -1057,7 +1065,8 @@ func (s *InstanceServiceImpl) attachPreCreatedENI(ctx context.Context, accountID
 	if eniInfo.Status == "in-use" {
 		return errors.New(awserrors.ErrorInvalidNetworkInterfaceInUse)
 	}
-	if _, err := s.eniCreator.AttachENI(ctx, accountID, eniID, instance.ID, 0); err != nil {
+	attachmentID, err := s.eniCreator.AttachENI(ctx, accountID, eniID, instance.ID, 0)
+	if err != nil {
 		return err
 	}
 
@@ -1074,18 +1083,24 @@ func (s *InstanceServiceImpl) attachPreCreatedENI(ctx context.Context, accountID
 		}
 		ec2Instance.SecurityGroups = groups
 	}
-	ec2Instance.NetworkInterfaces = []*ec2.InstanceNetworkInterface{{
+	primaryNIC := &ec2.InstanceNetworkInterface{
 		NetworkInterfaceId: aws.String(eniID),
 		PrivateIpAddress:   aws.String(eniInfo.PrivateIpAddress),
 		MacAddress:         aws.String(eniInfo.MacAddress),
 		SubnetId:           aws.String(eniInfo.SubnetID),
 		VpcId:              aws.String(eniInfo.VpcID),
 		Status:             aws.String("in-use"),
+		Description:        aws.String(eniInfo.Description),
 		Attachment: &ec2.InstanceNetworkInterfaceAttachment{
-			DeviceIndex: aws.Int64(0),
-			Status:      aws.String("attached"),
+			AttachmentId:        aws.String(attachmentID),
+			AttachTime:          ec2Instance.LaunchTime,
+			DeleteOnTermination: aws.Bool(eniInfo.DeleteOnTermination),
+			DeviceIndex:         aws.Int64(0),
+			Status:              aws.String("attached"),
 		},
-	}}
+	}
+	projectInterfaceFields(primaryNIC, accountID)
+	ec2Instance.NetworkInterfaces = []*ec2.InstanceNetworkInterface{primaryNIC}
 
 	slog.InfoContext(ctx, "PrepareRunInstances: attached pre-created ENI",
 		"instanceId", instance.ID,
@@ -1789,7 +1804,7 @@ func (s *InstanceServiceImpl) prepareRootVolume(ctx context.Context, input *ec2.
 // metadata, which is what DescribeVolumes reports as Attachments[].Device.
 func appendRootEBSRequest(instance *vm.VM, volumeID, deviceName string, deleteOnTermination bool) {
 	instance.EBSRequests.Mu.Lock()
-	instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{
+	instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, vm.EBSRequest{
 		Name:                volumeID,
 		DeviceName:          deviceName,
 		Boot:                true,
@@ -1803,7 +1818,7 @@ func appendRootEBSRequest(instance *vm.VM, volumeID, deviceName string, deleteOn
 // Attachments[].Device.
 func appendDataEBSRequest(instance *vm.VM, volumeID, deviceName string, deleteOnTermination bool) {
 	instance.EBSRequests.Mu.Lock()
-	instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{
+	instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, vm.EBSRequest{
 		Name:                volumeID,
 		DeviceName:          deviceName,
 		DeleteOnTermination: deleteOnTermination,
@@ -3021,19 +3036,11 @@ func (s *InstanceServiceImpl) deleteInstanceVolumes(ctx context.Context, instanc
 	}
 
 	for _, ebsRequest := range instance.EBSRequests.Requests {
-		// Internal volumes (EFI) are always cleaned up via ebs.delete.
+		// No release creates an EFI request and start refuses one, so nothing
+		// serves it; its objects go with the root volume or the orphan report.
 		if ebsRequest.EFI {
-			ebsDeleteData, err := json.Marshal(viperblocklegacyv1.EBSDeleteRequest{Volume: ebsRequest.Name})
-			if err != nil {
-				slog.ErrorContext(ctx, "TerminateStoppedInstance: failed to marshal ebs.delete request", "name", ebsRequest.Name, "err", err)
-				continue
-			}
-			deleteMsg, err := s.natsConn.Request(viperblocklegacyv1.DeleteSubject, ebsDeleteData, 30*time.Second)
-			if err != nil {
-				slog.WarnContext(ctx, "TerminateStoppedInstance: ebs.delete failed for internal volume", "name", ebsRequest.Name, "err", err)
-			} else {
-				slog.InfoContext(ctx, "TerminateStoppedInstance: ebs.delete sent for internal volume", "name", ebsRequest.Name, "data", string(deleteMsg.Data))
-			}
+			slog.WarnContext(ctx, "TerminateStoppedInstance: skipping leftover EFI volume request",
+				"name", ebsRequest.Name, "instanceId", instanceID)
 			continue
 		}
 
@@ -3338,7 +3345,7 @@ func (s *InstanceServiceImpl) deleteInstanceENI(ctx context.Context, instance *v
 	if instance.ENIId != "" && s.eniDeleter != nil {
 		// Detach first to clear the attachment record. A stopped instance's ENI may
 		// still show Status=in-use/AttachmentId set; without the detach the
-		// force=false DeleteNetworkInterface below fails InvalidNetworkInterface.InUse,
+		// force=false DeleteNetworkInterface below fails the in-use InvalidParameterValue,
 		// stranding the ENI record after the instance is already gone from the store
 		// and blocking VPC/subnet delete. Best-effort: the delete is
 		// the authoritative gate.
@@ -3602,17 +3609,20 @@ func buildInstanceStatus(v *vm.VM, systemImpaired bool, az string) *ec2.Instance
 		AvailabilityZone: aws.String(az),
 		InstanceId:       aws.String(v.ID),
 		InstanceState:    state,
-		InstanceStatus: &ec2.InstanceStatusSummary{
-			Status:  aws.String(status),
-			Details: []*ec2.InstanceStatusDetails{instDetail},
-		},
-		SystemStatus: &ec2.InstanceStatusSummary{
-			Status:  aws.String(systemStatus),
-			Details: []*ec2.InstanceStatusDetails{sysDetail},
-		},
+		InstanceStatus:   statusSummary(status, instDetail),
+		SystemStatus:     statusSummary(systemStatus, sysDetail),
 	}
 }
 
+// statusSummary omits the detail of a not-applicable check, as AWS does.
+func statusSummary(status string, detail *ec2.InstanceStatusDetails) *ec2.InstanceStatusSummary {
+	summary := &ec2.InstanceStatusSummary{Status: aws.String(status)}
+	if status != instanceStatusNotApplicable {
+		summary.Details = []*ec2.InstanceStatusDetails{detail}
+	}
+	return summary
+}
+
 // instanceInitializingGrace is the post-launch window during which a running VM
 // reports initializing, matching AWS's status-check grace period.
 const instanceInitializingGrace = 2 * time.Minute

spinifex/domains/ec2/instance/service_impl_test.go

@@ -15,7 +15,6 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/contracts/ec2/v1"
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
@@ -1855,8 +1854,8 @@ func (f *fakePublicIPReleaser) ReleaseIP(_ context.Context, pool, ip, ownerENIID
 }
 
 // embeddedNATS spins up an in-process NATS server scoped to the test and
-// returns a connected client. Used for service tests that exercise the
-// ebs.delete path inside TerminateStoppedInstance.
+// returns a connected client, for service tests that observe what
+// TerminateStoppedInstance sends over NATS.
 func embeddedNATS(t *testing.T) *nats.Conn {
 	t.Helper()
 	_, nc := testutil.StartTestNATS(t)
@@ -1927,7 +1926,7 @@ func TestTerminateStoppedInstance_TerminationProtected(t *testing.T) {
 			DisableApiTermination: aws.Bool(true),
 		},
 	}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+	v.EBSRequests.Requests = []vm.EBSRequest{
 		{Name: "vol-001", DeleteOnTermination: true},
 	}
 	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
@@ -2038,7 +2037,7 @@ func TestTerminateStoppedInstance_RetriesStoppedDelete(t *testing.T) {
 func TestTerminateStoppedInstance_UserVolumeDeleted(t *testing.T) {
 	id := "i-vol"
 	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc"}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+	v.EBSRequests.Requests = []vm.EBSRequest{
 		{Name: "vol-user-001", DeleteOnTermination: true},
 		{Name: "vol-keep-001", DeleteOnTermination: false},
 	}
@@ -2059,7 +2058,7 @@ func TestTerminateStoppedInstance_FailedSealOwesTheStoppingNodeAnUnpublish(t *te
 	id := "i-seal-failed"
 	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc", LastNode: "node-a",
 		UnpublishOwed: []string{"vol-root-001", "vol-keep-001"}}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+	v.EBSRequests.Requests = []vm.EBSRequest{
 		{Name: "vol-root-001", Boot: true, DeleteOnTermination: true},
 		{Name: "vol-keep-001", DeleteOnTermination: false},
 		{Name: "vol-sealed-001", DeleteOnTermination: false},
@@ -2081,7 +2080,7 @@ func TestTerminateStoppedInstance_FailedSealOwesTheStoppingNodeAnUnpublish(t *te
 func TestTerminateStoppedInstance_NoLastNodeOwesNothing(t *testing.T) {
 	id := "i-seal-failed-nonode"
 	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc", UnpublishOwed: []string{"vol-keep-001"}}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-keep-001"}}
+	v.EBSRequests.Requests = []vm.EBSRequest{{Name: "vol-keep-001"}}
 	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
 	vd := &fakeVolumeDeleter{}
 	svc := &InstanceServiceImpl{stoppedStore: store, volumeDeleter: vd}
@@ -2100,7 +2099,7 @@ func TestTerminateStoppedInstance_NoLastNodeOwesNothing(t *testing.T) {
 func TestTerminateStoppedInstance_StampsTeardownVolumesDone(t *testing.T) {
 	id := "i-teardown-done"
 	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc"}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+	v.EBSRequests.Requests = []vm.EBSRequest{
 		{Name: "vol-root-001", Boot: true, DeleteOnTermination: true},
 	}
 	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
@@ -2124,7 +2123,7 @@ func TestTerminateStoppedInstance_StampsTeardownVolumesDone(t *testing.T) {
 func TestTerminateStoppedInstance_StampsTeardownVolumesFailedOnDeleteError(t *testing.T) {
 	id := "i-teardown-failed"
 	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc"}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+	v.EBSRequests.Requests = []vm.EBSRequest{
 		{Name: "vol-root-002", Boot: true, DeleteOnTermination: true},
 	}
 	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
@@ -2149,7 +2148,7 @@ func TestTerminateStoppedInstance_StampsTeardownVolumesFailedOnDeleteError(t *te
 func TestTerminateStoppedInstance_NonDoTBootVolumeDetachedNotDeleted(t *testing.T) {
 	id := "i-nondot-boot"
 	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc"}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+	v.EBSRequests.Requests = []vm.EBSRequest{
 		{Name: "vol-root-nondot", Boot: true, DeleteOnTermination: false},
 	}
 	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
@@ -2173,7 +2172,7 @@ func TestTerminateStoppedInstance_NonDoTBootVolumeDetachedNotDeleted(t *testing.
 func TestTerminateStoppedInstance_NonDoTDataVolumeDetachedNotDeleted(t *testing.T) {
 	id := "i-nondot-data"
 	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc"}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+	v.EBSRequests.Requests = []vm.EBSRequest{
 		{Name: "vol-data-nondot", DeleteOnTermination: false},
 	}
 	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
@@ -2195,7 +2194,7 @@ func TestTerminateStoppedInstance_NonDoTDataVolumeDetachedNotDeleted(t *testing.
 func TestTerminateStoppedInstance_DetachOfAlreadyGoneVolumeIsNotAFailure(t *testing.T) {
 	id := "i-nondot-gone"
 	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc"}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+	v.EBSRequests.Requests = []vm.EBSRequest{
 		{Name: "vol-data-gone", DeleteOnTermination: false},
 	}
 	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
@@ -2213,7 +2212,7 @@ func TestTerminateStoppedInstance_DetachOfAlreadyGoneVolumeIsNotAFailure(t *test
 func TestTerminateStoppedInstance_NoVolumeDeleterSkipsGracefully(t *testing.T) {
 	id := "i-no-vd"
 	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc"}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+	v.EBSRequests.Requests = []vm.EBSRequest{
 		{Name: "vol-user-001", DeleteOnTermination: true},
 	}
 	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
@@ -2224,30 +2223,48 @@ func TestTerminateStoppedInstance_NoVolumeDeleterSkipsGracefully(t *testing.T) {
 	require.NotNil(t, store.WroteTerminated[id])
 }
 
-func TestTerminateStoppedInstance_InternalVolumesViaNATS(t *testing.T) {
-	id := "i-int-vol"
-	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc"}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
-		{Name: "vol-efi-001", EFI: true},
+// TestTerminateStoppedInstance_SkipsLeftoverEFIRequest pins that an EFI request
+// left on a record sends nothing and touches no volume, so the teardown mark is
+// decided by the instance's other volumes alone.
+func TestTerminateStoppedInstance_SkipsLeftoverEFIRequest(t *testing.T) {
+	for _, tc := range []struct {
+		name      string
+		deleteErr error
+		want      vm.TeardownState
+	}{
+		{"other volume deleted", nil, vm.TeardownDone},
+		{"other volume failed", errors.New("backend unavailable"), vm.TeardownFailed},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			id := "i-int-vol"
+			v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc"}
+			v.EBSRequests.Requests = []vm.EBSRequest{
+				{Name: "vol-root-001-efi", EFI: true, DeleteOnTermination: true},
+				{Name: "vol-root-001", Boot: true, DeleteOnTermination: true},
+			}
+			store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
+
+			nc := embeddedNATS(t)
+			deleteSub, err := nc.SubscribeSync("ebs.delete")
+			require.NoError(t, err)
+			defer deleteSub.Unsubscribe()
+
+			vd := &fakeVolumeDeleter{err: tc.deleteErr}
+			svc := &InstanceServiceImpl{stoppedStore: store, natsConn: nc, volumeDeleter: vd}
+
+			_, err = svc.TerminateStoppedInstance(context.Background(), &TerminateStoppedInstanceInput{InstanceID: id}, "acc")
+			require.NoError(t, err)
+
+			require.NoError(t, nc.Flush())
+			pending, _, err := deleteSub.Pending()
+			require.NoError(t, err)
+			assert.Zero(t, pending, "an EFI request must send nothing on ebs.delete")
+			assert.Equal(t, []string{"vol-root-001"}, vd.calls, "an EFI request must reach neither volume-service call")
+			assert.Empty(t, vd.detached)
+			require.NotNil(t, store.WroteTerminated[id])
+			assert.Equal(t, string(tc.want), store.WroteTerminated[id].Teardown[vm.TeardownVolumes])
+		})
 	}
-	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
-
-	nc := embeddedNATS(t)
-	var ebsDeleted []string
-	sub, err := nc.Subscribe("ebs.delete", func(msg *nats.Msg) {
-		var req viperblocklegacyv1.EBSDeleteRequest
-		_ = json.Unmarshal(msg.Data, &req)
-		ebsDeleted = append(ebsDeleted, req.Volume)
-		_ = msg.Respond([]byte(`{"Success":true}`))
-	})
-	require.NoError(t, err)
-	defer sub.Unsubscribe()
-
-	svc := &InstanceServiceImpl{stoppedStore: store, natsConn: nc}
-
-	_, err = svc.TerminateStoppedInstance(context.Background(), &TerminateStoppedInstanceInput{InstanceID: id}, "acc")
-	require.NoError(t, err)
-	assert.ElementsMatch(t, []string{"vol-efi-001"}, ebsDeleted)
 }
 
 func TestTerminateStoppedInstance_PublicIPReleased(t *testing.T) {
@@ -2815,10 +2832,10 @@ type raceVolumeMounter struct{}
 func (raceVolumeMounter) Mount(context.Context, *vm.VM) error           { return nil }
 func (raceVolumeMounter) Unmount(context.Context, *vm.VM) error         { return nil }
 func (raceVolumeMounter) Abandon(context.Context, *vm.VM, string) error { return nil }
-func (raceVolumeMounter) MountOne(context.Context, string, *viperblocklegacyv1.EBSRequest) error {
+func (raceVolumeMounter) MountOne(context.Context, string, *vm.EBSRequest) error {
 	return nil
 }
-func (raceVolumeMounter) UnmountOne(context.Context, string, viperblocklegacyv1.EBSRequest) error {
+func (raceVolumeMounter) UnmountOne(context.Context, string, vm.EBSRequest) error {
 	return nil
 }
 
@@ -3152,6 +3169,30 @@ func TestPrepareRunInstances_AMINotOwnedByCaller(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorInvalidAMIIDNotFound, err.Error())
 }
 
+func TestPrepareRunInstances_AMINotAvailable(t *testing.T) {
+	for _, state := range []string{ebsmetadata.AMIStatePending, ebsmetadata.AMIStateFailed} {
+		t.Run(state, func(t *testing.T) {
+			types, _ := defaultPrepareInstanceTypes()
+			svc := &InstanceServiceImpl{
+				instanceTypes: types,
+				amiLoader: &fakeAMILoader{byID: map[string]ebsmetadata.AMI{
+					"ami-1": {ImageOwnerAlias: "acc", State: state},
+				}},
+			}
+			_, _, _, err := svc.PrepareRunInstances(context.Background(), &ec2.RunInstancesInput{
+				InstanceType: aws.String("t3.micro"),
+				ImageId:      aws.String("ami-1"),
+				MinCount:     aws.Int64(1),
+				MaxCount:     aws.Int64(1),
+			}, "acc", "")
+			code, message, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok, "unregistered error: %v", err)
+			assert.Equal(t, awserrors.ErrorInvalidAMIIDUnavailable, code)
+			assert.Contains(t, message, state)
+		})
+	}
+}
+
 func TestPrepareRunInstances_KeyPairNotFound(t *testing.T) {
 	types, _ := defaultPrepareInstanceTypes()
 	svc := &InstanceServiceImpl{
@@ -3784,6 +3825,50 @@ func TestPrepareRunInstances_DefaultSubnetResolved(t *testing.T) {
 	assert.Equal(t, 1, eni.attachCalls)
 }
 
+// The RunInstances response carries the primary interface's attachment and
+// address fields, as AWS's does, rather than the bare launch-time entry.
+func TestPrepareRunInstances_PrimaryInterfaceFields(t *testing.T) {
+	eni := &fakeENICreator{
+		createOut: &ec2.CreateNetworkInterfaceOutput{
+			NetworkInterface: &ec2.NetworkInterface{
+				NetworkInterfaceId: aws.String("eni-1"),
+				MacAddress:         aws.String("aa:bb:cc:dd:ee:ff"),
+				PrivateIpAddress:   aws.String("10.0.0.10"),
+				VpcId:              aws.String("vpc-1"),
+				Description:        aws.String("Primary network interface for i-x"),
+			},
+		},
+	}
+	svc, _ := prepareSvcWithENI(t, eni, nil)
+
+	reservation, _, _, err := svc.PrepareRunInstances(context.Background(), &ec2.RunInstancesInput{
+		InstanceType: aws.String("t3.micro"),
+		ImageId:      aws.String("ami-1"),
+		SubnetId:     aws.String("subnet-1"),
+		MinCount:     aws.Int64(1),
+		MaxCount:     aws.Int64(1),
+	}, "acc", "")
+	require.NoError(t, err)
+	require.Len(t, reservation.Instances, 1)
+	inst := reservation.Instances[0]
+	require.Len(t, inst.NetworkInterfaces, 1)
+	nic := inst.NetworkInterfaces[0]
+
+	require.NotNil(t, nic.Description)
+	assert.Empty(t, *nic.Description)
+	assert.Equal(t, "interface", aws.StringValue(nic.InterfaceType))
+	assert.Equal(t, "acc", aws.StringValue(nic.OwnerId))
+	require.NotNil(t, nic.Attachment)
+	assert.Equal(t, "attached", aws.StringValue(nic.Attachment.AttachmentId), "the id AttachENI returned")
+	assert.Equal(t, aws.TimeValue(inst.LaunchTime), aws.TimeValue(nic.Attachment.AttachTime))
+	assert.True(t, aws.BoolValue(nic.Attachment.DeleteOnTermination))
+	require.NotNil(t, nic.Attachment.NetworkCardIndex)
+	assert.Equal(t, int64(0), *nic.Attachment.NetworkCardIndex)
+	assert.Equal(t, []*ec2.InstancePrivateIpAddress{{
+		Primary: aws.Bool(true), PrivateIpAddress: aws.String("10.0.0.10"),
+	}}, nic.PrivateIpAddresses)
+}
+
 // TestPrepareRunInstances_ForwardsPrivateIpAddress verifies a pinned
 // RunInstancesInput.PrivateIpAddress reaches CreateNetworkInterfaceInput,
 // where the IPAM claim path (eni.go) actually enforces it.
@@ -4708,6 +4793,18 @@ func TestBuildInstanceStatus_ErrorStateProjectsToStopped(t *testing.T) {
 	assert.Equal(t, int64(80), *is.InstanceState.Code)
 }
 
+// AWS reports a stopped instance's checks as not-applicable with no details.
+func TestBuildInstanceStatus_StoppedHasNoDetails(t *testing.T) {
+	stopped := runningVM("i-stopped", "111122223333")
+	stopped.Status = vm.StateStopped
+
+	is := buildInstanceStatus(stopped, false, "az-a")
+	assert.Equal(t, "not-applicable", *is.InstanceStatus.Status)
+	assert.Nil(t, is.InstanceStatus.Details)
+	assert.Equal(t, "not-applicable", *is.SystemStatus.Status)
+	assert.Nil(t, is.SystemStatus.Details)
+}
+
 // Every internal state must map onto a valid AWS InstanceStateName here, the
 // same guarantee vm.EC2APIState gives the DescribeInstances projection.
 func TestBuildInstanceStatus_NeverSurfacesNonAWSName(t *testing.T) {
@@ -5093,6 +5190,7 @@ func TestPrepareRunInstances_PreCreatedENIAttachedSkipsAutoCreate(t *testing.T)
 				VpcID:              "vpc-pre",
 				PrivateIpAddress:   "10.0.5.42",
 				MacAddress:         "aa:bb:cc:dd:ee:01",
+				Description:        "customer interface",
 				Status:             "available",
 				SecurityGroupIDs:   []string{"sg-pre"},
 			},
@@ -5100,7 +5198,7 @@ func TestPrepareRunInstances_PreCreatedENIAttachedSkipsAutoCreate(t *testing.T)
 	}
 	svc, _ := prepareSvcWithENI(t, eni, nil)
 
-	_, instances, _, err := svc.PrepareRunInstances(context.Background(), &ec2.RunInstancesInput{
+	reservation, instances, _, err := svc.PrepareRunInstances(context.Background(), &ec2.RunInstancesInput{
 		InstanceType: aws.String("t3.micro"),
 		ImageId:      aws.String("ami-1"),
 		MinCount:     aws.Int64(1),
@@ -5116,6 +5214,15 @@ func TestPrepareRunInstances_PreCreatedENIAttachedSkipsAutoCreate(t *testing.T)
 	assert.Equal(t, "aa:bb:cc:dd:ee:01", instances[0].ENIMac)
 	assert.Equal(t, 1, eni.attachCalls)
 	assert.Equal(t, 0, eni.createCalls, "auto-create must not run when NetworkInterfaceId is specified")
+
+	// A caller's interface keeps its own description and delete-on-termination.
+	nic := reservation.Instances[0].NetworkInterfaces[0]
+	assert.Equal(t, "customer interface", aws.StringValue(nic.Description))
+	require.NotNil(t, nic.Attachment.DeleteOnTermination)
+	assert.False(t, *nic.Attachment.DeleteOnTermination)
+	assert.Equal(t, "attached", aws.StringValue(nic.Attachment.AttachmentId))
+	assert.Equal(t, "interface", aws.StringValue(nic.InterfaceType))
+	assert.Equal(t, aws.TimeValue(reservation.Instances[0].LaunchTime), aws.TimeValue(nic.Attachment.AttachTime))
 }
 
 func TestPrepareRunInstances_PreCreatedENIInUseRejected(t *testing.T) {

spinifex/domains/ec2/instance/start_after_detach_test.go

@@ -7,7 +7,6 @@ import (
 	"sync"
 	"testing"
 
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -31,7 +30,7 @@ func (r *mountRecorder) Mount(_ context.Context, v *vm.VM) error {
 	return errors.New("stop after recording")
 }
 
-func (r *mountRecorder) MountOne(context.Context, string, *viperblocklegacyv1.EBSRequest) error {
+func (r *mountRecorder) MountOne(context.Context, string, *vm.EBSRequest) error {
 	return nil
 }
 

spinifex/domains/ec2/instance/start_public_ip_test.go

@@ -12,7 +12,6 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
 	vmmock "github.com/mulgadc/spinifex/spinifex/runtime/compute/vm/mock"
@@ -28,11 +27,11 @@ type refusingMounter struct{}
 
 func (refusingMounter) Mount(context.Context, *vm.VM) error   { return errors.New("no volumes here") }
 func (refusingMounter) Unmount(context.Context, *vm.VM) error { return nil }
-func (refusingMounter) MountOne(context.Context, string, *viperblocklegacyv1.EBSRequest) error {
+func (refusingMounter) MountOne(context.Context, string, *vm.EBSRequest) error {
 	return nil
 }
 func (refusingMounter) Abandon(context.Context, *vm.VM, string) error { return nil }
-func (refusingMounter) UnmountOne(context.Context, string, viperblocklegacyv1.EBSRequest) error {
+func (refusingMounter) UnmountOne(context.Context, string, vm.EBSRequest) error {
 	return nil
 }
 

spinifex/domains/ec2/instance/start_volumes.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
@@ -28,7 +27,7 @@ func (s *InstanceServiceImpl) reconcileStartVolumes(ctx context.Context, instanc
 	requests := slices.Clone(instance.EBSRequests.Requests)
 	instance.EBSRequests.Mu.Unlock()
 
-	if !slices.ContainsFunc(requests, func(r viperblocklegacyv1.EBSRequest) bool { return r.Boot }) {
+	if !slices.ContainsFunc(requests, func(r vm.EBSRequest) bool { return r.Boot }) {
 		return noRootVolumeError(instance)
 	}
 
@@ -65,7 +64,7 @@ func (s *InstanceServiceImpl) reconcileStartVolumes(ctx context.Context, instanc
 	}
 
 	instance.EBSRequests.Mu.Lock()
-	instance.EBSRequests.Requests = slices.DeleteFunc(instance.EBSRequests.Requests, func(r viperblocklegacyv1.EBSRequest) bool {
+	instance.EBSRequests.Requests = slices.DeleteFunc(instance.EBSRequests.Requests, func(r vm.EBSRequest) bool {
 		return slices.Contains(drop, r.Name)
 	})
 	instance.EBSRequests.Mu.Unlock()

spinifex/domains/ec2/instance/start_volumes_test.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
@@ -36,7 +35,7 @@ func startingInstance(id string, names ...string) *vm.VM {
 		if boot {
 			device = "/dev/sda1"
 		}
-		v.EBSRequests.Requests = append(v.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{Name: n, Boot: boot, DeviceName: device})
+		v.EBSRequests.Requests = append(v.EBSRequests.Requests, vm.EBSRequest{Name: n, Boot: boot, DeviceName: device})
 		v.Instance.BlockDeviceMappings = append(v.Instance.BlockDeviceMappings, &ec2.InstanceBlockDeviceMapping{
 			DeviceName: aws.String(device), Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String(n)},
 		})
@@ -136,7 +135,7 @@ func TestReconcileStartVolumes_DirectBootNeedsNoRootVolume(t *testing.T) {
 
 	sqs := &vm.VM{ID: "i-sqs", AccountID: startVolumesAccount, Status: vm.StateStopped, DirectBoot: true,
 		Instance: &ec2.Instance{InstanceId: aws.String("i-sqs")}}
-	sqs.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-store-gone", DeviceName: "/dev/vdb"}}
+	sqs.EBSRequests.Requests = []vm.EBSRequest{{Name: "vol-store-gone", DeviceName: "/dev/vdb"}}
 	require.NoError(t, svc.reconcileStartVolumes(context.Background(), sqs))
 	assert.Equal(t, []string{"vol-store-gone"}, requestedNames(sqs), "a system store volume is never pruned")
 }

spinifex/domains/ec2/instance/stopped_detach.go

@@ -8,18 +8,17 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
 )
 
-func findVolumeRequest(v *vm.VM, volumeID string) (viperblocklegacyv1.EBSRequest, bool) {
+func findVolumeRequest(v *vm.VM, volumeID string) (vm.EBSRequest, bool) {
 	v.EBSRequests.Mu.Lock()
 	defer v.EBSRequests.Mu.Unlock()
-	i := slices.IndexFunc(v.EBSRequests.Requests, func(r viperblocklegacyv1.EBSRequest) bool { return r.Name == volumeID })
+	i := slices.IndexFunc(v.EBSRequests.Requests, func(r vm.EBSRequest) bool { return r.Name == volumeID })
 	if i < 0 {
-		return viperblocklegacyv1.EBSRequest{}, false
+		return vm.EBSRequest{}, false
 	}
 	return v.EBSRequests.Requests[i], true
 }
@@ -29,7 +28,7 @@ func findVolumeRequest(v *vm.VM, volumeID string) (viperblocklegacyv1.EBSRequest
 func removeVolumeFromRecord(v *vm.VM, volumeID string) bool {
 	v.EBSRequests.Mu.Lock()
 	defer v.EBSRequests.Mu.Unlock()
-	i := slices.IndexFunc(v.EBSRequests.Requests, func(r viperblocklegacyv1.EBSRequest) bool { return r.Name == volumeID })
+	i := slices.IndexFunc(v.EBSRequests.Requests, func(r vm.EBSRequest) bool { return r.Name == volumeID })
 	if i < 0 || v.EBSRequests.Requests[i].EFI {
 		return false
 	}
@@ -93,7 +92,7 @@ func stoppedAttachDevice(v *vm.VM, requested string) (string, bool, error) {
 func addVolumeToRecord(v *vm.VM, volumeID, device string, boot bool) {
 	v.EBSRequests.Mu.Lock()
 	defer v.EBSRequests.Mu.Unlock()
-	v.EBSRequests.Requests = append(v.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{
+	v.EBSRequests.Requests = append(v.EBSRequests.Requests, vm.EBSRequest{
 		Name:       volumeID,
 		DeviceName: device,
 		Boot:       boot,

spinifex/domains/ec2/instance/stopped_detach_test.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
@@ -27,7 +26,7 @@ func stoppedWithVolumes(owner, id string) *vm.VM {
 			},
 		},
 	}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+	v.EBSRequests.Requests = []vm.EBSRequest{
 		{Name: "vol-root", Boot: true, DeviceName: "/dev/sda1"},
 		{Name: "vol-data", DeviceName: "/dev/sdf"},
 		{Name: "vol-other", DeviceName: "/dev/sdg"},
@@ -127,7 +126,7 @@ func TestDetachVolumeFromStopped_RefusesTheInternalEFIStore(t *testing.T) {
 	id := "i-stopped-efi"
 	stored := stoppedWithVolumes(owner, id)
 	stored.EBSRequests.Requests = append(stored.EBSRequests.Requests,
-		viperblocklegacyv1.EBSRequest{Name: "vol-efi", EFI: true})
+		vm.EBSRequest{Name: "vol-efi", EFI: true})
 	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: stored}}
 	svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), stoppedStore: store}
 

spinifex/domains/ec2/natgw/service_impl.go

@@ -137,15 +137,16 @@ func (s *NatGatewayServiceImpl) CreateNatGateway(ctx context.Context, input *ec2
 
 	natgwID := awsidentifiers.GenerateResourceID("nat")
 	record := NatGatewayRecord{
-		NatGatewayId: natgwID,
-		VpcId:        subnetRecord.VpcId,
-		SubnetId:     subnetID,
-		AllocationId: allocID,
-		PublicIp:     eipRecord.PublicIp,
-		State:        "available",
-		AccountID:    accountID,
-		Tags:         awstags.Extract(input.TagSpecifications, ec2.ResourceTypeNatgateway),
-		CreatedAt:    time.Now(),
+		NatGatewayId:  natgwID,
+		VpcId:         subnetRecord.VpcId,
+		SubnetId:      subnetID,
+		AllocationId:  allocID,
+		PublicIp:      eipRecord.PublicIp,
+		AssociationId: awsidentifiers.GenerateResourceID("eipassoc"),
+		State:         "available",
+		AccountID:     accountID,
+		Tags:          awstags.Extract(input.TagSpecifications, ec2.ResourceTypeNatgateway),
+		CreatedAt:     time.Now(),
 	}
 
 	data, err := json.Marshal(record)
@@ -161,7 +162,7 @@ func (s *NatGatewayServiceImpl) CreateNatGateway(ctx context.Context, input *ec2
 	// NAT GW reusing the same allocation). Linkage is the shared AllocationId;
 	// DeleteNatGateway clears it back to allocated. Non-fatal: the NAT GW is
 	// already stored, so a failed mark only weakens the reuse guard until then.
-	eipRecord.AssociationId = awsidentifiers.GenerateResourceID("eipassoc")
+	eipRecord.AssociationId = record.AssociationId
 	eipRecord.VpcId = subnetRecord.VpcId
 	eipRecord.State = "associated"
 	if eipData, err := json.Marshal(eipRecord); err == nil {
@@ -490,8 +491,13 @@ func recordToEC2(record *NatGatewayRecord) *ec2.NatGateway {
 			{
 				AllocationId: aws.String(record.AllocationId),
 				PublicIp:     aws.String(record.PublicIp),
+				IsPrimary:    aws.Bool(true),
+				Status:       aws.String(ec2.NatGatewayAddressStatusSucceeded),
 			},
 		}
+		if record.AssociationId != "" {
+			ngw.NatGatewayAddresses[0].AssociationId = aws.String(record.AssociationId)
+		}
 	}
 
 	ngw.Tags = awstags.MapToEC2(record.Tags)

spinifex/domains/ec2/natgw/service_impl_test.go

@@ -1,6 +1,7 @@
 package natgw
 
 import (
+	"encoding/json"
 	"testing"
 	"time"
 
@@ -95,6 +96,36 @@ func TestCreateNatGateway(t *testing.T) {
 	assert.Equal(t, "203.0.113.50", *ngw.NatGatewayAddresses[0].PublicIp)
 }
 
+// AWS reports the gateway's address as primary and succeeded, under the
+// association ID that DescribeAddresses shows for the same EIP.
+func TestDescribeNatGateways_AddressCarriesAssociation(t *testing.T) {
+	svc, js := setupTestServiceJS(t)
+	createOut, err := svc.CreateNatGateway(t.Context(), &ec2.CreateNatGatewayInput{
+		SubnetId:     aws.String("subnet-pub1"),
+		AllocationId: aws.String("eipalloc-test1"),
+	}, testAccountID)
+	require.NoError(t, err)
+
+	eipKV, err := js.KeyValue(t.Context(), ec2eip.KVBucketEIPs)
+	require.NoError(t, err)
+	entry, err := eipKV.Get(t.Context(), kvstore.AccountKey(testAccountID, "eipalloc-test1"))
+	require.NoError(t, err)
+	var eip ec2eip.EIPRecord
+	require.NoError(t, json.Unmarshal(entry.Value(), &eip))
+	require.NotEmpty(t, eip.AssociationId)
+
+	out, err := svc.DescribeNatGateways(t.Context(), &ec2.DescribeNatGatewaysInput{
+		NatGatewayIds: []*string{createOut.NatGateway.NatGatewayId},
+	}, testAccountID)
+	require.NoError(t, err)
+	require.Len(t, out.NatGateways, 1)
+	require.Len(t, out.NatGateways[0].NatGatewayAddresses, 1)
+	addr := out.NatGateways[0].NatGatewayAddresses[0]
+	assert.Equal(t, eip.AssociationId, aws.StringValue(addr.AssociationId))
+	assert.True(t, aws.BoolValue(addr.IsPrimary))
+	assert.Equal(t, ec2.NatGatewayAddressStatusSucceeded, aws.StringValue(addr.Status))
+}
+
 // TestCreateNatGateway_PersistsTagsForTagFilterDiscovery locks the
 // contract: CreateNatGateway must persist TagSpecifications so DeleteClusterCPVPC can
 // find the cluster NAT gateway by tag filter and release its EIP. A dropped tag

spinifex/domains/ec2/natgw/types.go

@@ -12,13 +12,14 @@ const (
 
 // NatGatewayRecord represents a stored NAT Gateway.
 type NatGatewayRecord struct {
-	NatGatewayId string            `json:"nat_gateway_id"`
-	VpcId        string            `json:"vpc_id"`
-	SubnetId     string            `json:"subnet_id"`     // public subnet where NAT GW lives
-	AllocationId string            `json:"allocation_id"` // EIP allocation
-	PublicIp     string            `json:"public_ip"`     // EIP address
-	State        string            `json:"state"`         // pending, available, deleting, deleted, failed
-	AccountID    string            `json:"account_id"`
-	Tags         map[string]string `json:"tags"`
-	CreatedAt    time.Time         `json:"created_at"`
+	NatGatewayId  string            `json:"nat_gateway_id"`
+	VpcId         string            `json:"vpc_id"`
+	SubnetId      string            `json:"subnet_id"`     // public subnet where NAT GW lives
+	AllocationId  string            `json:"allocation_id"` // EIP allocation
+	PublicIp      string            `json:"public_ip"`     // EIP address
+	AssociationId string            `json:"association_id,omitempty"`
+	State         string            `json:"state"` // pending, available, deleting, deleted, failed
+	AccountID     string            `json:"account_id"`
+	Tags          map[string]string `json:"tags"`
+	CreatedAt     time.Time         `json:"created_at"`
 }

spinifex/domains/ec2/routetable/service_impl.go

@@ -805,13 +805,13 @@ func (s *RouteTableServiceImpl) CreateRoute(ctx context.Context, input *ec2.Crea
 		return nil, err
 	}
 
-	// A repeat of an existing route (same destination, same target) is a
-	// successful no-op; a different target is a conflict.
+	// AWS answers a repeated gateway route with success; a repeated NAT
+	// gateway route, or any other target, is a conflict.
 	for _, r := range record.Routes {
 		if r.DestinationCidrBlock != destCidr {
 			continue
 		}
-		if r.GatewayId == route.GatewayId && r.NatGatewayId == route.NatGatewayId {
+		if route.GatewayId != "" && r.GatewayId == route.GatewayId && r.NatGatewayId == "" {
 			return &ec2.CreateRouteOutput{Return: aws.Bool(true)}, nil
 		}
 		return nil, awserrors.Errorf(awserrors.ErrorRouteAlreadyExists, "The route identified by %s already exists.", destCidr)

spinifex/domains/ec2/routetable/service_impl_test.go

@@ -404,6 +404,24 @@ func TestCreateRoute_UnknownNatGatewayKeepsInvalidIDCode(t *testing.T) {
 	assert.EqualError(t, err, awserrors.ErrorInvalidNatGatewayIDNotFound)
 }
 
+// AWS refuses a repeat of a NAT gateway route, unlike a gateway route.
+func TestCreateRoute_RepeatedNatGatewayRouteConflicts(t *testing.T) {
+	t.Parallel()
+	svc := setupTestService(t)
+	rtbID := createTestRtb(t, svc)
+	input := &ec2.CreateRouteInput{
+		RouteTableId:         aws.String(rtbID),
+		DestinationCidrBlock: aws.String("0.0.0.0/0"),
+		NatGatewayId:         aws.String("nat-test1"),
+	}
+
+	_, err := svc.CreateRoute(t.Context(), input, testAccountID)
+	require.NoError(t, err)
+
+	_, err = svc.CreateRoute(t.Context(), input, testAccountID)
+	testutil.RequireAWSError(t, err, awserrors.ErrorRouteAlreadyExists, "The route identified by 0.0.0.0/0 already exists.")
+}
+
 func TestCreateRoute_DuplicateDestination(t *testing.T) {
 	t.Parallel()
 	svc := setupTestService(t)

spinifex/domains/ec2/snapshot/claims.go

@@ -0,0 +1,197 @@
+package snapshot
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"sync"
+	"time"
+
+	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	otelsetup "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+)
+
+// ClaimPrefix is the kvsync prefix of every snapshot's claim, ClaimPrefix+ID,
+// naming the node that alone may call the provider for it or delete its
+// document. An AMI's claim, keyed by image ID, shares the prefix.
+const ClaimPrefix = "snapshot."
+
+// claimReleaseTimeout bounds the unlock after the last write, which runs even
+// when the request that held the claim has been cancelled.
+const claimReleaseTimeout = 5 * time.Second
+
+// NewClaimGroup returns node's group of snapshot claims over store. The daemon
+// builds it once per process, as kvsync requires of every prefix.
+func NewClaimGroup(store kvsync.KVStore, node string) *kvsync.ClaimGroup {
+	return kvsync.NewClaimGroup(store, ClaimPrefix, node)
+}
+
+// NodeLiveness reports when a node last heartbeated. found is false only for a
+// node with no heartbeat; a read that fails is an error, never a dead node.
+type NodeLiveness interface {
+	LastSeen(ctx context.Context, node string) (seen time.Time, found bool, err error)
+}
+
+// errClaimHeld is a snapshot claim another holder has: a request of this
+// process, or a node whose heartbeat is still fresh.
+var errClaimHeld = errors.New("snapshot claim is held")
+
+// Claims decides who may act on a pending snapshot. A kvsync claim names a
+// node, not a process, so the claims this process holds are also kept here:
+// the reaper on the node of a running request must not adopt that request's.
+type Claims struct {
+	group    *kvsync.ClaimGroup
+	node     string
+	liveness NodeLiveness
+	now      func() time.Time
+
+	mu    sync.Mutex
+	local map[string]struct{}
+}
+
+// NewClaims returns the claims of node over group. A nil liveness never judges
+// a holder dead, so a claim another node holds is never taken over.
+func NewClaims(group *kvsync.ClaimGroup, node string, liveness NodeLiveness) *Claims {
+	return &Claims{group: group, node: node, liveness: liveness, now: time.Now, local: map[string]struct{}{}}
+}
+
+// localClaims excludes only within this process: a service built without the
+// daemon's claim group has no other process to exclude.
+func localClaims() *Claims {
+	return NewClaims(nil, "", nil)
+}
+
+// heldClaim is one acquisition of a snapshot's claim.
+type heldClaim struct {
+	claims     *Claims
+	snapshotID string
+	claim      *kvsync.Claim
+	token      kvsync.Token
+}
+
+// reserve marks snapshotID held by this process, and is false when it already is.
+func (c *Claims) reserve(snapshotID string) bool {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if _, held := c.local[snapshotID]; held {
+		return false
+	}
+	c.local[snapshotID] = struct{}{}
+	return true
+}
+
+func (c *Claims) unreserve(snapshotID string) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	delete(c.local, snapshotID)
+}
+
+// lock takes snapshotID's claim for a creating request. It never takes over:
+// the ID is new, so any holder is a fault worth refusing the request for.
+func (c *Claims) lock(ctx context.Context, snapshotID string) (*heldClaim, error) {
+	return c.acquire(ctx, snapshotID, false)
+}
+
+// take acquires snapshotID's claim for the reaper. A claim another node holds
+// is taken over only once that node's heartbeat is LivenessWindow old or
+// absent; while it is fresh, or cannot be read, the claim is refused.
+func (c *Claims) take(ctx context.Context, snapshotID string) (*heldClaim, error) {
+	return c.acquire(ctx, snapshotID, true)
+}
+
+// acquire reserves snapshotID in this process and locks its claim, taking it
+// over from a dead holder only when takeOver allows.
+func (c *Claims) acquire(ctx context.Context, snapshotID string, takeOver bool) (*heldClaim, error) {
+	if !c.reserve(snapshotID) {
+		return nil, fmt.Errorf("%w: %s by this process", errClaimHeld, snapshotID)
+	}
+	held, err := c.acquireReserved(ctx, snapshotID, takeOver)
+	if err != nil {
+		c.unreserve(snapshotID)
+		return nil, err
+	}
+	return held, nil
+}
+
+func (c *Claims) acquireReserved(ctx context.Context, snapshotID string, takeOver bool) (*heldClaim, error) {
+	held := &heldClaim{claims: c, snapshotID: snapshotID}
+	if c.group == nil {
+		return held, nil
+	}
+	held.claim = c.group.Get(snapshotID)
+	tok, err := held.claim.TryLock(ctx)
+	if err == nil {
+		held.token = tok
+		return held, nil
+	}
+	he, ok := errors.AsType[*kvsync.HeldError](err)
+	if !takeOver || !ok || he.Current.Holder == "" {
+		return nil, fmt.Errorf("take snapshot claim %s: %w", snapshotID, err)
+	}
+	holder := he.Current
+	if c.liveness == nil {
+		return nil, fmt.Errorf("%w: %s by %s, and no liveness is wired to judge it", errClaimHeld, snapshotID, holder.Holder)
+	}
+	seen, found, err := c.liveness.LastSeen(ctx, holder.Holder)
+	if err != nil {
+		return nil, fmt.Errorf("%w: %s by %s, whose heartbeat cannot be read: %w", errClaimHeld, snapshotID, holder.Holder, err)
+	}
+	now := c.now()
+	if found && clusterv1.LiveAt(seen, now) {
+		return nil, fmt.Errorf("%w: %s by %s, which is still heartbeating", errClaimHeld, snapshotID, holder.Holder)
+	}
+	tok, err = held.claim.TakeOver(ctx, holder)
+	if err != nil {
+		return nil, fmt.Errorf("take over snapshot claim %s from %s: %w", snapshotID, holder.Holder, err)
+	}
+	attrs := []any{"snapshotId", snapshotID, "holder", holder.Holder, "fromGeneration", holder.Generation, "generation", tok.Generation}
+	if found {
+		attrs = append(attrs, "heartbeat_age_ms", otelsetup.Millis(now.Sub(seen)))
+	}
+	slog.WarnContext(ctx, "Took over a pending snapshot's claim from a node whose heartbeat has stopped", attrs...)
+	held.token = tok
+	return held, nil
+}
+
+// release gives the claim up after the holder's last write. A failed unlock
+// leaves the claim with this node until a restart or a takeover frees it, so
+// it is logged and not returned.
+func (h *heldClaim) release(ctx context.Context) {
+	defer h.claims.unreserve(h.snapshotID)
+	if h.claim == nil {
+		return
+	}
+	ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), claimReleaseTimeout)
+	defer cancel()
+	if err := h.claim.Unlock(ctx, h.token); err != nil {
+		slog.WarnContext(ctx, "Failed to release a snapshot claim",
+			"snapshotId", h.snapshotID, "generation", h.token.Generation, "err", err)
+	}
+}
+
+// releaseOwn unlocks snapshotID's claim when it names this node and no request
+// of this process holds it: a claim left by an earlier process of this node,
+// whose requests did not survive its restart.
+func (c *Claims) releaseOwn(ctx context.Context, snapshotID string) (bool, error) {
+	if c.group == nil || !c.reserve(snapshotID) {
+		return false, nil
+	}
+	defer c.unreserve(snapshotID)
+	claim := c.group.Get(snapshotID)
+	tok, ok, err := claim.Holder(ctx)
+	if err != nil {
+		return false, fmt.Errorf("read snapshot claim %s: %w", snapshotID, err)
+	}
+	if !ok || tok.Holder != c.node {
+		return false, nil
+	}
+	if err := claim.Unlock(ctx, tok); err != nil {
+		if errors.Is(err, kvsync.ErrLost) {
+			return false, nil
+		}
+		return false, fmt.Errorf("release snapshot claim %s: %w", snapshotID, err)
+	}
+	return true, nil
+}

spinifex/domains/ec2/snapshot/delete_pending_test.go

@@ -0,0 +1,581 @@
+package snapshot
+
+import (
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// fastClaims makes svc's writers retry a held claim without the real wait.
+func fastClaims(svc *SnapshotServiceImpl) *SnapshotServiceImpl {
+	svc.claimRetryDelay = time.Millisecond
+	return svc
+}
+
+func (f *pendingFixture) deleteSnapshot(svc *SnapshotServiceImpl, id string) error {
+	_, err := svc.DeleteSnapshot(f.t.Context(), &ec2.DeleteSnapshotInput{SnapshotId: aws.String(id)}, testAccountID)
+	return err
+}
+
+// assertHidden requires that every request answers id as a deleted snapshot.
+func (f *pendingFixture) assertHidden(svc *SnapshotServiceImpl, id string) {
+	f.t.Helper()
+	listed, err := svc.DescribeSnapshots(f.t.Context(), &ec2.DescribeSnapshotsInput{}, testAccountID)
+	require.NoError(f.t, err)
+	for _, snap := range listed.Snapshots {
+		assert.NotEqual(f.t, id, aws.StringValue(snap.SnapshotId), "DescribeSnapshots lists a deleted snapshot")
+	}
+	_, err = svc.DescribeSnapshots(f.t.Context(), &ec2.DescribeSnapshotsInput{SnapshotIds: aws.StringSlice([]string{id})}, testAccountID)
+	assert.EqualError(f.t, err, awserrors.ErrorInvalidSnapshotNotFound)
+	_, err = svc.DescribeSnapshotsStrict(f.t.Context(), &ec2.DescribeSnapshotsInput{SnapshotIds: aws.StringSlice([]string{id})}, testAccountID)
+	assert.EqualError(f.t, err, awserrors.ErrorInvalidSnapshotNotFound)
+	_, err = svc.CopySnapshot(f.t.Context(), &ec2.CopySnapshotInput{SourceSnapshotId: aws.String(id)}, testAccountID)
+	assert.EqualError(f.t, err, awserrors.ErrorInvalidSnapshotNotFound)
+	assert.EqualError(f.t, f.deleteSnapshot(svc, id), awserrors.ErrorInvalidSnapshotNotFound, "a repeated delete")
+}
+
+// DeleteSnapshot marks a pending or AmbiguousCreate snapshot and returns,
+// leaving the provider alone; any other is deleted as it always was.
+func TestDeleteSnapshot_MarksWhatTheProviderMayStillHold(t *testing.T) {
+	const id = "snap-markdelete0001"
+	cases := []struct {
+		name       string
+		doc        ebsmetadata.Snapshot
+		wantMarked bool
+	}{
+		{name: "pending", doc: ebsmetadata.Snapshot{State: snapshotStatePending}, wantMarked: true},
+		{name: "pending with an unanswered create", doc: ebsmetadata.Snapshot{State: snapshotStatePending, AmbiguousCreate: true}, wantMarked: true},
+		{name: "completed with an unanswered create", doc: ebsmetadata.Snapshot{State: ebsmetadata.SnapshotStateCompleted, AmbiguousCreate: true}, wantMarked: true},
+		{name: "errored with an unanswered create", doc: ebsmetadata.Snapshot{State: snapshotStateError, AmbiguousCreate: true}, wantMarked: true},
+		{name: "completed", doc: ebsmetadata.Snapshot{State: ebsmetadata.SnapshotStateCompleted}},
+		{name: "errored", doc: ebsmetadata.Snapshot{State: snapshotStateError}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newPendingFixture(t)
+			svc := f.node(nodeA)
+			tc.doc.SnapshotID = id
+			f.seedDoc(tc.doc, true)
+			providerSnapshot(f, id)
+
+			require.NoError(t, f.deleteSnapshot(svc, id))
+
+			doc, ok := f.doc(id)
+			if !tc.wantMarked {
+				assert.False(t, ok, "a settled snapshot is deleted as it always was")
+				assert.Equal(t, []string{"snapshot " + id}, f.provider.deleteCalls())
+				assert.Empty(t, f.refsOf(pendingVolumeID))
+				return
+			}
+			require.True(t, ok)
+			assert.True(t, doc.DeleteRequested)
+			assert.Equal(t, tc.doc.State, doc.State, "the mark changes nothing else")
+			assert.Empty(t, f.provider.deleteCalls(), "the provider is not asked while the snapshot may be in progress")
+			_, held := f.providerHolds(id)
+			assert.True(t, held)
+			assert.Equal(t, []string{id}, f.refsOf(pendingVolumeID), "the reference stays for the reaper")
+			assert.Empty(t, f.claimHolder(id), "the claim is released once the mark is written")
+			f.assertHidden(svc, id)
+		})
+	}
+}
+
+// The InUse checks run first, unchanged, on a pending snapshot too.
+func TestDeleteSnapshot_PendingInUseIsStillRefused(t *testing.T) {
+	const id = "snap-markinuse00001"
+	cases := []struct {
+		name string
+		seed func(f *pendingFixture)
+	}{
+		{name: "a volume was created from it", seed: func(f *pendingFixture) {
+			require.NoError(f.t, f.metadata.PutVolume(f.t.Context(), ebsmetadata.Volume{
+				VolumeID: "vol-fromsnap000001", TenantID: testAccountID, SnapshotID: id, State: "available",
+			}))
+		}},
+		{name: "a registered AMI names it", seed: func(f *pendingFixture) {
+			require.NoError(f.t, f.metadata.PutAMI(f.t.Context(), ebsmetadata.AMI{ImageID: "ami-0aaaaaaaaaaaaaaa1", SnapshotID: id, State: ebsmetadata.AMIStatePending}))
+		}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newPendingFixture(t)
+			svc := f.node(nodeA)
+			f.seedDoc(ebsmetadata.Snapshot{SnapshotID: id, State: snapshotStatePending}, true)
+			tc.seed(f)
+
+			err := f.deleteSnapshot(svc, id)
+			require.Error(t, err)
+			assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidSnapshotInUse), "got %v", err)
+			doc, ok := f.doc(id)
+			require.True(t, ok)
+			assert.False(t, doc.DeleteRequested)
+		})
+	}
+}
+
+// A claim another live node holds is retried three times and then answered
+// IncorrectState; one a stopped node holds is taken over.
+func TestDeleteSnapshot_ClaimContention(t *testing.T) {
+	const id = "snap-markheld000001"
+	cases := []struct {
+		name        string
+		holder      string
+		holderSeen  time.Time
+		wantErr     string
+		wantRefused int
+	}{
+		{name: "held by a live node", holder: nodeC, holderSeen: time.Now(), wantErr: awserrors.ErrorIncorrectState, wantRefused: claimRetries + 1},
+		{name: "held by a node whose heartbeat stopped", holder: nodeC, holderSeen: time.Now().Add(-2 * clusterv1.LivenessWindow)},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newPendingFixture(t)
+			svc := fastClaims(f.node(nodeA))
+			refused := 0
+			svc.claimRefused = func(string) { refused++ }
+			f.seedDoc(ebsmetadata.Snapshot{SnapshotID: id, State: snapshotStatePending}, true)
+			f.holdClaim(tc.holder, id)
+			f.liveness.beat(tc.holder, tc.holderSeen)
+
+			err := f.deleteSnapshot(svc, id)
+
+			assert.Equal(t, tc.wantRefused, refused)
+			doc, ok := f.doc(id)
+			require.True(t, ok)
+			if tc.wantErr != "" {
+				require.Error(t, err)
+				assert.True(t, awserrors.IsErrorCode(err, tc.wantErr), "got %v", err)
+				assert.False(t, doc.DeleteRequested)
+				assert.Equal(t, tc.holder, f.claimHolder(id))
+				return
+			}
+			require.NoError(t, err)
+			assert.True(t, doc.DeleteRequested)
+			assert.Empty(t, f.claimHolder(id))
+		})
+	}
+}
+
+// Each row of the marked column: the reaper deletes a marked snapshot once
+// the provider settles it, never repeats one the provider does not hold, and
+// keeps a tombstone in place of a document that had an unanswered create.
+func TestPendingSnapshotReaper_MarkedColumn(t *testing.T) {
+	const id = "snap-markedrow00001"
+	cases := []struct {
+		name      string
+		ambiguous bool
+		setup     func(f *pendingFixture)
+		// wantKept: the document stays marked, its reference and provider snapshot untouched.
+		wantKept      bool
+		wantTombstone bool
+		wantDeletes   int // provider DeleteSnapshot calls
+	}{
+		{name: "provider pending", setup: func(f *pendingFixture) {
+			f.mem.AnswerSnapshotsPending(true)
+			providerSnapshot(f, id)
+		}, wantKept: true},
+		{name: "provider completed", setup: func(f *pendingFixture) { providerSnapshot(f, id) }, wantDeletes: 1},
+		{name: "provider error", setup: func(f *pendingFixture) {
+			f.mem.AnswerSnapshotsPending(true)
+			providerSnapshot(f, id)
+			require.NoError(f.t, f.mem.SettleSnapshot(id, ebsprovider.SnapshotStateError))
+		}, wantDeletes: 1},
+		{name: "provider completed, create unanswered", ambiguous: true, setup: func(f *pendingFixture) { providerSnapshot(f, id) },
+			wantTombstone: true, wantDeletes: 1},
+		{name: "provider delete answers not_found", setup: func(f *pendingFixture) {
+			providerSnapshot(f, id)
+			f.provider.onDelete = func(ebsprovider.DeleteSnapshotRequest) error { return ebsprovider.ErrNotFound }
+		}, wantDeletes: 1},
+		{name: "absent", wantDeletes: 0},
+		{name: "absent, create unanswered", ambiguous: true, wantTombstone: true},
+		{name: "absent, claim held by a live node", setup: func(f *pendingFixture) { f.holdClaim(nodeC, id) }, wantKept: true},
+		{name: "provider read fails", setup: func(f *pendingFixture) {
+			f.provider.onList = func(ebsprovider.ListSnapshotsRequest) (bool, *ebsprovider.ListSnapshotsResponse, error) {
+				return true, nil, ebsprovider.ErrUnavailable
+			}
+		}, wantKept: true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newPendingFixture(t)
+			reaper := f.node(nodeB)
+			f.seedDoc(ebsmetadata.Snapshot{SnapshotID: id, State: snapshotStatePending, DeleteRequested: true, AmbiguousCreate: tc.ambiguous}, true)
+			if tc.setup != nil {
+				tc.setup(f)
+			}
+			_, heldBefore := f.providerHolds(id)
+
+			f.sweep(reaper)
+
+			assert.Zero(t, f.provider.createCalls(), "a marked snapshot is never repeated")
+			assert.Len(t, f.provider.deleteCalls(), tc.wantDeletes)
+			doc, ok := f.doc(id)
+			switch {
+			case tc.wantKept:
+				require.True(t, ok)
+				assert.True(t, doc.DeleteRequested)
+				assert.False(t, doc.Tombstoned())
+				assert.Equal(t, []string{id}, f.refsOf(pendingVolumeID))
+				_, held := f.providerHolds(id)
+				assert.Equal(t, heldBefore, held)
+			case tc.wantTombstone:
+				require.True(t, ok, "a document that had an unanswered create is kept as a tombstone")
+				assert.True(t, doc.Tombstoned())
+				assert.Empty(t, f.refsOf(pendingVolumeID))
+				f.assertHidden(f.node(nodeA), id)
+			default:
+				assert.False(t, ok)
+				assert.Empty(t, f.refsOf(pendingVolumeID))
+			}
+			if !tc.wantKept && tc.wantDeletes > 0 && f.provider.onDelete == nil {
+				_, held := f.providerHolds(id)
+				assert.False(t, held)
+			}
+		})
+	}
+}
+
+// A failure or interruption between the provider delete, the reference's
+// removal and the document's removal leaves the marked document for the next
+// sweep, which finishes; no reference is ever left without its document.
+func TestPendingSnapshotReaper_CleanupInterrupted(t *testing.T) {
+	const id = "snap-cleanupstep001"
+	cases := []struct {
+		name      string
+		ambiguous bool
+		break_    func(f *pendingFixture, reaper *SnapshotServiceImpl)
+		heal      func(f *pendingFixture, reaper *SnapshotServiceImpl)
+		wantRef   bool // the reference is still there after the broken sweep
+	}{
+		{name: "the provider delete fails", break_: func(f *pendingFixture, _ *SnapshotServiceImpl) {
+			f.provider.onDelete = func(ebsprovider.DeleteSnapshotRequest) error { return ebsprovider.ErrUnavailable }
+		}, heal: func(f *pendingFixture, _ *SnapshotServiceImpl) { f.provider.onDelete = nil }, wantRef: true},
+		{name: "interrupted after the provider delete", break_: func(_ *pendingFixture, r *SnapshotServiceImpl) {
+			r.reapAt = func(step reapStep, _ string) bool { return step == reapProviderDeleted }
+		}, heal: func(_ *pendingFixture, r *SnapshotServiceImpl) { r.reapAt = nil }, wantRef: true},
+		{name: "the reference removal fails", break_: func(f *pendingFixture, r *SnapshotServiceImpl) {
+			r.snapRefs = f.unreachableRefs()
+		}, heal: func(f *pendingFixture, r *SnapshotServiceImpl) { r.snapRefs = f.node(nodeB).snapRefs }, wantRef: true},
+		{name: "interrupted after the reference removal", break_: func(_ *pendingFixture, r *SnapshotServiceImpl) {
+			r.reapAt = func(step reapStep, _ string) bool { return step == reapRefRemoved }
+		}, heal: func(_ *pendingFixture, r *SnapshotServiceImpl) { r.reapAt = nil }},
+		{name: "the document removal fails", break_: func(f *pendingFixture, _ *SnapshotServiceImpl) { f.store.failDelete = true },
+			heal: func(f *pendingFixture, _ *SnapshotServiceImpl) { f.store.failDelete = false }},
+		{name: "the tombstone write fails", ambiguous: true, break_: func(f *pendingFixture, _ *SnapshotServiceImpl) { f.store.putsBefore = 0 },
+			heal: func(f *pendingFixture, _ *SnapshotServiceImpl) { f.store.putsBefore = -1 }},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newPendingFixture(t)
+			reaper := f.node(nodeB)
+			f.seedDoc(ebsmetadata.Snapshot{SnapshotID: id, State: snapshotStatePending, DeleteRequested: true, AmbiguousCreate: tc.ambiguous}, true)
+			providerSnapshot(f, id)
+
+			tc.break_(f, reaper)
+			f.sweep(reaper)
+
+			doc, ok := f.doc(id)
+			require.True(t, ok, "the document is the record the next sweep finds the work from")
+			assert.True(t, doc.DeleteRequested)
+			assert.False(t, doc.Tombstoned())
+			if tc.wantRef {
+				assert.Equal(t, []string{id}, f.refsOf(pendingVolumeID))
+			} else {
+				assert.Empty(t, f.refsOf(pendingVolumeID))
+			}
+
+			tc.heal(f, reaper)
+			f.sweep(reaper)
+
+			_, held := f.providerHolds(id)
+			assert.False(t, held)
+			assert.Empty(t, f.refsOf(pendingVolumeID))
+			doc, ok = f.doc(id)
+			if tc.ambiguous {
+				require.True(t, ok)
+				assert.True(t, doc.Tombstoned())
+			} else {
+				assert.False(t, ok)
+			}
+		})
+	}
+}
+
+// A DeleteSnapshot that lands while the reaper holds the claim, between its
+// reread and its write, waits on the claim and ends with the snapshot deleted,
+// whether it runs on another node or on the reaper's own.
+func TestDeleteSnapshot_LandsBetweenTheReapersRereadAndWrite(t *testing.T) {
+	const id = "snap-deleteraced001"
+	for _, sameNode := range []bool{false, true} {
+		name := "another node"
+		if sameNode {
+			name = "the reaper's node"
+		}
+		t.Run(name, func(t *testing.T) {
+			f := newPendingFixture(t)
+			reaper := fastClaims(f.node(nodeB))
+			deleter := fastClaims(f.node(nodeA))
+			if sameNode {
+				deleter = reaper
+			}
+			f.seedDoc(ebsmetadata.Snapshot{SnapshotID: id, State: snapshotStatePending}, true)
+			providerSnapshot(f, id)
+
+			refused := make(chan struct{}, claimRetries+1)
+			deleter.claimRefused = func(string) { refused <- struct{}{} }
+			var wg sync.WaitGroup
+			var deleteErr error
+			reaper.reapAt = func(step reapStep, _ string) bool {
+				if step == reapReread {
+					wg.Go(func() { deleteErr = f.deleteSnapshot(deleter, id) })
+					<-refused
+				}
+				return false
+			}
+			deleter.claimRetryDelay = 50 * time.Millisecond
+
+			f.sweep(reaper)
+			wg.Wait()
+
+			require.NoError(t, deleteErr, "the delete waits out the reaper's claim")
+			_, ok := f.doc(id)
+			assert.False(t, ok, "the delete is not undone by the reaper's settle")
+			_, held := f.providerHolds(id)
+			assert.False(t, held)
+			assert.Empty(t, f.refsOf(pendingVolumeID))
+		})
+	}
+}
+
+// A request whose provider call returns after its claim was taken over and the
+// snapshot marked leaves the document marked and pending, for the reaper.
+func TestCreateSnapshot_AnswerAfterTheMarkIsNotWritten(t *testing.T) {
+	f := newPendingFixture(t)
+	svc := f.node(nodeA)
+	f.liveness.beat(nodeA, time.Now().Add(-2*clusterv1.LivenessWindow))
+	deleter := fastClaims(f.node(nodeB))
+	f.provider.onCreate = func(n int, req ebsprovider.CreateSnapshotRequest) (bool, *ebsprovider.Snapshot, error) {
+		if n == 1 {
+			require.NoError(t, f.deleteSnapshot(deleter, req.SnapshotID))
+		}
+		return false, nil, nil
+	}
+
+	_, err := f.create(svc)
+	require.NoError(t, err, "the provider answered, so the request succeeds")
+
+	doc := f.onlyDoc()
+	assert.True(t, doc.DeleteRequested)
+	assert.Equal(t, snapshotStatePending, doc.State, "a marked document is never settled by the request")
+	assert.True(t, doc.AmbiguousCreate)
+
+	f.sweep(f.node(nodeC))
+	assert.True(t, f.onlyDoc().Tombstoned())
+	_, held := f.providerHolds(doc.SnapshotID)
+	assert.False(t, held)
+}
+
+// AmbiguousCreate is on the document whenever the provider is called, and is
+// cleared only by a definite answer to a document with no earlier call left
+// unanswered.
+func TestAmbiguousCreate_SetBeforeEveryCall(t *testing.T) {
+	const reaped = "snap-ambiguousreap1"
+	cases := []struct {
+		name string
+		// reaper seeds a pending document with the flag clear and sweeps it,
+		// instead of a CreateSnapshot.
+		reaper  bool
+		pending bool              // the memory provider answers pending
+		onCall  func(n int) error // nil error: the memory provider answers
+		crash   createStep
+		want    bool
+		wantErr bool
+	}{
+		{name: "request answered completed", want: false},
+		{name: "request answered pending", pending: true, want: false},
+		{name: "request first call unanswered, repeat completed", onCall: firstFails(errAmbiguous), want: true},
+		{name: "request crashed after sending", crash: stepAnswered, want: true, wantErr: true},
+		{name: "request repeat refused for good", onCall: func(n int) error {
+			if n == 1 {
+				return errAmbiguous
+			}
+			return ebsprovider.ErrAlreadyExists
+		}, want: true, wantErr: true},
+		{name: "reaper repeat answered", reaper: true, want: false},
+		{name: "reaper repeat unanswered", reaper: true, onCall: firstFails(errAmbiguous), want: true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newPendingFixture(t)
+			svc := f.node(nodeA)
+			f.mem.AnswerSnapshotsPending(tc.pending)
+			var flagged []bool
+			f.provider.onCreate = func(n int, req ebsprovider.CreateSnapshotRequest) (bool, *ebsprovider.Snapshot, error) {
+				doc, _ := f.doc(req.SnapshotID)
+				flagged = append(flagged, doc.AmbiguousCreate)
+				if tc.onCall != nil {
+					if err := tc.onCall(n); err != nil {
+						return true, nil, err
+					}
+				}
+				return false, nil, nil
+			}
+			if tc.crash != 0 {
+				svc.crashAt = func(step createStep) bool { return step == tc.crash }
+			}
+
+			var id string
+			if tc.reaper {
+				id = reaped
+				f.seedDoc(ebsmetadata.Snapshot{SnapshotID: id, State: snapshotStatePending}, true)
+				f.sweep(f.node(nodeB))
+			} else {
+				_, err := f.create(svc)
+				assert.Equal(t, tc.wantErr, err != nil, "err %v", err)
+				id = f.onlyDoc().SnapshotID
+			}
+
+			require.NotEmpty(t, flagged)
+			for i, set := range flagged {
+				assert.True(t, set, "call %d was sent with AmbiguousCreate clear", i+1)
+			}
+			doc, ok := f.doc(id)
+			require.True(t, ok)
+			assert.Equal(t, tc.want, doc.AmbiguousCreate)
+		})
+	}
+}
+
+// firstFails answers the first call with err and lets the rest through.
+func firstFails(err error) func(int) error {
+	return func(n int) error {
+		if n == 1 {
+			return err
+		}
+		return nil
+	}
+}
+
+// The original call of a snapshot that a repeat completed and the customer
+// then deleted is accepted late: the snapshot it recreates is deleted under
+// the tombstone once it settles, and the tombstone is dropped after 7 days.
+func TestTombstone_DeletesALateAcceptedSnapshotAndExpires(t *testing.T) {
+	f := newPendingFixture(t)
+	svc := f.node(nodeA)
+	reaper := f.node(nodeB)
+	now := time.Now()
+	reaper.now = func() time.Time { return now }
+	var late ebsprovider.CreateSnapshotRequest
+	f.provider.onCreate = func(n int, req ebsprovider.CreateSnapshotRequest) (bool, *ebsprovider.Snapshot, error) {
+		if n <= 2 {
+			late = req
+			return true, nil, errAmbiguous
+		}
+		return false, nil, nil
+	}
+	_, err := f.create(svc)
+	require.EqualError(t, err, awserrors.ErrorServerInternal)
+	id := f.onlyDoc().SnapshotID
+
+	// The reaper's repeat completes it, and the customer deletes it.
+	f.sweep(reaper)
+	doc := f.onlyDoc()
+	require.Equal(t, ebsmetadata.SnapshotStateCompleted, doc.State)
+	require.True(t, doc.AmbiguousCreate, "the request's calls were never answered")
+	require.NoError(t, f.deleteSnapshot(svc, id))
+	f.sweep(reaper)
+	require.True(t, f.onlyDoc().Tombstoned())
+	_, held := f.providerHolds(id)
+	require.False(t, held)
+
+	// The request's original call is accepted now, and is still in progress.
+	f.mem.AnswerSnapshotsPending(true)
+	_, err = f.mem.CreateSnapshot(t.Context(), late)
+	require.NoError(t, err)
+	f.sweep(reaper)
+	_, held = f.providerHolds(id)
+	assert.True(t, held, "a snapshot still pending is not deleted")
+	require.NoError(t, f.mem.SettleSnapshot(id, ebsprovider.SnapshotStateCompleted))
+	f.sweep(reaper)
+	_, held = f.providerHolds(id)
+	assert.False(t, held, "the late snapshot is deleted under the tombstone")
+	assert.True(t, f.onlyDoc().Tombstoned())
+	f.assertHidden(svc, id)
+	assert.Equal(t, 3, f.provider.createCalls(), "the request's two calls and one repeat; nothing repeats a tombstone's create")
+
+	now = now.Add(tombstoneRetention - time.Minute)
+	f.sweep(reaper)
+	assert.Len(t, f.docs(), 1, "the tombstone is kept for its retention")
+	now = now.Add(2 * time.Minute)
+	f.sweep(reaper)
+	assert.Empty(t, f.docs(), "the tombstone is dropped after its retention")
+	assert.Empty(t, f.refsOf(pendingVolumeID))
+}
+
+// The AMI pass settles each pending AMI from its snapshot's document.
+func TestPendingSnapshotReaper_SettlesPendingAMIs(t *testing.T) {
+	const snapID = "snap-amisettle00001"
+	cases := []struct {
+		name     string
+		amiState string
+		snap     *ebsmetadata.Snapshot // nil: no snapshot document
+		want     string
+	}{
+		{name: "snapshot completed", amiState: ebsmetadata.AMIStatePending, snap: &ebsmetadata.Snapshot{State: ebsmetadata.SnapshotStateCompleted}, want: ebsmetadata.AMIStateAvailable},
+		{name: "snapshot errored", amiState: ebsmetadata.AMIStatePending, snap: &ebsmetadata.Snapshot{State: snapshotStateError}, want: ebsmetadata.AMIStateFailed},
+		{name: "snapshot document missing", amiState: ebsmetadata.AMIStatePending, want: ebsmetadata.AMIStateFailed},
+		{name: "snapshot pending", amiState: ebsmetadata.AMIStatePending, snap: &ebsmetadata.Snapshot{State: snapshotStatePending}, want: ebsmetadata.AMIStatePending},
+		{name: "AMI already available", amiState: ebsmetadata.AMIStateAvailable, want: ebsmetadata.AMIStateAvailable},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newPendingFixture(t)
+			if tc.snap != nil {
+				tc.snap.SnapshotID = snapID
+				f.seedDoc(*tc.snap, true)
+				if tc.snap.State == snapshotStatePending {
+					f.mem.AnswerSnapshotsPending(true)
+					providerSnapshot(f, snapID)
+				}
+			}
+			const amiID = "ami-0bbbbbbbbbbbbbbb1"
+			require.NoError(t, f.metadata.PutAMI(t.Context(), ebsmetadata.AMI{
+				ImageID: amiID, SnapshotID: snapID, ImageOwnerAlias: testAccountID, State: tc.amiState,
+			}))
+
+			f.sweep(f.node(nodeB))
+
+			ami, err := f.metadata.GetAMI(t.Context(), amiID)
+			require.NoError(t, err)
+			assert.Equal(t, tc.want, ami.State)
+		})
+	}
+}
+
+func TestCopySnapshot_RefusesASourceThatIsNotCompleted(t *testing.T) {
+	for _, state := range []string{snapshotStatePending, "error"} {
+		t.Run(state, func(t *testing.T) {
+			svc, _ := providerSnapshotService(t, "vol-copygate")
+			require.NoError(t, svc.MetadataStore().PutSnapshot(t.Context(), ebsmetadata.Snapshot{
+				SnapshotID: "snap-copygate00001", VolumeID: "vol-copygate", VolumeSize: 4, State: state, OwnerID: testAccountID,
+			}))
+
+			_, err := svc.CopySnapshot(t.Context(), &ec2.CopySnapshotInput{SourceSnapshotId: aws.String("snap-copygate00001")}, testAccountID)
+			code, _, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok, "unregistered error: %v", err)
+			assert.Equal(t, awserrors.ErrorIncorrectState, code)
+			snaps, err := svc.MetadataStore().ListAllSnapshots(t.Context())
+			require.NoError(t, err)
+			assert.Len(t, snaps, 1, "a refused copy writes no snapshot document")
+		})
+	}
+}

spinifex/domains/ec2/snapshot/pending.go

@@ -0,0 +1,113 @@
+package snapshot
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+)
+
+// The snapshot document states CreateSnapshot and the reaper move between.
+const (
+	snapshotStatePending = string(ebsprovider.SnapshotStatePending)
+	snapshotStateError   = string(ebsprovider.SnapshotStateError)
+)
+
+// definiteRefusal reports a CreateSnapshot error that is the provider's answer
+// that it holds no snapshot for the call. Anything else, a timeout, a lost
+// connection or unavailable, may follow an accepted call.
+func definiteRefusal(err error) bool {
+	return errors.Is(err, ebsprovider.ErrInvalidArgument) || errors.Is(err, ebsprovider.ErrNotFound) ||
+		errors.Is(err, ebsprovider.ErrAlreadyExists) || errors.Is(err, ebsprovider.ErrVolumeInUse)
+}
+
+// refusalSettlesError reports a refusal no later call can change: the source
+// volume is gone, or the ID is another volume's snapshot. Only these make a
+// pending document error once an earlier call may still be accepted.
+func refusalSettlesError(err error) bool {
+	return errors.Is(err, ebsprovider.ErrNotFound) || errors.Is(err, ebsprovider.ErrAlreadyExists)
+}
+
+// applyProviderAnswer moves a document to what the provider answered: completed
+// at 100%, or pending with the provider's handle for the reaper to follow. The
+// call was answered, so AmbiguousCreate stays only if an earlier one was not.
+func applyProviderAnswer(created *ebsprovider.Snapshot, earlierUnanswered bool) func(*ebsmetadata.Snapshot) {
+	return func(doc *ebsmetadata.Snapshot) {
+		doc.AmbiguousCreate = earlierUnanswered
+		if created.Handle != "" {
+			doc.ProviderHandle = created.Handle
+		}
+		if created.State == ebsprovider.SnapshotStateCompleted {
+			doc.State = ebsmetadata.SnapshotStateCompleted
+			doc.Progress = "100%"
+		}
+	}
+}
+
+// repeatOutcome is the write a CreateSnapshot answer after an unanswered call
+// makes: answered, the provider's answer; a refusal no later call can change,
+// error; anything else, nil, leaving the document pending.
+func repeatOutcome(created *ebsprovider.Snapshot, err error, earlierUnanswered bool) (mutate func(*ebsmetadata.Snapshot), answered bool) {
+	switch {
+	case err == nil && created != nil:
+		return applyProviderAnswer(created, earlierUnanswered), true
+	case refusalSettlesError(err):
+		return func(d *ebsmetadata.Snapshot) {
+			markSnapshotError(d)
+			d.AmbiguousCreate = earlierUnanswered
+		}, false
+	}
+	return nil, false
+}
+
+func markSnapshotCompleted(doc *ebsmetadata.Snapshot) {
+	doc.State = ebsmetadata.SnapshotStateCompleted
+	doc.Progress = "100%"
+}
+
+func markSnapshotError(doc *ebsmetadata.Snapshot) {
+	doc.State = snapshotStateError
+}
+
+// updatePending rereads a snapshot document and applies mutate only while it
+// is still pending, so a delete or settlement since the caller read it is not
+// undone. It returns the document as read or written, zero when it is gone.
+func (s *SnapshotServiceImpl) updatePending(ctx context.Context, accountID, snapshotID string, mutate func(*ebsmetadata.Snapshot)) (ebsmetadata.Snapshot, bool, error) {
+	return s.rewrite(ctx, accountID, snapshotID, func(doc *ebsmetadata.Snapshot) bool {
+		// A marked document is the reaper's to delete, never to settle.
+		if doc.State != snapshotStatePending || doc.Hidden() {
+			return false
+		}
+		mutate(doc)
+		return true
+	})
+}
+
+// rewrite rereads snapshotID's document and writes it back when mutate asks
+// to. It returns the document as read or written and whether it wrote; one
+// that is gone is zero, and mutate never sees it.
+func (s *SnapshotServiceImpl) rewrite(ctx context.Context, accountID, snapshotID string, mutate func(*ebsmetadata.Snapshot) bool) (ebsmetadata.Snapshot, bool, error) {
+	doc, err := s.metadata.GetSnapshot(ctx, accountID, snapshotID)
+	if err != nil {
+		if objectstore.IsNoSuchKeyError(err) {
+			return ebsmetadata.Snapshot{}, false, nil
+		}
+		return ebsmetadata.Snapshot{}, false, fmt.Errorf("read snapshot %s: %w", snapshotID, err)
+	}
+	if !mutate(&doc) {
+		return doc, false, nil
+	}
+	if err := s.metadata.PutSnapshot(ctx, doc); err != nil {
+		return ebsmetadata.Snapshot{}, false, fmt.Errorf("write snapshot %s: %w", snapshotID, err)
+	}
+	return doc, true, nil
+}
+
+// writeContext bounds a write made after the provider answered, which has to
+// land even when the request that asked has been cancelled.
+func writeContext(ctx context.Context) (context.Context, context.CancelFunc) {
+	return context.WithTimeout(context.WithoutCancel(ctx), snapshotCleanupTimeout)
+}

spinifex/domains/ec2/snapshot/pending_reaper.go

@@ -0,0 +1,464 @@
+package snapshot
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	otelsetup "github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+)
+
+// PendingSnapshotReaper settles every pending snapshot document from the
+// provider. CreateSnapshot returns once the provider answers, and a snapshot
+// that settles later, or a request that crashed or failed ambiguously, leaves
+// its document pending; this sweep is where that work converges (ADR-0003:S8).
+//
+// It deletes nothing a customer did not ask to delete (ADR-0003:S7): only a
+// snapshot whose document DeleteSnapshot marked, its reference and document,
+// and a late-accepted snapshot under the tombstone that delete left.
+type PendingSnapshotReaper struct {
+	svc *SnapshotServiceImpl
+}
+
+// tombstoneRetention is how long a deleted AmbiguousCreate snapshot's document
+// stays, so a create accepted that late is still found and deleted.
+const tombstoneRetention = 7 * 24 * time.Hour
+
+// reapStep is a point in the reaper's work on one document, for a test to
+// interrupt it at.
+type reapStep int
+
+const (
+	reapReread reapStep = iota + 1
+	reapProviderDeleted
+	reapRefRemoved
+)
+
+// NewPendingSnapshotReaper builds the reaper over this service's documents,
+// provider and claims.
+func (s *SnapshotServiceImpl) NewPendingSnapshotReaper() *PendingSnapshotReaper {
+	return &PendingSnapshotReaper{svc: s}
+}
+
+func (r *PendingSnapshotReaper) Class() string { return "pending-snapshot" }
+
+// Sweep reads every snapshot document and settles each pending one it can,
+// carries out each accepted delete, and rereads each tombstone; then it settles
+// each pending AMI from its snapshot. It reports how many documents it wrote.
+func (r *PendingSnapshotReaper) Sweep(ctx context.Context) (int, error) {
+	if r.svc.provider == nil {
+		return 0, nil
+	}
+	docs, err := r.svc.metadata.ListAllSnapshotsStrict(ctx)
+	if err != nil {
+		return 0, fmt.Errorf("list snapshot documents: %w", err)
+	}
+	written := 0
+	for _, doc := range docs {
+		if err := ctx.Err(); err != nil {
+			return written, err
+		}
+		var wrote bool
+		switch {
+		case doc.Tombstoned():
+			wrote = r.sweepTombstone(ctx, doc)
+		case doc.DeleteRequested, doc.State == snapshotStatePending:
+			wrote = r.settle(ctx, doc)
+		default:
+			continue
+		}
+		if wrote {
+			written++
+		}
+	}
+	amis, err := r.settleAMIs(ctx)
+	return written + amis, err
+}
+
+// interrupted reports a test stopping the reaper at step for snapshotID.
+func (r *PendingSnapshotReaper) interrupted(step reapStep, snapshotID string) bool {
+	return r.svc.reapAt != nil && r.svc.reapAt(step, snapshotID)
+}
+
+// providerSnapshot reads snapshotID from the provider. found is false when the
+// provider holds no snapshot by that ID.
+func (r *PendingSnapshotReaper) providerSnapshot(ctx context.Context, doc ebsmetadata.Snapshot) (ebsprovider.SnapshotRef, bool, error) {
+	ref, found, err := ebsprovider.ReadSnapshot(ctx, r.svc.provider, doc.SnapshotID)
+	if err != nil {
+		slog.WarnContext(ctx, "pending-snapshot: provider read failed, retrying next sweep",
+			"snapshotId", doc.SnapshotID, "accountId", doc.OwnerID, "err", err)
+	}
+	return ref, found, err
+}
+
+// settle follows the provider's state for doc, carries out its accepted
+// delete, or repeats its CreateSnapshot when the provider holds no snapshot by
+// that ID. It reports whether it wrote.
+func (r *PendingSnapshotReaper) settle(ctx context.Context, doc ebsmetadata.Snapshot) bool {
+	// A marked document is only cleaned up, from a provider read under its claim.
+	if doc.DeleteRequested {
+		return r.underClaim(ctx, doc, nil)
+	}
+	ref, found, err := r.providerSnapshot(ctx, doc)
+	if err != nil {
+		return false
+	}
+	if found {
+		return r.follow(ctx, doc, ref)
+	}
+	return r.underClaim(ctx, doc, func(current ebsmetadata.Snapshot) bool { return r.repeat(ctx, current) })
+}
+
+// providerSettled reports a provider snapshot that is completed or errored,
+// the states in which a provider deletes it.
+func providerSettled(ref ebsprovider.SnapshotRef) bool {
+	return ref.State == ebsprovider.SnapshotStateCompleted || ref.State == ebsprovider.SnapshotStateError
+}
+
+// withClaim runs fn under id's claim, which take acquires, and reports false
+// without running it when the claim is not taken.
+func withClaim(ctx context.Context, id string, take func(context.Context, string) (*heldClaim, error), fn func() bool) bool {
+	held, err := take(ctx, id)
+	if err != nil {
+		level := slog.LevelWarn
+		if errors.Is(err, errClaimHeld) {
+			level = slog.LevelInfo
+		}
+		slog.Log(ctx, level, "pending-snapshot: claim not taken, left for the next sweep", "id", id, "err", err)
+		return false
+	}
+	defer held.release(ctx)
+	return fn()
+}
+
+// underClaim takes doc's claim, rereads its document and runs act on it while
+// it is unmarked and pending. A marked document, however it was read before,
+// is cleaned up instead.
+func (r *PendingSnapshotReaper) underClaim(ctx context.Context, doc ebsmetadata.Snapshot, act func(ebsmetadata.Snapshot) bool) bool {
+	return withClaim(ctx, doc.SnapshotID, r.svc.claims.take, func() bool {
+		// The claim may have been taken after the creating request rolled back or
+		// settled the document, so it is read again before anything is written.
+		current, err := r.svc.metadata.GetSnapshot(ctx, doc.OwnerID, doc.SnapshotID)
+		if err != nil || current.Tombstoned() {
+			return false
+		}
+		if r.interrupted(reapReread, doc.SnapshotID) {
+			return false
+		}
+		if current.DeleteRequested {
+			return r.cleanUpIfSettled(ctx, current)
+		}
+		if current.State != snapshotStatePending {
+			return false
+		}
+		return act(current)
+	})
+}
+
+// cleanUpIfSettled reads doc's snapshot from the provider and carries out its
+// accepted delete once the provider no longer holds it pending.
+func (r *PendingSnapshotReaper) cleanUpIfSettled(ctx context.Context, doc ebsmetadata.Snapshot) bool {
+	ref, found, err := r.providerSnapshot(ctx, doc)
+	if err != nil || (found && !providerSettled(ref)) {
+		return false
+	}
+	return r.cleanUp(ctx, doc, ref, found)
+}
+
+// deleteAtProvider deletes snapshotID at the provider; one already gone there
+// is deleted.
+func (r *PendingSnapshotReaper) deleteAtProvider(ctx context.Context, snapshotID, handle string) error {
+	err := r.svc.provider.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{
+		Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, Handle: handle,
+	})
+	if errors.Is(err, ebsprovider.ErrNotFound) {
+		return nil
+	}
+	return err
+}
+
+// follow writes the state the provider reports for a snapshot it holds.
+func (r *PendingSnapshotReaper) follow(ctx context.Context, doc ebsmetadata.Snapshot, ref ebsprovider.SnapshotRef) bool {
+	var mutate func(*ebsmetadata.Snapshot)
+	switch ref.State {
+	case ebsprovider.SnapshotStateCompleted:
+		mutate = func(d *ebsmetadata.Snapshot) {
+			markSnapshotCompleted(d)
+			if d.ProviderHandle == "" {
+				d.ProviderHandle = ref.Handle
+			}
+		}
+	case ebsprovider.SnapshotStateError:
+		mutate = markSnapshotError
+	case ebsprovider.SnapshotStatePending:
+		return false
+	default:
+		slog.WarnContext(ctx, "pending-snapshot: provider reported no state for a held snapshot, retrying next sweep",
+			"snapshotId", doc.SnapshotID, "state", ref.State)
+		return false
+	}
+	return r.underClaim(ctx, doc, func(current ebsmetadata.Snapshot) bool { return r.write(ctx, current, mutate) })
+}
+
+// repeat asks the provider again, under the snapshot's claim, for a pending
+// snapshot it does not hold. Absence alone settles nothing, because a call the
+// creating request sent may still be accepted; the repeat returns that snapshot or makes it.
+func (r *PendingSnapshotReaper) repeat(ctx context.Context, current ebsmetadata.Snapshot) bool {
+	handle, ok := r.volumeHandle(ctx, current)
+	if !ok {
+		return false
+	}
+	// The flag goes on before the call, so a crash once it is sent leaves it.
+	earlierUnanswered := current.AmbiguousCreate
+	if !earlierUnanswered {
+		if _, ok, err := r.svc.updatePending(ctx, current.OwnerID, current.SnapshotID, func(d *ebsmetadata.Snapshot) {
+			d.AmbiguousCreate = true
+		}); err != nil || !ok {
+			slog.WarnContext(ctx, "pending-snapshot: could not record the repeat before sending it, retrying next sweep",
+				"snapshotId", current.SnapshotID, "err", err)
+			return false
+		}
+	}
+	created, err := r.svc.provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{
+		Versioned: ebsprovider.NewLatestVersioned(), SnapshotID: current.SnapshotID, VolumeID: current.VolumeID, VolumeHandle: handle,
+	})
+	mutate, answered := repeatOutcome(created, err, earlierUnanswered)
+	switch {
+	case answered:
+		return r.write(ctx, current, mutate)
+	case mutate != nil:
+		slog.WarnContext(ctx, "pending-snapshot: provider refused the repeat for good, marking the snapshot error",
+			"snapshotId", current.SnapshotID, "volumeId", current.VolumeID, "err", err)
+		return r.write(ctx, current, mutate)
+	default:
+		slog.WarnContext(ctx, "pending-snapshot: repeat not answered, retrying next sweep",
+			"snapshotId", current.SnapshotID, "volumeId", current.VolumeID, "err", err)
+		return false
+	}
+}
+
+// cleanUp carries out an accepted delete, under the snapshot's claim: the
+// provider's delete, then the reference, then the document, or a tombstone if
+// a create on it went unanswered. Each step must succeed before the next.
+func (r *PendingSnapshotReaper) cleanUp(ctx context.Context, doc ebsmetadata.Snapshot, ref ebsprovider.SnapshotRef, found bool) bool {
+	// A marked snapshot the provider does not hold is never repeated: the
+	// customer asked for it gone, and a tombstone catches a late acceptance.
+	if found {
+		handle := ref.Handle
+		if handle == "" {
+			handle = doc.ProviderHandle
+		}
+		if err := r.deleteAtProvider(ctx, doc.SnapshotID, handle); err != nil {
+			slog.WarnContext(ctx, "pending-snapshot: provider delete of a deleted snapshot failed, retrying next sweep",
+				"snapshotId", doc.SnapshotID, "err", err)
+			return false
+		}
+		if r.interrupted(reapProviderDeleted, doc.SnapshotID) {
+			return false
+		}
+	}
+	// The document goes last: it is the only record this sweep discovers work
+	// from, and a reference left without it would block DeleteVolume for good.
+	if err := r.svc.removeSnapshotRef(ctx, doc.VolumeID, doc.SnapshotID); err != nil {
+		slog.WarnContext(ctx, "pending-snapshot: reference of a deleted snapshot not removed, retrying next sweep",
+			"snapshotId", doc.SnapshotID, "volumeId", doc.VolumeID, "err", err)
+		return false
+	}
+	if r.interrupted(reapRefRemoved, doc.SnapshotID) {
+		return false
+	}
+	if doc.AmbiguousCreate {
+		doc.TombstonedAt = r.svc.clock().UTC()
+		if err := r.svc.metadata.PutSnapshot(ctx, doc); err != nil {
+			slog.WarnContext(ctx, "pending-snapshot: tombstone of a deleted snapshot not written, retrying next sweep",
+				"snapshotId", doc.SnapshotID, "err", err)
+			return false
+		}
+		slog.InfoContext(ctx, "pending-snapshot: deleted, keeping a tombstone for a create that may still be accepted",
+			"snapshotId", doc.SnapshotID)
+		return true
+	}
+	if err := r.svc.metadata.DeleteSnapshot(ctx, doc.OwnerID, doc.SnapshotID); err != nil && !objectstore.IsNoSuchKeyError(err) {
+		slog.WarnContext(ctx, "pending-snapshot: document of a deleted snapshot not removed, retrying next sweep",
+			"snapshotId", doc.SnapshotID, "err", err)
+		return false
+	}
+	slog.InfoContext(ctx, "pending-snapshot: deleted", "snapshotId", doc.SnapshotID)
+	return true
+}
+
+// sweepTombstone reads the provider for a tombstone's ID and deletes a snapshot a late
+// create made there once it is no longer pending. A tombstone past its
+// retention, with nothing pending under it, is dropped.
+func (r *PendingSnapshotReaper) sweepTombstone(ctx context.Context, doc ebsmetadata.Snapshot) bool {
+	return withClaim(ctx, doc.SnapshotID, r.svc.claims.take, func() bool {
+		ref, found, err := r.providerSnapshot(ctx, doc)
+		if err != nil {
+			return false
+		}
+		if found {
+			if !providerSettled(ref) {
+				return false
+			}
+			if err := r.deleteAtProvider(ctx, doc.SnapshotID, ref.Handle); err != nil {
+				slog.WarnContext(ctx, "pending-snapshot: late-accepted snapshot of a deleted ID not deleted, retrying next sweep",
+					"snapshotId", doc.SnapshotID, "err", err)
+				return false
+			}
+			slog.WarnContext(ctx, "pending-snapshot: deleted a snapshot accepted after the customer deleted it",
+				"snapshotId", doc.SnapshotID)
+		}
+		age := r.svc.clock().Sub(doc.TombstonedAt)
+		if age < tombstoneRetention {
+			return found
+		}
+		if err := r.svc.metadata.DeleteSnapshot(ctx, doc.OwnerID, doc.SnapshotID); err != nil && !objectstore.IsNoSuchKeyError(err) {
+			slog.WarnContext(ctx, "pending-snapshot: expired tombstone not dropped, retrying next sweep",
+				"snapshotId", doc.SnapshotID, "err", err)
+			return found
+		}
+		slog.InfoContext(ctx, "pending-snapshot: dropped an expired tombstone", "snapshotId", doc.SnapshotID, "age_ms", otelsetup.Millis(age))
+		return true
+	})
+}
+
+// settleAMIs moves each pending AMI to its snapshot's state: completed makes it
+// available, and error or a missing or deleted snapshot makes it failed.
+func (r *PendingSnapshotReaper) settleAMIs(ctx context.Context) (int, error) {
+	amis, err := r.svc.metadata.ListAMIs(ctx)
+	if err != nil {
+		return 0, fmt.Errorf("list AMI documents: %w", err)
+	}
+	written := 0
+	for _, ami := range amis {
+		if err := ctx.Err(); err != nil {
+			return written, err
+		}
+		if ami.State != ebsmetadata.AMIStatePending || ami.SnapshotID == "" {
+			continue
+		}
+		state, ok := r.amiStateFromSnapshot(ctx, ami)
+		if !ok {
+			continue
+		}
+		if r.svc.settlePendingAMI(ctx, ami.ImageID, state, r.svc.claims.take) {
+			written++
+		}
+	}
+	return written, nil
+}
+
+// amiStateFromSnapshot is the state ami's snapshot document gives it, and
+// false while that snapshot is pending or cannot be read.
+func (r *PendingSnapshotReaper) amiStateFromSnapshot(ctx context.Context, ami ebsmetadata.AMI) (string, bool) {
+	owner := awsidentifiers.GlobalAccountID
+	if awsidentifiers.IsAccountID(ami.ImageOwnerAlias) {
+		owner = ami.ImageOwnerAlias
+	}
+	snap, err := r.svc.metadata.GetSnapshot(ctx, owner, ami.SnapshotID)
+	switch {
+	case objectstore.IsNoSuchKeyError(err):
+		return ebsmetadata.AMIStateFailed, true
+	case err != nil:
+		slog.WarnContext(ctx, "pending-snapshot: snapshot of a pending AMI unreadable, retrying next sweep",
+			"imageId", ami.ImageID, "snapshotId", ami.SnapshotID, "err", err)
+		return "", false
+	case snap.Hidden() || snap.State == snapshotStateError:
+		return ebsmetadata.AMIStateFailed, true
+	case snap.State == ebsmetadata.SnapshotStateCompleted:
+		return ebsmetadata.AMIStateAvailable, true
+	default:
+		return "", false
+	}
+}
+
+// volumeHandle is the provider handle of doc's source volume, which the
+// repeat sends as the request did. A volume document that is gone gives no
+// handle, and the provider answers for the volume; one unreadable is retried.
+func (r *PendingSnapshotReaper) volumeHandle(ctx context.Context, doc ebsmetadata.Snapshot) (string, bool) {
+	volume, err := r.svc.metadata.GetVolume(ctx, doc.OwnerID, doc.VolumeID)
+	if err == nil {
+		return volume.ProviderHandle, true
+	}
+	if objectstore.IsNoSuchKeyError(err) {
+		return "", true
+	}
+	slog.WarnContext(ctx, "pending-snapshot: source volume document unreadable, retrying next sweep",
+		"snapshotId", doc.SnapshotID, "volumeId", doc.VolumeID, "err", err)
+	return "", false
+}
+
+// write applies mutate to doc's current document while it is still pending,
+// under a claim the caller holds. A request that crashed between its document
+// and its reference left the reference missing, so it is restored first.
+func (r *PendingSnapshotReaper) write(ctx context.Context, doc ebsmetadata.Snapshot, mutate func(*ebsmetadata.Snapshot)) bool {
+	if err := r.svc.ensureSnapshotRef(ctx, doc.VolumeID, doc.SnapshotID); err != nil {
+		slog.WarnContext(ctx, "pending-snapshot: volume reference not restored, retrying next sweep",
+			"snapshotId", doc.SnapshotID, "volumeId", doc.VolumeID, "err", err)
+		return false
+	}
+	defer r.dropRefIfDeleted(ctx, doc)
+	written, ok, err := r.svc.updatePending(ctx, doc.OwnerID, doc.SnapshotID, mutate)
+	if err != nil {
+		slog.WarnContext(ctx, "pending-snapshot: document write failed, retrying next sweep",
+			"snapshotId", doc.SnapshotID, "accountId", doc.OwnerID, "err", err)
+		return false
+	}
+	if ok && written.State != snapshotStatePending {
+		slog.InfoContext(ctx, "pending-snapshot: settled", "snapshotId", doc.SnapshotID, "state", written.State)
+	}
+	return ok
+}
+
+// dropRefIfDeleted removes doc's reference again when its document is gone: a
+// rollback or DeleteSnapshot may have removed both before write restored it.
+func (r *PendingSnapshotReaper) dropRefIfDeleted(ctx context.Context, doc ebsmetadata.Snapshot) {
+	_, err := r.svc.metadata.GetSnapshot(ctx, doc.OwnerID, doc.SnapshotID)
+	if !objectstore.IsNoSuchKeyError(err) {
+		return
+	}
+	if err := r.svc.removeSnapshotRef(ctx, doc.VolumeID, doc.SnapshotID); err != nil {
+		slog.WarnContext(ctx, "pending-snapshot: reference of a deleted snapshot not removed",
+			"snapshotId", doc.SnapshotID, "volumeId", doc.VolumeID, "err", err)
+	}
+}
+
+// ReleaseNodeClaims releases every snapshot and image claim that names this
+// node and no request of this process holds, whatever its document's state. A
+// daemon runs it as it starts and before it serves requests: none survived.
+func (s *SnapshotServiceImpl) ReleaseNodeClaims(ctx context.Context) (int, error) {
+	docs, err := s.metadata.ListAllSnapshotsStrict(ctx)
+	if err != nil {
+		return 0, fmt.Errorf("list snapshot documents: %w", err)
+	}
+	amis, err := s.metadata.ListAMIsStrict(ctx)
+	if err != nil {
+		return 0, fmt.Errorf("list AMI documents: %w", err)
+	}
+	ids := make([]string, 0, len(docs)+len(amis))
+	for _, doc := range docs {
+		ids = append(ids, doc.SnapshotID)
+	}
+	for _, ami := range amis {
+		ids = append(ids, ami.ImageID)
+	}
+	released := 0
+	var errs []error
+	for _, id := range ids {
+		ok, err := s.claims.releaseOwn(ctx, id)
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		if ok {
+			released++
+		}
+	}
+	return released, errors.Join(errs...)
+}

spinifex/domains/ec2/snapshot/service_impl.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/tagmirror"
 	"log/slog"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -52,6 +53,32 @@ type SnapshotServiceImpl struct {
 	mutex    sync.RWMutex
 	metadata *ebsmetadata.Store
 	provider ebsprovider.EBSProvider
+	claims   *Claims
+
+	// crashAt stops CreateSnapshot after a step, as a crash there would. Tests only.
+	crashAt func(createStep) bool
+	// reapAt interrupts the reaper at a step for snapshotID. Tests only.
+	reapAt func(step reapStep, snapshotID string) bool
+	// now is the reaper's clock for tombstones; nil is time.Now. Tests only.
+	now func() time.Time
+	// claimRetryDelay overrides the wait between claim attempts. Tests only.
+	claimRetryDelay time.Duration
+	// claimRefused observes each refused claim attempt of a writer. Tests only.
+	claimRefused func(snapshotID string)
+}
+
+// clock is the time the reaper judges a tombstone's age by.
+func (s *SnapshotServiceImpl) clock() time.Time {
+	if s.now != nil {
+		return s.now()
+	}
+	return time.Now()
+}
+
+// SetClaims wires the snapshot claims shared by CreateSnapshot and the
+// pending-snapshot reaper. Without it they exclude only within this process.
+func (s *SnapshotServiceImpl) SetClaims(claims *Claims) {
+	s.claims = claims
 }
 
 // SetEBSProvider injects the provider boundary for snapshot operations.
@@ -102,6 +129,7 @@ func NewSnapshotServiceImplWithNATS(ctx context.Context, cfg *config.Config, nat
 		natsConn: natsConn,
 		snapRefs: refs,
 		metadata: ebsmetadata.NewStore(store, cfg.Predastore.Bucket),
+		claims:   localClaims(),
 	}, kv, nil
 }
 
@@ -113,6 +141,7 @@ func NewSnapshotServiceImplWithStore(cfg *config.Config, store objectstore.Objec
 		store:    store,
 		natsConn: natsConn,
 		metadata: ebsmetadata.NewStore(store, cfg.Predastore.Bucket),
+		claims:   localClaims(),
 	}
 	if len(snapshotKV) > 0 {
 		svc.snapRefs = kvstore.Over[[]string](nil, snapshotKV[0], snapshotRefsConfig())
@@ -135,7 +164,8 @@ func snapshotRefsConfig() kvstore.Config {
 }
 
 // getSnapshotConfig reads the caller's own snapshot document. A snapshot
-// outside the caller's prefix is absent, so ownership needs no second check.
+// outside the caller's prefix is absent, so ownership needs no second check,
+// and so is one whose delete was accepted.
 func (s *SnapshotServiceImpl) getSnapshotConfig(ctx context.Context, accountID, snapshotID string) (ebsmetadata.Snapshot, error) {
 	cfg, err := s.metadata.GetSnapshot(ctx, accountID, snapshotID)
 	if err != nil {
@@ -144,6 +174,9 @@ func (s *SnapshotServiceImpl) getSnapshotConfig(ctx context.Context, accountID,
 		}
 		return ebsmetadata.Snapshot{}, err
 	}
+	if cfg.Hidden() {
+		return ebsmetadata.Snapshot{}, errors.New(awserrors.ErrorInvalidSnapshotNotFound)
+	}
 	return cfg, nil
 }
 
@@ -203,47 +236,288 @@ func (s *SnapshotServiceImpl) CreateSnapshot(ctx context.Context, input *ec2.Cre
 		return nil, errors.New(awserrors.ErrorIncorrectState)
 	}
 
-	created, err := s.provider.CreateSnapshot(ctx, ebsprovider.CreateSnapshotRequest{
-		Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: volumeID, VolumeHandle: volume.ProviderHandle,
-	})
-	if err != nil || created == nil {
-		slog.ErrorContext(ctx, "CreateSnapshot: provider CreateSnapshot failed", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
-		return nil, ProviderCreateError(volumeID, err)
-	}
-	now := time.Now()
-	snapshotCfg := ebsmetadata.Snapshot{
-		SnapshotID: snapshotID, VolumeID: volumeID,
+	doc := pendingDocument(volume, snapshotID, accountID, aws.StringValue(input.Description))
+	doc.Tags = awstags.Extract(input.TagSpecifications, "snapshot")
+	return s.createSnapshot(ctx, doc, providerCreateRequest(volume, snapshotID), createHooks{})
+}
+
+// pendingDocument is the document a snapshot of volume is recorded as before
+// the provider is called. AmbiguousCreate is set because that call is next.
+func pendingDocument(volume ebsmetadata.Volume, snapshotID, accountID, description string) ebsmetadata.Snapshot {
+	return ebsmetadata.Snapshot{
+		SnapshotID: snapshotID, VolumeID: volume.VolumeID,
 		VolumeSize: safecast.Uint64ToInt64(volume.CapacityGiB),
-		State:      string(created.State), Progress: "100%", StartTime: now,
-		Encrypted: volume.Encrypted,
-		OwnerID:   accountID, AvailabilityZone: volume.AvailabilityZone,
-		Tags:           awstags.Extract(input.TagSpecifications, "snapshot"),
-		ProviderHandle: created.Handle,
+		State:      snapshotStatePending, Progress: "0%", StartTime: time.Now(),
+		Description: description, Encrypted: volume.Encrypted,
+		OwnerID: accountID, AvailabilityZone: volume.AvailabilityZone,
+		AmbiguousCreate: true,
 	}
-	if snapshotCfg.State == "" {
-		snapshotCfg.State = "completed"
+}
+
+func providerCreateRequest(volume ebsmetadata.Volume, snapshotID string) ebsprovider.CreateSnapshotRequest {
+	return ebsprovider.CreateSnapshotRequest{
+		Versioned: ebsprovider.NewLatestVersioned(), SnapshotID: snapshotID, VolumeID: volume.VolumeID, VolumeHandle: volume.ProviderHandle,
 	}
-	if input.Description != nil {
-		snapshotCfg.Description = *input.Description
+}
+
+// ImageSnapshotHooks are the AMI document's writes around an image's snapshot.
+// Recorded runs after the snapshot's document and reference and before the
+// provider call; its error rolls the snapshot back.
+type ImageSnapshotHooks struct {
+	Recorded func(ctx context.Context) error
+	// Refused removes the AMI document when the provider refused the first call.
+	Refused func(ctx context.Context)
+	// Completed runs once the snapshot's document is written completed.
+	Completed func(ctx context.Context)
+}
+
+// ImageSnapshotCreator takes the snapshot behind a CreateImage through the
+// same document, reference, claim and provider path as CreateSnapshot.
+type ImageSnapshotCreator interface {
+	CreateImageSnapshot(ctx context.Context, volume ebsmetadata.Volume, snapshotID, accountID, description string, hooks ImageSnapshotHooks) (*ec2.Snapshot, error)
+}
+
+var _ ImageSnapshotCreator = (*SnapshotServiceImpl)(nil)
+
+// ImageClaims serialises the settling of a pending AMI with DeregisterImage:
+// each holds the image's claim, in the snapshot claim group keyed by image ID.
+type ImageClaims interface {
+	LockImage(ctx context.Context, imageID string) (unlock func(), err error)
+	SettlePendingAMI(ctx context.Context, imageID, state string)
+}
+
+var _ ImageClaims = (*SnapshotServiceImpl)(nil)
+
+// LockImage takes imageID's claim for a request, as writeUnderClaim takes a
+// snapshot's; one still held after the retries is a retryable IncorrectState.
+func (s *SnapshotServiceImpl) LockImage(ctx context.Context, imageID string) (func(), error) {
+	held, err := s.takeClaimWithRetry(ctx, "image", imageID)
+	if err != nil {
+		return nil, err
+	}
+	return func() { held.release(ctx) }, nil
+}
+
+// SettlePendingAMI moves imageID's AMI to state for a request, retrying a held
+// claim as LockImage does; an AMI it leaves pending is settled by the reaper.
+func (s *SnapshotServiceImpl) SettlePendingAMI(ctx context.Context, imageID, state string) {
+	s.settlePendingAMI(ctx, imageID, state, func(ctx context.Context, id string) (*heldClaim, error) {
+		return s.takeClaimWithRetry(ctx, "image", id)
+	})
+}
+
+// settlePendingAMI writes state to imageID's AMI under the claim take returns,
+// which DeregisterImage also takes, and only while a reread finds it pending.
+func (s *SnapshotServiceImpl) settlePendingAMI(ctx context.Context, imageID, state string, take func(context.Context, string) (*heldClaim, error)) bool {
+	return withClaim(ctx, imageID, take, func() bool {
+		current, err := s.metadata.GetAMI(ctx, imageID)
+		if err != nil || current.State != ebsmetadata.AMIStatePending {
+			return false
+		}
+		current.State = state
+		if err := s.metadata.PutAMI(ctx, current); err != nil {
+			slog.WarnContext(ctx, "pending AMI's state not written, left for the reaper", "imageId", imageID, "err", err)
+			return false
+		}
+		slog.InfoContext(ctx, "pending AMI settled", "imageId", imageID, "snapshotId", current.SnapshotID, "state", state)
+		return true
+	})
+}
+
+// CreateImageSnapshot snapshots an image's root volume as CreateSnapshot does,
+// running hooks to write the AMI document beside the snapshot's.
+func (s *SnapshotServiceImpl) CreateImageSnapshot(ctx context.Context, volume ebsmetadata.Volume, snapshotID, accountID, description string, hooks ImageSnapshotHooks) (*ec2.Snapshot, error) {
+	if s.provider == nil {
+		slog.ErrorContext(ctx, "no EBS provider configured", "op", "CreateImageSnapshot")
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	doc := pendingDocument(volume, snapshotID, accountID, description)
+	return s.createSnapshot(ctx, doc, providerCreateRequest(volume, snapshotID), createHooks(hooks))
+}
+
+// createHooks are the writes a CreateImage adds around createSnapshot's.
+type createHooks ImageSnapshotHooks
+
+// createStep is a point in createSnapshot after which a crash leaves its
+// writes so far, for a test to stop the request at.
+type createStep int
+
+const (
+	stepClaimed createStep = iota + 1
+	stepRecorded
+	stepReferenced
+	stepAnswered
+	stepSettled
+	stepRollbackMarked
+	stepRollbackUnreferenced
+)
+
+// createAttempts bounds createSnapshot's provider calls: the first, and one
+// repeat of a call that may have been accepted.
+const createAttempts = 2
+
+// errCreateCrashed is what createSnapshot returns when a test stops it at a step.
+var errCreateCrashed = errors.New("snapshot create stopped at a test crash point")
+
+// createSnapshot records doc pending and then asks the provider for it, so a
+// provider snapshot never exists without a document. Only a refusal of the
+// first call removes the document; anything else leaves it for the reaper.
+func (s *SnapshotServiceImpl) createSnapshot(ctx context.Context, doc ebsmetadata.Snapshot, req ebsprovider.CreateSnapshotRequest, hooks createHooks) (*ec2.Snapshot, error) {
+	snapshotID, volumeID, accountID := doc.SnapshotID, doc.VolumeID, doc.OwnerID
+
+	held, err := s.claims.lock(ctx, snapshotID)
+	if err != nil {
+		slog.ErrorContext(ctx, "CreateSnapshot could not take the snapshot claim", "snapshotId", snapshotID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	crashed := false
+	defer func() {
+		if !crashed {
+			held.release(ctx)
+		}
+	}()
+	crash := func(step createStep) bool {
+		crashed = s.crashAt != nil && s.crashAt(step)
+		return crashed
+	}
+	if crash(stepClaimed) {
+		return nil, errCreateCrashed
 	}
 
-	// Track the volume→snapshot dependency in KV before persisting to S3.
-	// This ensures we never have an untracked snapshot in S3.
+	// The document comes first because it is what the reaper discovers work
+	// from; a reference with no document would block DeleteVolume for good.
+	if err := s.metadata.PutSnapshot(ctx, doc); err != nil {
+		slog.ErrorContext(ctx, "CreateSnapshot failed to write the pending snapshot document", "snapshotId", snapshotID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if crash(stepRecorded) {
+		return nil, errCreateCrashed
+	}
 	if err := s.addSnapshotRef(ctx, volumeID, snapshotID); err != nil {
 		slog.ErrorContext(ctx, "CreateSnapshot failed to add snapshot ref to KV", "snapshotId", snapshotID, "volumeId", volumeID, "err", err)
-		_ = s.provider.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, Handle: created.Handle})
+		if s.rollBackCreate(ctx, doc, crash) {
+			return nil, errCreateCrashed
+		}
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-	if err := s.metadata.PutSnapshot(ctx, snapshotCfg); err != nil {
-		slog.ErrorContext(ctx, "CreateSnapshot failed to write config", "snapshotId", snapshotID, "err", err)
-		_ = s.removeSnapshotRefForCleanup(ctx, volumeID, snapshotID)
-		_ = s.provider.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, Handle: created.Handle})
-		return nil, errors.New(awserrors.ErrorServerInternal)
+	if crash(stepReferenced) {
+		return nil, errCreateCrashed
+	}
+	if hooks.Recorded != nil {
+		if err := hooks.Recorded(ctx); err != nil {
+			slog.ErrorContext(ctx, "CreateSnapshot failed to record the image beside its snapshot", "snapshotId", snapshotID, "err", err)
+			if s.rollBackCreate(ctx, doc, crash) {
+				return nil, errCreateCrashed
+			}
+			return nil, errors.New(awserrors.ErrorServerInternal)
+		}
 	}
 
-	slog.InfoContext(ctx, "CreateSnapshot completed", "snapshotId", snapshotID, "volumeId", volumeID)
+	// The document carries AmbiguousCreate from its first write, so a crash
+	// once a call is sent leaves the flag set. A call that may have been
+	// accepted is repeated once: the provider answers it with what it holds.
+	var created *ebsprovider.Snapshot
+	earlierUnanswered := false
+	for range createAttempts {
+		created, err = s.provider.CreateSnapshot(ctx, req)
+		if crash(stepAnswered) {
+			return nil, errCreateCrashed
+		}
+		if (err == nil && created != nil) || earlierUnanswered {
+			break
+		}
+		if definiteRefusal(err) {
+			// The request holds the claim, so no other call for this ID exists that
+			// a provider could still accept: the document can go.
+			slog.WarnContext(ctx, "CreateSnapshot refused by the provider", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
+			if hooks.Refused != nil {
+				hooks.Refused(ctx)
+			}
+			if s.rollBackCreate(ctx, doc, crash) {
+				return nil, errCreateCrashed
+			}
+			return nil, ProviderCreateError(volumeID, err)
+		}
+		slog.WarnContext(ctx, "CreateSnapshot provider call failed ambiguously, repeating it", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
+		earlierUnanswered = true
+	}
 
-	return snapshotConfigToEC2(snapshotCfg), nil
+	mutate, answered := repeatOutcome(created, err, earlierUnanswered)
+	switch {
+	case answered:
+		return s.answerCreate(ctx, doc, mutate, hooks, crash)
+	case mutate != nil:
+		slog.WarnContext(ctx, "CreateSnapshot repeat refused for good, marking the snapshot error", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
+		wctx, cancel := writeContext(ctx)
+		defer cancel()
+		if _, _, werr := s.updatePending(wctx, accountID, snapshotID, mutate); werr != nil {
+			slog.ErrorContext(ctx, "CreateSnapshot failed to mark the snapshot error", "snapshotId", snapshotID, "err", werr)
+		}
+		return nil, ProviderCreateError(volumeID, err)
+	case definiteRefusal(err):
+		// The first call may still be accepted once this refusal clears.
+		slog.WarnContext(ctx, "CreateSnapshot repeat refused, leaving the snapshot pending", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
+		return nil, ProviderCreateError(volumeID, err)
+	}
+	slog.ErrorContext(ctx, "CreateSnapshot provider call failed ambiguously twice, leaving the snapshot pending", "volumeId", volumeID, "snapshotId", snapshotID, "err", err)
+	return nil, errors.New(awserrors.ErrorServerInternal)
+}
+
+// answerCreate writes the provider's answer onto the pending document and
+// returns the snapshot as written. The answer fixed the contents, so a failed
+// write still reports success; the reaper settles the document.
+func (s *SnapshotServiceImpl) answerCreate(ctx context.Context, doc ebsmetadata.Snapshot, answer func(*ebsmetadata.Snapshot),
+	hooks createHooks, crash func(createStep) bool) (*ec2.Snapshot, error) {
+	wctx, cancel := writeContext(ctx)
+	defer cancel()
+	written, ok, err := s.updatePending(wctx, doc.OwnerID, doc.SnapshotID, answer)
+	completed := ok && written.State == ebsmetadata.SnapshotStateCompleted
+	switch {
+	case err != nil:
+		slog.ErrorContext(ctx, "CreateSnapshot failed to record the provider's answer", "snapshotId", doc.SnapshotID, "err", err)
+		written = doc
+	case !ok && written.SnapshotID == "":
+		// Deleted while the provider answered; report what the answer was.
+		written = doc
+		answer(&written)
+	}
+	if crash(stepSettled) {
+		return nil, errCreateCrashed
+	}
+	if completed && hooks.Completed != nil {
+		hooks.Completed(wctx)
+	}
+	slog.InfoContext(ctx, "CreateSnapshot completed", "snapshotId", doc.SnapshotID, "volumeId", doc.VolumeID, "state", written.State)
+	return snapshotConfigToEC2(written), nil
+}
+
+// rollBackCreate removes a snapshot the provider refused or never saw, in the
+// reaper's order: the document is marked deleted with no call outstanding, then
+// the reference and the document go. It reports a test crash.
+func (s *SnapshotServiceImpl) rollBackCreate(ctx context.Context, doc ebsmetadata.Snapshot, crash func(createStep) bool) bool {
+	ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), snapshotCleanupTimeout)
+	defer cancel()
+	// A marked document a failure below leaves is the reaper's to delete, and it
+	// never repeats the create of one, so the refusal the caller saw stands.
+	doc.DeleteRequested, doc.AmbiguousCreate = true, false
+	if err := s.metadata.PutSnapshot(ctx, doc); err != nil {
+		slog.ErrorContext(ctx, "CreateSnapshot failed to mark a refused snapshot's document deleted", "snapshotId", doc.SnapshotID, "err", err)
+	}
+	if crash(stepRollbackMarked) {
+		return true
+	}
+	if err := s.removeSnapshotRef(ctx, doc.VolumeID, doc.SnapshotID); err != nil {
+		slog.ErrorContext(ctx, "CreateSnapshot failed to remove a refused snapshot's ref, leaving its document for the reaper",
+			"snapshotId", doc.SnapshotID, "volumeId", doc.VolumeID, "err", err)
+		return false
+	}
+	if crash(stepRollbackUnreferenced) {
+		return true
+	}
+	if err := s.metadata.DeleteSnapshot(ctx, doc.OwnerID, doc.SnapshotID); err != nil && !objectstore.IsNoSuchKeyError(err) {
+		slog.ErrorContext(ctx, "CreateSnapshot failed to remove a refused snapshot's document, leaving it for the reaper", "snapshotId", doc.SnapshotID, "err", err)
+	}
+	return false
 }
 
 // describeSnapshotsValidFilters defines the set of filter names accepted by DescribeSnapshots.
@@ -316,6 +590,9 @@ func (s *SnapshotServiceImpl) describeSnapshots(ctx context.Context, input *ec2.
 
 	var snapshots []*ec2.Snapshot
 	for _, cfg := range configs {
+		if cfg.Hidden() {
+			continue
+		}
 		if len(parsedFilters) > 0 && !snapshotMatchesFilters(cfg, parsedFilters) {
 			continue
 		}
@@ -413,7 +690,7 @@ func (s *SnapshotServiceImpl) describeSnapshotsByIDs(ctx context.Context, input
 
 		// The read was keyed by the ID inside the caller's prefix, so the
 		// document is theirs and is the one named. Both stay as assertions.
-		if result.Document.SnapshotID != snapshotIDs[i] || result.Document.OwnerID != accountID {
+		if result.Document.SnapshotID != snapshotIDs[i] || result.Document.OwnerID != accountID || result.Document.Hidden() {
 			return nil, errors.New(awserrors.ErrorInvalidSnapshotNotFound)
 		}
 
@@ -649,6 +926,20 @@ func (s *SnapshotServiceImpl) DeleteSnapshot(ctx context.Context, input *ec2.Del
 		slog.ErrorContext(ctx, "no EBS provider configured", "op", "DeleteSnapshot")
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
+
+	// A provider may refuse to delete a snapshot still in progress, and an
+	// unanswered create may still be accepted, so either is marked for the reaper.
+	if deferDelete(cfg) {
+		marked, current, err := s.requestDelete(ctx, cfg)
+		if err != nil {
+			return nil, err
+		}
+		if marked {
+			slog.InfoContext(ctx, "DeleteSnapshot accepted; the snapshot is deleted once it settles", "snapshotId", snapshotID)
+			return &ec2.DeleteSnapshotOutput{}, nil
+		}
+		cfg = current
+	}
 	if err := s.provider.DeleteSnapshot(ctx, ebsprovider.DeleteSnapshotRequest{Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, Handle: cfg.ProviderHandle}); err != nil {
 		slog.ErrorContext(ctx, "DeleteSnapshot provider deletion failed", "snapshotId", snapshotID, "err", err)
 		return nil, providerDeleteError(snapshotID, err)
@@ -673,6 +964,104 @@ func (s *SnapshotServiceImpl) DeleteSnapshot(ctx context.Context, input *ec2.Del
 	return &ec2.DeleteSnapshotOutput{}, nil
 }
 
+// deferDelete reports a snapshot DeleteSnapshot marks rather than deletes.
+func deferDelete(doc ebsmetadata.Snapshot) bool {
+	return doc.State == snapshotStatePending || doc.AmbiguousCreate
+}
+
+// requestDelete marks doc DeleteRequested under its claim. A document that
+// settled with no call outstanding since it was read is returned unmarked,
+// for the caller to delete as any other.
+func (s *SnapshotServiceImpl) requestDelete(ctx context.Context, doc ebsmetadata.Snapshot) (bool, ebsmetadata.Snapshot, error) {
+	marked := false
+	var current ebsmetadata.Snapshot
+	err := s.writeUnderClaim(ctx, doc.OwnerID, doc.SnapshotID, func(d *ebsmetadata.Snapshot) (bool, error) {
+		if d.Hidden() {
+			return false, errors.New(awserrors.ErrorInvalidSnapshotNotFound)
+		}
+		current = *d
+		if !deferDelete(*d) {
+			return false, nil
+		}
+		d.DeleteRequested = true
+		marked = true
+		return true, nil
+	})
+	if err != nil {
+		return false, ebsmetadata.Snapshot{}, err
+	}
+	if current.SnapshotID == "" {
+		return false, ebsmetadata.Snapshot{}, errors.New(awserrors.ErrorInvalidSnapshotNotFound)
+	}
+	return marked, current, nil
+}
+
+// Contention on a snapshot's claim is bounded: three retries over two seconds.
+const (
+	claimRetries    = 3
+	claimRetryDelay = 2 * time.Second / claimRetries
+)
+
+// writeUnderClaim rereads snapshotID's document under its claim and writes it
+// when mutate asks to, so no write lands between another writer's reread and
+// its write. A document that is gone is not mutated, and mutate never sees it.
+func (s *SnapshotServiceImpl) writeUnderClaim(ctx context.Context, accountID, snapshotID string,
+	mutate func(*ebsmetadata.Snapshot) (bool, error)) error {
+	held, err := s.takeClaimWithRetry(ctx, "snapshot", snapshotID)
+	if err != nil {
+		return err
+	}
+	defer held.release(ctx)
+	var mutateErr error
+	_, _, err = s.rewrite(ctx, accountID, snapshotID, func(doc *ebsmetadata.Snapshot) bool {
+		var write bool
+		write, mutateErr = mutate(doc)
+		return write && mutateErr == nil
+	})
+	if mutateErr != nil {
+		return mutateErr
+	}
+	if err != nil {
+		return awserrors.Errorf(awserrors.ErrorServerInternal, "%v", err)
+	}
+	return nil
+}
+
+// takeClaimWithRetry takes the claim on resource id, of kind, as the
+// reaper does, retrying a holder that is live; one still held after the
+// retries is IncorrectState, which the caller retries.
+func (s *SnapshotServiceImpl) takeClaimWithRetry(ctx context.Context, kind, id string) (*heldClaim, error) {
+	delay := claimRetryDelay
+	if s.claimRetryDelay > 0 {
+		delay = s.claimRetryDelay
+	}
+	for attempt := 0; ; attempt++ {
+		held, err := s.claims.take(ctx, id)
+		if err == nil {
+			return held, nil
+		}
+		if !errors.Is(err, errClaimHeld) {
+			slog.ErrorContext(ctx, "Snapshot claim could not be taken", "kind", kind, "id", id, "err", err)
+			return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "take the claim of %s %s: %v", kind, id, err)
+		}
+		if s.claimRefused != nil {
+			s.claimRefused(id)
+		}
+		if attempt == claimRetries {
+			slog.InfoContext(ctx, "Snapshot claim still held after retries", "kind", kind, "id", id, "err", err)
+			return nil, awserrors.Errorf(awserrors.ErrorIncorrectState,
+				"The %s '%s' is being updated. Retry the request.", kind, id)
+		}
+		timer := time.NewTimer(delay)
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+			return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "take the claim of %s %s: %v", kind, id, ctx.Err())
+		case <-timer.C:
+		}
+	}
+}
+
 // CopySnapshot duplicates a snapshot under a new ID owned by the caller.
 // Served only through the provider: the copy needs its own snapshot metadata
 // on the backend, and on an encrypted volume that metadata is AEAD-sealed with
@@ -703,6 +1092,9 @@ func (s *SnapshotServiceImpl) CopySnapshot(ctx context.Context, input *ec2.CopyS
 		slog.WarnContext(ctx, "CopySnapshot: account does not own source snapshot", "snapshotId", sourceSnapshotID, "accountID", accountID, "ownerID", sourceCfg.OwnerID)
 		return nil, errors.New(awserrors.ErrorUnauthorizedOperation)
 	}
+	if !sourceCfg.Ready() {
+		return nil, sourceCfg.NotReadyError()
+	}
 
 	// Without a provider the only reachable path writes the control-plane
 	// config alone. That copy described and created volumes fine, then failed
@@ -783,6 +1175,25 @@ func (s *SnapshotServiceImpl) addSnapshotRef(ctx context.Context, volumeID, snap
 	return nil
 }
 
+// ensureSnapshotRef adds snapshotID to the volume's snapshot list unless it
+// is already there, restoring a reference a crashed request did not write.
+func (s *SnapshotServiceImpl) ensureSnapshotRef(ctx context.Context, volumeID, snapshotID string) error {
+	if s.snapRefs == nil {
+		return nil
+	}
+	err := s.snapRefs.Upsert(ctx, volumeID, func(snapshots *[]string) (bool, error) {
+		if slices.Contains(*snapshots, snapshotID) {
+			return false, nil
+		}
+		*snapshots = append(*snapshots, snapshotID)
+		return true, nil
+	})
+	if err != nil {
+		return fmt.Errorf("ensureSnapshotRef: failed to record %s against volume %s: %w", snapshotID, volumeID, err)
+	}
+	return nil
+}
+
 // removeSnapshotRefForCleanup removes a reference with a bounded context that
 // survives request cancellation after the snapshot operation has committed.
 func (s *SnapshotServiceImpl) removeSnapshotRefForCleanup(ctx context.Context, volumeID, snapshotID string) error {
@@ -914,10 +1325,31 @@ func (s *SnapshotServiceImpl) mirrorSnapshotTags(resources []*string, accountID
 			}
 			return err
 		}
-		if cfg.Tags == nil {
-			cfg.Tags = map[string]string{}
+		if cfg.Hidden() {
+			continue
 		}
-		mut(cfg.Tags)
+		apply := func(d *ebsmetadata.Snapshot) {
+			if d.Tags == nil {
+				d.Tags = map[string]string{}
+			}
+			mut(d.Tags)
+		}
+		// A document the reaper or a DeleteSnapshot may still write is written
+		// under its claim, so the tags cannot undo their write or it theirs.
+		if deferDelete(cfg) {
+			err := s.writeUnderClaim(context.Background(), accountID, *res, func(d *ebsmetadata.Snapshot) (bool, error) {
+				if d.Hidden() {
+					return false, nil
+				}
+				apply(d)
+				return true, nil
+			})
+			if err != nil {
+				return err
+			}
+			continue
+		}
+		apply(&cfg)
 		if err := s.metadata.PutSnapshot(context.Background(), cfg); err != nil {
 			return err
 		}

spinifex/domains/ec2/snapshot/service_impl_test.go

@@ -1246,10 +1246,9 @@ func (s *snapshotMetadataFailingObjectStore) PutObject(ctx context.Context, inpu
 	return s.ObjectStore.PutObject(ctx, input)
 }
 
-// TestCreateSnapshot_Provider_RollbackOnMetadataWriteFailure covers
-// CreateSnapshot's rollback path: when persisting the snapshot config fails
-// after the provider has already created the snapshot, CreateSnapshot must
-// delete the just-created provider snapshot rather than orphan it.
+// TestCreateSnapshot_Provider_RollbackOnMetadataWriteFailure: the pending
+// document is written before the provider is called, so a failed write leaves
+// no provider snapshot behind to orphan.
 func TestCreateSnapshot_Provider_RollbackOnMetadataWriteFailure(t *testing.T) {
 	store := &snapshotMetadataFailingObjectStore{ObjectStore: objectstore.NewMemoryObjectStore()}
 	cfg := &config.Config{Predastore: config.PredastoreConfig{Bucket: "test-bucket"}}
@@ -1273,13 +1272,10 @@ func TestCreateSnapshot_Provider_RollbackOnMetadataWriteFailure(t *testing.T) {
 
 	snapshotID := strings.TrimSuffix(path.Base(store.attemptedKey), ".json")
 	_, err = provider.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-rollback"})
-	require.NoError(t, err, "the source volume must be unaffected by the snapshot rollback")
+	require.NoError(t, err, "the source volume must be unaffected by the failed create")
 
-	// Recreating the same snapshot ID against a different source volume only
-	// succeeds if the original snapshot no longer exists (MemoryProvider
-	// returns already_exists for a conflicting source volume on a live
-	// snapshot ID), proving the rollback actually deleted it rather than
-	// merely leaving it orphaned but reachable.
+	// The ID is free on another source volume only if the provider never took
+	// it: MemoryProvider refuses a held ID on a different volume.
 	_, err = provider.CreateVolume(context.Background(), ebsprovider.CreateVolumeRequest{
 		Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-rollback-check", CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 * 1024 * 1024 * 1024},
 	})
@@ -1287,7 +1283,7 @@ func TestCreateSnapshot_Provider_RollbackOnMetadataWriteFailure(t *testing.T) {
 	_, err = provider.CreateSnapshot(context.Background(), ebsprovider.CreateSnapshotRequest{
 		Versioned: ebsprovider.NewVersioned(), SnapshotID: snapshotID, VolumeID: "vol-rollback-check",
 	})
-	require.NoError(t, err, "rollback must have deleted the just-created provider snapshot")
+	require.NoError(t, err, "a failed document write must leave no provider snapshot")
 }
 
 // providerSnapshotService builds a snapshot service backed by the memory
@@ -1547,17 +1543,17 @@ func TestCreateSnapshot_ProviderVolumeInUseIsIncorrectState(t *testing.T) {
 	assert.Contains(t, message, "Detach")
 }
 
-// A provider that cannot answer now, such as one refusing a volume another
-// node holds, is retry-later for the client; any other failure is internal.
+// A failure that is not a refusal may follow an accepted call, so after its
+// repeat fails the same way the request is internal and the snapshot is left
+// pending for the reaper rather than removed.
 func TestCreateSnapshot_ProviderFailureMapsToSpecificError(t *testing.T) {
 	cases := []struct {
-		name     string
-		err      error
-		wantCode string
+		name string
+		err  error
 	}{
-		{"provider unavailable", &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnavailable, Message: "held by node-2"}, awserrors.ErrorServiceUnavailable},
-		{"internal failure", &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeInternal, Message: "boom"}, awserrors.ErrorServerInternal},
-		{"transport failure", errors.New("connection reset"), awserrors.ErrorServerInternal},
+		{"provider unavailable", &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnavailable, Message: "held by node-2"}},
+		{"internal failure", &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeInternal, Message: "boom"}},
+		{"transport failure", errors.New("connection reset")},
 	}
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
@@ -1567,7 +1563,11 @@ func TestCreateSnapshot_ProviderFailureMapsToSpecificError(t *testing.T) {
 
 			_, err := svc.CreateSnapshot(context.Background(), &ec2.CreateSnapshotInput{VolumeId: aws.String("vol-create-fail")}, testAccountID)
 			require.Error(t, err)
-			assert.Equal(t, tc.wantCode, err.Error())
+			assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+			docs, err := svc.metadata.ListAllSnapshotsStrict(context.Background())
+			require.NoError(t, err)
+			require.Len(t, docs, 1)
+			assert.Equal(t, "pending", docs[0].State)
 		})
 	}
 }

spinifex/domains/ec2/volume/lifecycle_invariants_test.go

@@ -2,7 +2,6 @@ package volume
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 	"sync/atomic"
 	"testing"
@@ -10,7 +9,6 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/s3"
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
@@ -162,12 +160,9 @@ func TestVolumeTagMirror_WritesOnlyControlPlaneDocument(t *testing.T) {
 	svc.metadata = ebsmetadata.NewStore(store, "test-bucket")
 
 	var configRequests atomic.Int32
-	sub, err := svc.natsConn.Subscribe(viperblocklegacyv1.ConfigUpdateSubject, func(msg *nats.Msg) {
+	sub, err := svc.natsConn.Subscribe("ebs.config", func(msg *nats.Msg) {
 		configRequests.Add(1)
-		data, marshalErr := json.Marshal(viperblocklegacyv1.EBSConfigUpdateResponse{Volume: volumeID, Success: true})
-		if marshalErr == nil {
-			_ = msg.Respond(data)
-		}
+		_ = msg.Respond([]byte(`{"Success":true}`))
 	})
 	require.NoError(t, err)
 	require.NoError(t, svc.natsConn.Flush())

spinifex/domains/ec2/volume/provider_snapshot_source_test.go

@@ -51,6 +51,7 @@ func writeSnapshotMetadata(t *testing.T, store objectstore.ObjectStore, snapshot
 	t.Helper()
 	require.NoError(t, ebsmetadata.NewStore(store, "test-bucket").PutSnapshot(context.Background(), ebsmetadata.Snapshot{
 		SnapshotID: snapshotID, VolumeID: sourceVolumeID, VolumeSize: sizeGiB, OwnerID: ownerID,
+		State: ebsmetadata.SnapshotStateCompleted,
 	}))
 }
 

spinifex/domains/ec2/volume/service_impl.go

@@ -229,10 +229,15 @@ func (s *VolumeServiceImpl) createVolume(ctx context.Context, input *ec2.CreateV
 		// The ownership check is the key: a snapshot outside the caller's prefix
 		// is not found, so the endpoint never confirms another account's IDs.
 		snapMeta, err := s.metadata.GetSnapshot(ctx, accountID, snapshotID)
-		if err != nil {
+		if err != nil || snapMeta.Hidden() {
 			slog.ErrorContext(ctx, "CreateVolume: snapshot not found", "snapshotId", snapshotID, "err", err)
 			return nil, errors.New(awserrors.ErrorInvalidSnapshotNotFound)
 		}
+		// A pending snapshot's contents are not readable yet, and an errored
+		// one's never are.
+		if !snapMeta.Ready() {
+			return nil, snapMeta.NotReadyError()
+		}
 		sourceVolumeName = snapMeta.VolumeID
 		snapshotSizeGiB = snapMeta.VolumeSize
 	}
@@ -332,16 +337,7 @@ func (s *VolumeServiceImpl) createVolume(ctx context.Context, input *ec2.CreateV
 	return &ec2.Volume{VolumeId: aws.String(volumeID), Size: aws.Int64(size), VolumeType: aws.String(volumeType),
 		State: aws.String(string(initialState)), AvailabilityZone: input.AvailabilityZone, CreateTime: aws.Time(now),
 		Iops: aws.Int64(int64(iops)), Throughput: aws.Int64(int64(throughput)), Encrypted: aws.Bool(encrypted),
-		SnapshotId: snapshotIDOrNil(snapshotID), Tags: awstags.MapToEC2(tags)}, nil
-}
-
-// snapshotIDOrNil keeps SnapshotId absent for a volume that was not cloned,
-// rather than serialising an empty string.
-func snapshotIDOrNil(snapshotID string) *string {
-	if snapshotID == "" {
-		return nil
-	}
-	return aws.String(snapshotID)
+		SnapshotId: aws.String(snapshotID), MultiAttachEnabled: aws.Bool(false), Tags: awstags.MapToEC2(tags)}, nil
 }
 
 // requireProvider fails the call when no EBS provider is wired. Every write
@@ -929,6 +925,8 @@ func metadataVolumeToEC2(meta ebsmetadata.Volume) *ec2.Volume {
 		VolumeId: aws.String(meta.VolumeID), Size: aws.Int64(safecast.Uint64ToInt64(meta.CapacityGiB)),
 		State: aws.String(state), AvailabilityZone: aws.String(meta.AvailabilityZone), CreateTime: aws.Time(meta.CreatedAt),
 		VolumeType: aws.String(volumeType), Encrypted: aws.Bool(meta.Encrypted), Tags: awstags.MapToEC2(meta.Tags),
+		// AWS reports SnapshotId as "" for a volume not made from a snapshot.
+		SnapshotId: aws.String(meta.SnapshotID), MultiAttachEnabled: aws.Bool(false),
 	}
 	if meta.IOPS > 0 {
 		volume.Iops = aws.Int64(int64(meta.IOPS))
@@ -936,9 +934,6 @@ func metadataVolumeToEC2(meta ebsmetadata.Volume) *ec2.Volume {
 	if meta.Throughput > 0 {
 		volume.Throughput = aws.Int64(int64(meta.Throughput))
 	}
-	if meta.SnapshotID != "" {
-		volume.SnapshotId = aws.String(meta.SnapshotID)
-	}
 	if meta.AttachedInstance != "" {
 		// A recorded attachment on a volume that is not in-use is drift, so the
 		// attachment reports detached rather than claiming a live attach.

spinifex/domains/ec2/volume/service_impl_test.go

@@ -300,6 +300,27 @@ func TestCreateVolume_PassesValidation(t *testing.T) {
 	}
 }
 
+// AWS returns MultiAttachEnabled false and SnapshotId "" for a volume not
+// made from a snapshot, from both CreateVolume and DescribeVolumes.
+func TestCreateVolume_NoSnapshotShape(t *testing.T) {
+	svc := newTestVolumeService("ap-southeast-2a")
+	svc.snapshotKV = setupTestVolumeKV(t)
+	ctx := context.Background()
+
+	created, err := svc.CreateVolume(ctx, &ec2.CreateVolumeInput{Size: aws.Int64(1), AvailabilityZone: aws.String("ap-southeast-2a")}, testVolAccountID)
+	require.NoError(t, err)
+	out, err := svc.DescribeVolumes(ctx, &ec2.DescribeVolumesInput{VolumeIds: []*string{created.VolumeId}}, testVolAccountID)
+	require.NoError(t, err)
+	require.Len(t, out.Volumes, 1)
+
+	for name, vol := range map[string]*ec2.Volume{"CreateVolume": created, "DescribeVolumes": out.Volumes[0]} {
+		require.NotNil(t, vol.SnapshotId, name)
+		assert.Empty(t, *vol.SnapshotId, name)
+		require.NotNil(t, vol.MultiAttachEnabled, name)
+		assert.False(t, *vol.MultiAttachEnabled, name)
+	}
+}
+
 func TestCreateVolume_UsesInjectedProvider(t *testing.T) {
 	svc := newTestVolumeService("ap-southeast-2a")
 	svc.snapshotKV = setupTestVolumeKV(t)
@@ -743,6 +764,7 @@ func putTestSnapshotMetadata(t *testing.T, store *objectstore.MemoryObjectStore,
 		VolumeID:   "vol-source",
 		VolumeSize: sizeGiB,
 		OwnerID:    ownerID,
+		State:      ebsmetadata.SnapshotStateCompleted,
 	}))
 }
 

spinifex/domains/ec2/volume/snapshot_readiness_test.go

@@ -0,0 +1,48 @@
+package volume
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCreateVolume_FromSnapshot_RefusesASnapshotThatIsNotCompleted(t *testing.T) {
+	tests := []struct {
+		name     string
+		snapshot ebsmetadata.Snapshot
+		wantCode string
+	}{
+		{name: "pending", snapshot: ebsmetadata.Snapshot{State: "pending"}, wantCode: awserrors.ErrorIncorrectState},
+		{name: "error", snapshot: ebsmetadata.Snapshot{State: "error"}, wantCode: awserrors.ErrorIncorrectState},
+		{name: "deleted while pending", snapshot: ebsmetadata.Snapshot{State: "pending", DeleteRequested: true}, wantCode: awserrors.ErrorInvalidSnapshotNotFound},
+		{name: "deleted after completing", snapshot: ebsmetadata.Snapshot{State: "completed", DeleteRequested: true, AmbiguousCreate: true}, wantCode: awserrors.ErrorInvalidSnapshotNotFound},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			store := objectstore.NewMemoryObjectStore()
+			svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
+			metadata := ebsmetadata.NewStore(store, "test-bucket")
+			snap := tc.snapshot
+			snap.SnapshotID, snap.VolumeID, snap.VolumeSize, snap.OwnerID = "snap-gate000001", "vol-source", 8, testVolAccountID
+			require.NoError(t, metadata.PutSnapshot(context.Background(), snap))
+
+			_, err := svc.CreateVolume(context.Background(), &ec2.CreateVolumeInput{
+				AvailabilityZone: aws.String("ap-southeast-2a"),
+				SnapshotId:       aws.String(snap.SnapshotID),
+			}, testVolAccountID)
+			code, _, ok := awserrors.ResolveErrorDetail(err)
+			require.True(t, ok, "unregistered error: %v", err)
+			assert.Equal(t, tc.wantCode, code)
+			volumes, err := metadata.ListAllVolumes(context.Background())
+			require.NoError(t, err)
+			assert.Empty(t, volumes, "a refused CreateVolume writes no volume")
+		})
+	}
+}

spinifex/domains/ec2/vpc/eni.go

@@ -56,6 +56,7 @@ type ENIRecord struct {
 	// the instance and its IAM role under this account. Empty means same-account.
 	InstanceOwnerId  string            `json:"instance_owner_id,omitempty"`
 	DeviceIndex      int64             `json:"device_index"`
+	AttachTime       time.Time         `json:"attach_time,omitzero"`
 	PublicIpAddress  string            `json:"public_ip_address,omitempty"` // Auto-assigned or EIP
 	PublicIpPool     string            `json:"public_ip_pool,omitempty"`    // Pool name the public IP came from
 	SecurityGroupIds []string          `json:"security_group_ids,omitempty"`
@@ -277,7 +278,7 @@ func (s *VPCServiceImpl) deleteNetworkInterface(ctx context.Context, eniId, acco
 	// instance forces teardown. An ENI whose instance is gone is detached by
 	// terminate (historic PROP-LIFECYCLE-003 §2) and falls through as deletable.
 	if !force && eniIsLiveAttachment(&record) {
-		return nil, errors.New(awserrors.ErrorInvalidNetworkInterfaceInUse)
+		return nil, awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "Network interface '%s' is currently in use.", eniId)
 	}
 
 	s.releaseENISideEffects(ctx, eniId, accountID, &record)
@@ -376,6 +377,7 @@ func (s *VPCServiceImpl) DetachAndDeleteENI(ctx context.Context, accountID, eniI
 		record.AttachmentId = ""
 		record.InstanceId = ""
 		record.DeviceIndex = 0
+		record.AttachTime = time.Time{}
 
 		data, marshalErr := json.Marshal(record)
 		if marshalErr != nil {
@@ -427,6 +429,9 @@ func ValidateModifyNetworkInterfaceAttributeAttributes(input *ec2.ModifyNetworkI
 	if len(input.Groups) > 0 {
 		set = append(set, "securityGroups")
 	}
+	if input.Attachment != nil {
+		set = append(set, "attachment")
+	}
 	if len(set) == 0 {
 		return errors.New(awserrors.ErrorInvalidParameterValue)
 	}
@@ -437,6 +442,9 @@ func ValidateModifyNetworkInterfaceAttributeAttributes(input *ec2.ModifyNetworkI
 	if input.SourceDestCheck != nil && input.SourceDestCheck.Value != nil && !*input.SourceDestCheck.Value {
 		return errors.New(awserrors.ErrorUnsupported)
 	}
+	if input.Attachment != nil && (input.Attachment.AttachmentId == nil || *input.Attachment.AttachmentId == "") {
+		return errors.New(awserrors.ErrorMissingParameter)
+	}
 	return nil
 }
 
@@ -447,8 +455,9 @@ func multipleAttributesError(names []string) error {
 		"Fields for multiple attribute types specified: %s", strings.Join(names, ", "))
 }
 
-// ModifyNetworkInterfaceAttribute modifies one ENI attribute (security groups
-// or description). SourceDestCheck=true is accepted as a no-op.
+// ModifyNetworkInterfaceAttribute modifies one ENI attribute (security groups,
+// description or the attachment's DeleteOnTermination). SourceDestCheck=true
+// is accepted as a no-op.
 func (s *VPCServiceImpl) ModifyNetworkInterfaceAttribute(ctx context.Context, input *ec2.ModifyNetworkInterfaceAttributeInput, accountID string) (*ec2.ModifyNetworkInterfaceAttributeOutput, error) {
 	if input.NetworkInterfaceId == nil || *input.NetworkInterfaceId == "" {
 		return nil, errors.New(awserrors.ErrorMissingParameter)
@@ -490,6 +499,15 @@ func (s *VPCServiceImpl) ModifyNetworkInterfaceAttribute(ctx context.Context, in
 		record.Description = *input.Description.Value
 	}
 
+	if input.Attachment != nil {
+		if record.AttachmentId == "" || record.AttachmentId != *input.Attachment.AttachmentId {
+			return nil, errors.New(awserrors.ErrorInvalidAttachmentIDNotFound)
+		}
+		if input.Attachment.DeleteOnTermination != nil {
+			record.DeleteOnTermination = aws.Bool(*input.Attachment.DeleteOnTermination)
+		}
+	}
+
 	data, err := json.Marshal(record)
 	if err != nil {
 		slog.ErrorContext(ctx, "ModifyNetworkInterfaceAttribute: failed to marshal ENI record", "eniId", eniId, "accountID", accountID, "err", err)
@@ -695,17 +713,24 @@ func eniMatchesFilters(record *ENIRecord, filters map[string][]string) bool {
 // AttachENI marks an ENI as attached to an instance (internal use by RunInstances).
 // accountID scopes the lookup to the correct KV key.
 func (s *VPCServiceImpl) AttachENI(accountID, eniId, instanceId string, deviceIndex int64) (string, error) {
-	return s.attachENI(context.Background(), accountID, eniId, instanceId, deviceIndex, "")
+	return s.attachENI(context.Background(), accountID, eniId, instanceId, deviceIndex, "", nil)
+}
+
+// AttachNetworkInterface attaches an ENI on a customer's AttachNetworkInterface
+// call. As on AWS, the new attachment keeps the interface when the instance
+// terminates, written in the same CAS as the attach.
+func (s *VPCServiceImpl) AttachNetworkInterface(accountID, eniId, instanceId string, deviceIndex int64) (string, error) {
+	return s.attachENI(context.Background(), accountID, eniId, instanceId, deviceIndex, "", aws.Bool(false))
 }
 
 // AttachENIWithOwner attaches the ENI and records instanceOwnerID as the owner
 // of the attached instance in the same write, so no later write can lose the
 // stamp. An empty owner leaves the record's owner untouched.
 func (s *VPCServiceImpl) AttachENIWithOwner(accountID, eniId, instanceId string, deviceIndex int64, instanceOwnerID string) (string, error) {
-	return s.attachENI(context.Background(), accountID, eniId, instanceId, deviceIndex, instanceOwnerID)
+	return s.attachENI(context.Background(), accountID, eniId, instanceId, deviceIndex, instanceOwnerID, nil)
 }
 
-func (s *VPCServiceImpl) attachENI(ctx context.Context, accountID, eniId, instanceId string, deviceIndex int64, instanceOwnerID string) (string, error) {
+func (s *VPCServiceImpl) attachENI(ctx context.Context, accountID, eniId, instanceId string, deviceIndex int64, instanceOwnerID string, deleteOnTermination *bool) (string, error) {
 	attachmentId := awsidentifiers.GenerateResourceID("eni-attach")
 	err := s.casENI(ctx, kvstore.AccountKey(accountID, eniId), eniId, func(record *ENIRecord) error {
 		if record.Status == "in-use" {
@@ -715,13 +740,15 @@ func (s *VPCServiceImpl) attachENI(ctx context.Context, accountID, eniId, instan
 		record.AttachmentId = attachmentId
 		record.InstanceId = instanceId
 		record.DeviceIndex = deviceIndex
+		record.AttachTime = time.Now().UTC()
 		if instanceOwnerID != "" {
 			record.InstanceOwnerId = instanceOwnerID
 		}
-		// Default DeleteOnTermination to true on first attach, matching what
-		// DescribeNetworkInterfaces already advertises; a record carrying an
-		// explicit value from an earlier attach keeps it across re-attach.
-		if record.DeleteOnTermination == nil {
+		// An explicit value is stamped; otherwise default to true on first attach
+		// and keep an explicit value from an earlier attach across re-attach.
+		if deleteOnTermination != nil {
+			record.DeleteOnTermination = aws.Bool(*deleteOnTermination)
+		} else if record.DeleteOnTermination == nil {
 			record.DeleteOnTermination = aws.Bool(true)
 		}
 		return nil
@@ -752,6 +779,7 @@ func (s *VPCServiceImpl) DetachENI(ctx context.Context, accountID, eniId string)
 	record.AttachmentId = ""
 	record.InstanceId = ""
 	record.DeviceIndex = 0
+	record.AttachTime = time.Time{}
 
 	data, err := json.Marshal(record)
 	if err != nil {
@@ -1159,9 +1187,13 @@ func (s *VPCServiceImpl) eniRecordToEC2(record *ENIRecord, accountID string, gro
 			InstanceId:          aws.String(record.InstanceId),
 			InstanceOwnerId:     aws.String(instanceOwner),
 			DeviceIndex:         aws.Int64(record.DeviceIndex),
+			NetworkCardIndex:    aws.Int64(0),
 			Status:              aws.String("attached"),
 			DeleteOnTermination: aws.Bool(record.DeleteOnTermination == nil || *record.DeleteOnTermination),
 		}
+		if !record.AttachTime.IsZero() {
+			eni.Attachment.AttachTime = aws.Time(record.AttachTime)
+		}
 	}
 
 	eni.TagSet = tags.MapToEC2(record.Tags)

spinifex/domains/ec2/vpc/eni_test.go

@@ -411,7 +411,38 @@ func TestDeleteNetworkInterface_InUse(t *testing.T) {
 	_, err = svc.DeleteNetworkInterface(context.Background(), &ec2.DeleteNetworkInterfaceInput{
 		NetworkInterfaceId: aws.String(eniId),
 	}, testAccountID)
-	assert.ErrorContains(t, err, "InvalidNetworkInterface.InUse")
+	code, msg, _ := awserrors.ResolveErrorDetail(err)
+	assert.Equal(t, awserrors.ErrorInvalidParameterValue, code)
+	assert.Equal(t, "Network interface '"+eniId+"' is currently in use.", msg)
+}
+
+func TestAttachENI_RecordsAttachTimeAndNetworkCardIndex(t *testing.T) {
+	t.Parallel()
+	svc := setupTestVPCService(t)
+	vpcId := createTestVPC(t, svc, "10.0.0.0/16")
+	subnetId := createTestSubnet(t, svc, vpcId, "10.0.1.0/24")
+	eniId := createTestENI(t, svc, subnetId)
+
+	before := time.Now().UTC().Truncate(time.Second)
+	_, err := svc.AttachENI(testAccountID, eniId, "i-test123", 1)
+	require.NoError(t, err)
+
+	out, err := svc.DescribeNetworkInterfaces(context.Background(), &ec2.DescribeNetworkInterfacesInput{
+		NetworkInterfaceIds: []*string{aws.String(eniId)},
+	}, testAccountID)
+	require.NoError(t, err)
+	require.Len(t, out.NetworkInterfaces, 1)
+	att := out.NetworkInterfaces[0].Attachment
+	require.NotNil(t, att)
+	require.NotNil(t, att.AttachTime)
+	assert.False(t, att.AttachTime.Before(before))
+	require.NotNil(t, att.NetworkCardIndex)
+	assert.Equal(t, int64(0), *att.NetworkCardIndex)
+
+	require.NoError(t, svc.DetachENI(context.Background(), testAccountID, eniId))
+	rec, err := svc.GetENIRecord(testAccountID, eniId)
+	require.NoError(t, err)
+	assert.True(t, rec.AttachTime.IsZero())
 }
 
 func TestDeleteNetworkInterface_ReleasesIP(t *testing.T) {
@@ -1660,3 +1691,69 @@ func TestModifyNetworkInterfaceAttribute_MultipleAttributes(t *testing.T) {
 	assert.Contains(t, msg, "description")
 	assert.Contains(t, msg, "securityGroups")
 }
+
+// RunInstances' interface goes with the instance; one the customer attaches
+// stays, even when an earlier attachment had marked it delete-on-termination.
+func TestAttachENI_DeleteOnTerminationDefaults(t *testing.T) {
+	t.Parallel()
+	svc := setupTestVPCService(t)
+	vpcId := createTestVPC(t, svc, "10.0.0.0/16")
+	subnetId := createTestSubnet(t, svc, vpcId, "10.0.1.0/24")
+	launchENI := createTestENI(t, svc, subnetId)
+	customerENI := createTestENI(t, svc, subnetId)
+
+	_, err := svc.AttachENI(testAccountID, launchENI, "i-launch", 0)
+	require.NoError(t, err)
+	_, err = svc.AttachENI(testAccountID, customerENI, "i-launch", 1)
+	require.NoError(t, err)
+	require.NoError(t, svc.DetachENI(context.Background(), testAccountID, customerENI))
+	_, err = svc.AttachNetworkInterface(testAccountID, customerENI, "i-other", 1)
+	require.NoError(t, err)
+
+	desc, err := svc.DescribeNetworkInterfaces(context.Background(), &ec2.DescribeNetworkInterfacesInput{
+		NetworkInterfaceIds: []*string{aws.String(launchENI), aws.String(customerENI)},
+	}, testAccountID)
+	require.NoError(t, err)
+	got := map[string]bool{}
+	for _, ni := range desc.NetworkInterfaces {
+		require.NotNil(t, ni.Attachment)
+		got[*ni.NetworkInterfaceId] = aws.BoolValue(ni.Attachment.DeleteOnTermination)
+	}
+	assert.Equal(t, map[string]bool{launchENI: true, customerENI: false}, got)
+}
+
+func TestModifyNetworkInterfaceAttribute_AttachmentDeleteOnTermination(t *testing.T) {
+	t.Parallel()
+	svc := setupTestVPCService(t)
+	vpcId := createTestVPC(t, svc, "10.0.0.0/16")
+	subnetId := createTestSubnet(t, svc, vpcId, "10.0.1.0/24")
+	eniId := createTestENI(t, svc, subnetId)
+	attachId, err := svc.AttachNetworkInterface(testAccountID, eniId, "i-test123", 1)
+	require.NoError(t, err)
+
+	modify := func(attachmentID *string, dot bool) error {
+		_, err := svc.ModifyNetworkInterfaceAttribute(context.Background(), &ec2.ModifyNetworkInterfaceAttributeInput{
+			NetworkInterfaceId: aws.String(eniId),
+			Attachment: &ec2.NetworkInterfaceAttachmentChanges{
+				AttachmentId:        attachmentID,
+				DeleteOnTermination: aws.Bool(dot),
+			},
+		}, testAccountID)
+		return err
+	}
+
+	assert.True(t, awserrors.IsErrorCode(modify(nil, true), awserrors.ErrorMissingParameter))
+	assert.True(t, awserrors.IsErrorCode(modify(aws.String("eni-attach-0000000000000000"), true), awserrors.ErrorInvalidAttachmentIDNotFound))
+
+	require.NoError(t, modify(aws.String(attachId), true))
+	rec, err := svc.GetENIRecord(testAccountID, eniId)
+	require.NoError(t, err)
+	require.NotNil(t, rec.DeleteOnTermination)
+	assert.True(t, *rec.DeleteOnTermination)
+
+	require.NoError(t, modify(aws.String(attachId), false))
+	rec, err = svc.GetENIRecord(testAccountID, eniId)
+	require.NoError(t, err)
+	require.NotNil(t, rec.DeleteOnTermination)
+	assert.False(t, *rec.DeleteOnTermination)
+}