test(e2e): attach and detach a volume on a stopped instance, as AWS allows (#1508)

* test: pin guest poweroff handling and stopped-instance volume state

* test: gofmt the poweroff and stopped-volume tests

* test: pin start after detach, one-poll poweroff, reattach of released volumes and the delete refusal

* test: separate the embedded mounter from the recorder fields

* fix: end a guest poweroff as a stop and keep stopped instances' data volumes attached

A clean QEMU exit after the heartbeat saw the guest power off now stops the
instance instead of waiting for crash detection. Unmount no longer releases
data volumes. DetachVolume on a stopped instance edits the record, then
releases the volume. Starting an instance stopped on an older build drops data
volumes that are attached elsewhere or deleted and fails on a missing boot
volume.

* fix: keep the stopped-instance detach inside the instance service file and format start reconcile

* docs: describe guest poweroff, stopped-instance volumes and detach on a stopped instance

* fix: allow detaching the root volume of a stopped instance and refuse a start with no root volume as AWS does

* feat: attach volumes to a stopped instance and fix root-detach start fixtures

* docs: describe attach and root detach on a stopped instance

* test: Unmount seals only and leaves every attached volume in-use

Changed with the person's explicit approval, relayed by dev-design on 11-03.
EBS root and data volumes persist across stop on AWS, so stop and crash
unmount no longer release any attached volume.

* docs: note that the EFI variable store cannot be detached from a stopped instance

* test: unmount contract tests expect no volume state write on stop

* test: order imports in stopped-volume daemon tests

* refactor: drop the unused volume state updater from the volume mounter

contracts/viperblockd/legacy/v1/README.md

@@ -51,19 +51,6 @@ request/reply replies.
 removed its local state and export. These routes and JSON shapes are deployed
 legacy compatibility behaviour, not a generic EBS-provider interface.
 
-## Fence route
-
-`VolumeFencedSubject(node)` is `ebs.<node>.fenced`, or `ebs.fenced` with no
-node for a single-node daemon. The Viperblock adapter publishes
-`VolumeFencedEvent` on it after tearing down an export whose volume lease
-moved to another node; the local daemon subscribes to stop the guest that
-was using it. Node-addressed, like mount and unmount: the guest is local, and
-no other node's daemon has anything to stop.
-
-The event is sent as a request and repeated until the daemon replies with `VolumeFencedAck`, because the fence often runs while the node's NATS server is restarting and a single publish can reach no subscriber.
-The daemon replies only when the event carries a reply subject, so a plain publish is still accepted.
-Repeats are harmless, since fencing a guest that is already fenced changes nothing.
-
 ## Abandon route
 
 `VolumeAbandonSubject(node)` is `ebs.<node>.abandon`. The daemon uses

contracts/viperblockd/legacy/v1/abandon_test.go

@@ -9,9 +9,8 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-// TestVolumeAbandonSubject_BothForms pins the abandon route the same way the
-// fenced route is pinned: node-addressed, with an empty-node fallback for a
-// single-node daemon.
+// TestVolumeAbandonSubject_BothForms pins the abandon route: node-addressed,
+// with an empty-node fallback for a single-node daemon.
 func TestVolumeAbandonSubject_BothForms(t *testing.T) {
 	assert.Equal(t, "ebs.node-a.abandon", viperblocklegacyv1.VolumeAbandonSubject("node-a"))
 	assert.Equal(t, "ebs.abandon", viperblocklegacyv1.VolumeAbandonSubject(""))

contracts/viperblockd/legacy/v1/fence.go

@@ -1,29 +0,0 @@
-package viperblocklegacyv1
-
-// VolumeFencedSubject is where a fenced volume is announced so the daemon on
-// this node can stop the guest that was using it. Node-addressed: the guest is
-// local, and every other node's daemon has nothing to do about it.
-func VolumeFencedSubject(node string) string {
-	if node == "" {
-		return "ebs.fenced"
-	}
-	return "ebs." + node + ".fenced"
-}
-
-// VolumeFencedEvent announces that a node tore down a volume's export because
-// the lease moved. Winner is who holds it now, for the operator reading the
-// guest's stop reason.
-type VolumeFencedEvent struct {
-	Volume string `json:"volume"`
-	Node   string `json:"node"`
-	Winner string `json:"winner"`
-	Reason string `json:"reason"`
-}
-
-// VolumeFencedAck is the daemon's answer to a VolumeFencedEvent. Any answer
-// ends the sender's retries; Instance names the guest stopped, if one was, and
-// Error says why the daemon could not act on the event.
-type VolumeFencedAck struct {
-	Instance string `json:"instance,omitempty"`
-	Error    string `json:"error,omitempty"`
-}

contracts/viperblockd/legacy/v1/fence_test.go

@@ -1,54 +0,0 @@
-package viperblocklegacyv1_test
-
-import (
-	"encoding/json"
-	"testing"
-
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// TestVolumeFencedSubject_IsNodeAddressed pins the routing. A fenced guest is on
-// the node that lost the volume, so a subject without the node in it would reach
-// daemons with nothing to stop and, on a queue group, miss the one that has.
-func TestVolumeFencedSubject_IsNodeAddressed(t *testing.T) {
-	assert.Equal(t, "ebs.node-a.fenced", viperblocklegacyv1.VolumeFencedSubject("node-a"))
-	assert.Equal(t, "ebs.fenced", viperblocklegacyv1.VolumeFencedSubject(""),
-		"a single-node daemon has no node name, and still has to hear its own fences")
-}
-
-// TestVolumeFencedEvent_JSON pins the exact field spelling of the fenced
-// event, published by viperblockd and consumed by the daemon.
-func TestVolumeFencedEvent_JSON(t *testing.T) {
-	golden := `{"volume":"vol-1","node":"node-a","winner":"node-b","reason":"volume lease moved to node-b while this node had the volume open"}`
-	event := viperblocklegacyv1.VolumeFencedEvent{
-		Volume: "vol-1",
-		Node:   "node-a",
-		Winner: "node-b",
-		Reason: "volume lease moved to node-b while this node had the volume open",
-	}
-
-	marshaled, err := json.Marshal(event)
-	require.NoError(t, err)
-	assert.JSONEq(t, golden, string(marshaled))
-
-	var decoded viperblocklegacyv1.VolumeFencedEvent
-	require.NoError(t, json.Unmarshal([]byte(golden), &decoded))
-	assert.Equal(t, event, decoded)
-}
-
-// TestVolumeFencedAck_JSON pins the daemon's answer, which is what ends
-// viperblockd's retries of a fence notice.
-func TestVolumeFencedAck_JSON(t *testing.T) {
-	golden := `{"instance":"i-1","error":"bad event"}`
-	ack := viperblocklegacyv1.VolumeFencedAck{Instance: "i-1", Error: "bad event"}
-
-	marshaled, err := json.Marshal(ack)
-	require.NoError(t, err)
-	assert.JSONEq(t, golden, string(marshaled))
-
-	empty, err := json.Marshal(viperblocklegacyv1.VolumeFencedAck{})
-	require.NoError(t, err)
-	assert.JSONEq(t, `{}`, string(empty), "an ack for a volume with no local guest carries nothing")
-}

docs/compute/launching-instances/README.md

@@ -23,6 +23,7 @@ resources:
 - [Overview](#overview)
 - [Launch](#launch)
 - [Manage](#manage)
+- [Stopping and Data Volumes](#stopping-and-data-volumes)
 - [Modify Instance Attributes](#modify-instance-attributes)
 - [Instance Metadata Options](#instance-metadata-options)
 - [Spot Instances](#spot-instances)
@@ -73,6 +74,31 @@ aws ec2 terminate-instances --instance-ids $INSTANCE_ID
 aws ec2 reboot-instances --instance-ids $INSTANCE_ID
 ```
 
+## Stopping and Data Volumes
+
+A guest that powers itself off (`poweroff`, `shutdown -h`) goes `stopping` then `stopped`, the same as `stop-instances`.
+It is not restarted automatically.
+`InstanceInitiatedShutdownBehavior=terminate` is not honoured yet: a guest shutdown always stops the instance.
+
+Data volumes stay `in-use` and attached to a stopped instance, and `start-instances` mounts them again.
+While the instance is stopped, `attach-volume` of one of its data volumes to another instance returns `VolumeInUse`, and `delete-volume` is refused.
+
+`detach-volume` works on a stopped instance, with or without `--instance-id`.
+The volume becomes `available` and the next start does not attach it.
+The root volume of a stopped instance can be detached too, as on AWS.
+The internal EFI variable store is not a customer volume: it never appears in `describe-volumes` or `BlockDeviceMappings`, and detaching it is refused with `OperationNotPermitted`.
+The instance then cannot start: `start-instances` returns `InvalidParameterValue` with "Instance does not have a volume attached at root (/dev/sda1)".
+
+`attach-volume` works on a stopped instance too.
+The volume must be `available` and in the instance's availability zone, otherwise the call returns `VolumeInUse` or `InvalidVolume.ZoneMismatch`.
+The attachment is recorded on the stopped instance and the volume becomes `in-use` straight away.
+Attaching at the root device name (for example `/dev/sda1`) makes the volume the root again, and the next `start-instances` boots from it.
+
+An instance stopped by an earlier build had its data volumes released to `available` while its record still listed them.
+On start, a listed data volume that was deleted or is now attached to another instance is dropped with a warning in the daemon log, and the start continues.
+A listed volume that is still `available` and unattached is attached again.
+A missing root volume fails the start with the same `InvalidParameterValue` answer.
+
 ## Modify Instance Attributes
 
 Change instance type, user data, or termination protection. Instance type and user data require the instance to be **stopped** first.
@@ -135,6 +161,30 @@ aws ec2 get-console-output --instance-id $INSTANCE_ID \
   --query 'Output' --output text | base64 -d
 ```
 
+Like AWS, the call returns the most recent 64 KiB of the console.
+While an instance is shutting down after a terminate request, the call reads the renamed file described below.
+Once the instance has left the node's running set, which includes a stopped or terminated instance, the node no longer answers and the call returns `InvalidInstanceID.NotFound`.
+AWS still lists a terminated instance for a while after termination, so a caller may expect this call to work then; it does not here, a known gap.
+Whether the API should answer for a stopped instance is not settled here; only the file on the host is kept.
+
+### Where console logs live and how big they get
+
+The serial console of each instance is written on its host to `/run/spinifex/console-<instance-id>.log`.
+`/run` is memory-backed, so these files are cleared when the host reboots, including any left by older releases.
+The daemon keeps them bounded.
+A file over 4 MiB is cut back in place to its newest 1 MiB, and the guest keeps writing while it happens.
+The console is opened in append mode, so a reboot or auto-restart of the guest adds to the same file instead of starting over.
+A live `console-<instance-id>.log` is deleted only when it has not changed for an hour and no QEMU process on the host has it open, so a stopped instance keeps its console for an hour and a quiet running guest keeps it indefinitely.
+If the daemon cannot read the open files of every QEMU process on the host, it deletes nothing on that pass and logs a warning.
+This also clears the file of an instance that was stopped and then terminated, or that was recovered on another node, within about an hour of its last write.
+When an instance is terminated, including a failed first launch and the system instances behind load balancers, queues and EKS control planes, its file is renamed to `console-<instance-id>.log.terminated`.
+The daemon removes a `.terminated` file an hour after its last write.
+
+Known limits:
+
+- The log shipper follows only `console-*.log`, so it stops following a file when it is renamed on terminate, and the last few seconds of output that it had not yet read can be lost.
+- The instance password retrieval reads only the live file name, so it does not find a console renamed after terminate.
+
 ## Instance Types
 
 List instance types available on the current host. The catalog is generated from the host CPU (Intel, AMD, or ARM) and includes burstable (t-family), general purpose (m-family), compute optimised (c-family), and memory optimised (r-family) types.

scripts/images/eks-node/eks-node-role.sh

@@ -159,6 +159,7 @@ case "${ROLE}" in
         svc_start mulga-eks-state-report
         svc_start mulga-eks-addon-sync
         svc_start mulga-eks-node-gc
+        svc_optional mulga-eks-log-forward
         svc_optional eks-pod-identity-webhook
         ;;
     server-join)
@@ -178,12 +179,14 @@ case "${ROLE}" in
         svc_start konnectivity-server
         svc_start mulga-eks-state-report
         svc_start mulga-eks-node-gc
+        svc_optional mulga-eks-log-forward
         svc_optional eks-pod-identity-webhook
         ;;
     agent)
         log "configuring agent role"
         svc_enable k3s-agent
         svc_start k3s-agent
+        svc_optional mulga-eks-log-forward
         ;;
     *)
         die "SPINIFEX_K3S_ROLE missing or invalid: '${ROLE}'"

scripts/images/eks-node/eks-node-role_test.sh

@@ -146,6 +146,15 @@ for INIT in openrc systemd; do
     )"
     check "server(${INIT}): enables node-gc" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-node-gc)")"
     check "server(${INIT}): starts node-gc" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-node-gc)")"
+    check "server(${INIT}): enables log-forward" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "server(${INIT}): starts log-forward" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "server(${INIT}): starts log-forward before the webhook" "yes" "$(
+        awk -v f="$(start_cmd "${INIT}" mulga-eks-log-forward)" -v w="$(start_cmd "${INIT}" eks-pod-identity-webhook)" '
+            index($0, f) == 1 { fn = NR }
+            index($0, w) == 1 { wn = NR }
+            END { print (fn > 0 && wn > fn) ? "yes" : "no" }
+        ' "${CALLS}"
+    )"
     check "server(${INIT}): no k3s-agent" "no" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
     check "server(${INIT}): self-disables" "yes" "$(present "${CALLS}" "$(disable_cmd "${INIT}" eks-node-role)")"
     check "server(${INIT}): pod-identity-webhook is the last call" "$(start_cmd "${INIT}" eks-pod-identity-webhook)" "$(grep -v "$(disable_cmd "${INIT}" eks-node-role)" "${CALLS}" | tail -n 1)"
@@ -196,6 +205,15 @@ for INIT in openrc systemd; do
     # published the cluster-identical artifacts, and a join re-publish would
     # only race the bootstrap bus. This is deliberate, not an oversight.
     check "server-join(${INIT}): pod-identity-webhook is the last call" "$(start_cmd "${INIT}" eks-pod-identity-webhook)" "$(grep -v "$(disable_cmd "${INIT}" eks-node-role)" "${CALLS}" | tail -n 1)"
+    check "server-join(${INIT}): enables log-forward" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "server-join(${INIT}): starts log-forward" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "server-join(${INIT}): starts log-forward before the webhook" "yes" "$(
+        awk -v f="$(start_cmd "${INIT}" mulga-eks-log-forward)" -v w="$(start_cmd "${INIT}" eks-pod-identity-webhook)" '
+            index($0, f) == 1 { fn = NR }
+            index($0, w) == 1 { wn = NR }
+            END { print (fn > 0 && wn > fn) ? "yes" : "no" }
+        ' "${CALLS}"
+    )"
     check "server-join(${INIT}): no k3s-first-boot" "yes" "$(absent_all "${CALLS}" "k3s-first-boot")"
     check "server-join(${INIT}): no k3s-agent" "yes" "$(absent_all "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
     check "server-join(${INIT}): self-disables" "yes" "$(present "${CALLS}" "$(disable_cmd "${INIT}" eks-node-role)")"
@@ -207,8 +225,39 @@ for INIT in openrc systemd; do
     check "agent(${INIT}): exit 0" 0 "${rc}"
     check "agent(${INIT}): role file" "agent" "$(cat "${ROLE_FILE}")"
     check "agent(${INIT}): enables k3s-agent" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
+    check "agent(${INIT}): enables log-forward" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "agent(${INIT}): starts log-forward" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" mulga-eks-log-forward)")"
+    check "agent(${INIT}): log-forward is enabled after k3s-agent" "yes" "$(
+        awk -v k="$(enable_cmd "${INIT}" k3s-agent)" -v f="$(enable_cmd "${INIT}" mulga-eks-log-forward)" '
+            index($0, k) == 1 { kn = NR }
+            index($0, f) == 1 { fn = NR }
+            END { print (kn > 0 && fn > kn) ? "yes" : "no" }
+        ' "${CALLS}"
+    )"
+
     check "agent(${INIT}): no server services" "yes" "$(absent_all "${CALLS}" "eks-token-webhook" "eks-pod-identity-webhook" "k3s-first-boot" "mulga-eks-node-gc" "$(enable_cmd "${INIT}" k3s)")"
 
+    # A broken log forwarder must never stop a node joining.
+    for FAILING in enable start; do
+        run_case "${INIT}" agent_logfwd_${FAILING}_fail
+        printf 'SPINIFEX_K3S_ROLE=agent\n' > "${EKS_NODE_ENVFILE}"
+        if [ "${FAILING}" = enable ]; then
+            STUB_FAIL="$(enable_cmd "${INIT}" mulga-eks-log-forward)"
+            export STUB_FAIL
+            STUB_FAIL="${STUB_FAIL#rc-update add }"; STUB_FAIL="${STUB_FAIL#systemctl enable }"
+        else
+            STUB_FAIL="$(start_cmd "${INIT}" mulga-eks-log-forward)"
+            export STUB_FAIL
+            STUB_FAIL="${STUB_FAIL#rc-service }"; STUB_FAIL="${STUB_FAIL#systemctl start --no-block }"
+        fi
+        invoke
+        unset STUB_FAIL
+        check "agent(${INIT}): log-forward ${FAILING} failure still exits 0" 0 "${rc}"
+        check "agent(${INIT}): log-forward ${FAILING} failure still enables k3s-agent" "yes" "$(present "${CALLS}" "$(enable_cmd "${INIT}" k3s-agent)")"
+        check "agent(${INIT}): log-forward ${FAILING} failure still starts k3s-agent" "yes" "$(present "${CALLS}" "$(start_cmd "${INIT}" k3s-agent)")"
+        check "agent(${INIT}): log-forward ${FAILING} failure still self-disables" "yes" "$(present "${CALLS}" "$(disable_cmd "${INIT}" eks-node-role)")"
+    done
+
     # --- agent (inferred from agent.env, no explicit role) ---
     run_case "${INIT}" agent_inferred
     printf 'K3S_URL=https://nlb:443\nK3S_TOKEN=abc\n' > "${EKS_NODE_AGENT_ENVFILE}"

scripts/images/eks-node/manifest.conf

@@ -45,6 +45,8 @@ scripts/images/eks-node/mulga-eks-addon-sync.initd:/etc/init.d/mulga-eks-addon-s
 scripts/images/eks-node/mulga-eks-addon-sync.sh:/usr/local/sbin/mulga-eks-addon-sync \
 scripts/images/eks-node/mulga-eks-node-gc.initd:/etc/init.d/mulga-eks-node-gc \
 scripts/images/eks-node/mulga-eks-node-gc.sh:/usr/local/sbin/mulga-eks-node-gc \
+scripts/images/eks-node/mulga-eks-log-forward.initd:/etc/init.d/mulga-eks-log-forward \
+scripts/images/eks-node/mulga-eks-log-forward.sh:/usr/local/sbin/mulga-eks-log-forward \
 scripts/images/eks-node/konnectivity-server.initd:/etc/init.d/konnectivity-server \
 scripts/images/eks-node/konnectivity-server-prestart.sh:/usr/local/sbin/konnectivity-server-prestart \
 scripts/images/eks-node/konnectivity/konnectivity-agent.yaml:/usr/share/spinifex-eks/konnectivity/konnectivity-agent.yaml \

scripts/images/eks-node/mulga-eks-log-forward.initd

@@ -0,0 +1,30 @@
+#!/sbin/openrc-run
+# mulga-eks-log-forward — copies k3s warnings and errors to the serial console
+# so they reach the Elasticsearch sink. Best effort: nothing depends on it and
+# a failure here never affects the node.
+
+description="Forward k3s warnings and errors to the serial console"
+command="/bin/sh"
+command_background="yes"
+supervisor="supervise-daemon"
+respawn_delay=5
+pidfile="/run/mulga-eks-log-forward.pid"
+output_log="/dev/null"
+error_log="/dev/null"
+
+depend() {
+    after k3s k3s-agent
+}
+
+start_pre() {
+    role=$(cat /etc/spinifex-eks/role 2>/dev/null || true)
+    if [ "${role}" = "agent" ]; then
+        log=/var/log/k3s-agent.log
+        FORWARD_PREFIX="[k3s-agent]"
+    else
+        log=/var/log/k3s.log
+        FORWARD_PREFIX="[k3s]"
+    fi
+    export FORWARD_PREFIX
+    command_args="-c 'tail -n +1 -F ${log} 2>/dev/null | /usr/local/sbin/mulga-eks-log-forward'"
+}

scripts/images/eks-node/mulga-eks-log-forward.sh

@@ -0,0 +1,90 @@
+#!/bin/sh
+# mulga-eks-log-forward — copy the warnings and errors of a k3s log to the
+# serial console, where the host picks them up for the Elasticsearch sink.
+#
+# Reads the tailed log on stdin. Lines that matter are redacted first, then
+# cut to FORWARD_MAX_BYTES, then rate limited: at most FORWARD_RATE_PER_MIN a
+# minute reach the console and the rest are reported as one "N lines dropped"
+# line. Every console write has a timeout, so a stuck console loses lines
+# instead of stalling the tail or k3s.
+#
+# Env: FORWARD_CONSOLE (default /dev/console), FORWARD_PREFIX, FORWARD_MAX_BYTES
+# (512), FORWARD_RATE_PER_MIN (60), FORWARD_WRITE_TIMEOUT seconds (2) and
+# FORWARD_CLOCK_CMD (default: seconds of uptime, read without forking).
+
+CONSOLE=${FORWARD_CONSOLE:-/dev/console}
+PREFIX=${FORWARD_PREFIX:-[k3s]}
+MAX_BYTES=${FORWARD_MAX_BYTES:-512}
+RATE=${FORWARD_RATE_PER_MIN:-60}
+WRITE_TIMEOUT=${FORWARD_WRITE_TIMEOUT:-2}
+CLOCK_CMD=${FORWARD_CLOCK_CMD:-}
+
+now() {
+    if [ -n "${CLOCK_CMD}" ]; then
+        sh -c "${CLOCK_CMD}"
+    else
+        read -r up _ < /proc/uptime && echo "${up%%.*}"
+    fi
+}
+
+# emit TEXT: one console line, abandoned after WRITE_TIMEOUT. Returns non-zero
+# when the console did not take it.
+emit() {
+    timeout "${WRITE_TIMEOUT}" sh -c 'printf "%s\n" "$1" >> "$2"' sh "$1" "${CONSOLE}" 2>/dev/null
+}
+
+# Filter, redact and truncate in one awk pass so lines that do not matter cost
+# no fork. Redaction runs before the cut so a secret straddling the limit
+# cannot leave a readable prefix behind.
+select_lines() {
+    LC_ALL=C awk -v max="${MAX_BYTES}" '
+    {
+        line = $0
+        gsub(/\r/, "", line)
+        keep = 0
+        if (line ~ /level=(warning|error)/) keep = 1
+        else if (line ~ /^[WE][0-9][0-9][0-9][0-9] /) keep = 1
+        else if (line ~ /NodeNotReady|PLEG|[Ll]ease|[Nn]ode status/) keep = 1
+        if (!keep) next
+
+        gsub(/[Bb]earer +[A-Za-z0-9._~+\/=-]+/, "Bearer [REDACTED]", line)
+        gsub(/K10[0-9a-f]+(::[A-Za-z0-9-]+:[^ "\047]+)?/, "[REDACTED]", line)
+        gsub(/eyJ[A-Za-z0-9_.-]+/, "[REDACTED]", line)
+        gsub(/(AKIA|ASIA)[A-Z0-9]+/, "[REDACTED]", line)
+        gsub(/X-Amz-Signature=[^& "\047]*/, "X-Amz-Signature=[REDACTED]", line)
+        gsub(/X-Amz-Credential=[^& "\047]*/, "X-Amz-Credential=[REDACTED]", line)
+        gsub(/X-Amz-Security-Token=[^& "\047]*/, "X-Amz-Security-Token=[REDACTED]", line)
+        gsub(/[^[:print:]\t]/, " ", line)
+        print substr(line, 1, max)
+    }'
+}
+
+win_start=0
+sent=0
+dropped=0
+
+flush_dropped() {
+    [ "${dropped}" -gt 0 ] || return 0
+    emit "${PREFIX} ${dropped} lines dropped"
+    dropped=0
+}
+
+select_lines | {
+    while IFS= read -r line; do
+        t=$(now) || t=0
+        if [ "${win_start}" -eq 0 ] || [ $((t - win_start)) -ge 60 ]; then
+            flush_dropped
+            win_start=${t}
+            sent=0
+            [ "${win_start}" -eq 0 ] && win_start=1
+        fi
+        if [ "${sent}" -ge "${RATE}" ]; then
+            dropped=$((dropped + 1))
+            continue
+        fi
+        sent=$((sent + 1))
+        emit "${PREFIX} ${line}" || dropped=$((dropped + 1))
+    done
+    flush_dropped
+}
+exit 0

scripts/images/eks-node/mulga-eks-log-forward_test.sh

@@ -0,0 +1,190 @@
+#!/bin/sh
+# Self-contained POSIX test for mulga-eks-log-forward.sh: the line filter,
+# redaction, truncation, rate cap and blocked-console behaviour. The script
+# reads the tailed log on stdin and writes to FORWARD_CONSOLE, which the test
+# points at a temp file (or a reader-less FIFO), so no root or tty is needed.
+#
+# Run: sh scripts/images/eks-node/mulga-eks-log-forward_test.sh
+set -eu
+
+SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd)
+SCRIPT="${SCRIPT_DIR}/mulga-eks-log-forward.sh"
+WORK=$(mktemp -d)
+trap 'rm -rf "${WORK}"' EXIT
+
+PASS=0
+FAIL=0
+check() { # check <desc> <expected> <actual>
+    if [ "$2" = "$3" ]; then
+        PASS=$((PASS + 1))
+    else
+        FAIL=$((FAIL + 1))
+        echo "FAIL: $1"
+        echo "  expected: [$2]"
+        echo "  actual:   [$3]"
+    fi
+}
+
+# forward <name>: feeds stdin to the script with a per-case console file and a
+# fixed clock, leaving the forwarded text in ${OUT}.
+forward() {
+    CASE="${WORK}/$1"
+    mkdir -p "${CASE}"
+    OUT="${CASE}/console"
+    : > "${OUT}"
+    FORWARD_CONSOLE="${OUT}" FORWARD_PREFIX="[k3s-agent]" \
+        FORWARD_CLOCK_CMD="echo 1000" sh "${SCRIPT}" 2>/dev/null || true
+}
+forwarded() { grep -c . "${OUT}" || true; }
+
+# --- filter: lines that must pass ---
+forward filter_pass <<'EOF'
+time="2026-11-03T03:00:00Z" level=warning msg="slow response from apiserver"
+time="2026-11-03T03:00:01Z" level=error msg="failed to renew"
+W1103 03:00:02.123456    1234 reflector.go:561] watch of *v1.Pod ended
+E1103 03:00:03.123456    1234 kubelet.go:2400] node not found
+I1103 03:00:04.123456    1234 kubelet_node_status.go:73] Node status update failed
+I1103 03:00:05.123456    1234 controller.go:193] Failed to update lease
+I1103 03:00:06.123456    1234 kubelet.go:2200] PLEG is not healthy
+time="2026-11-03T03:00:07Z" level=info msg="Event: NodeNotReady kubelet"
+EOF
+check "filter: all eight interesting lines pass" "8" "$(forwarded)"
+check "filter: lines carry the prefix" "8" "$(grep -c '^\[k3s-agent\] ' "${OUT}" || true)"
+
+# --- filter: lines that must be dropped ---
+forward filter_drop <<'EOF'
+time="2026-11-03T03:00:00Z" level=info msg="Starting k3s agent"
+time="2026-11-03T03:00:01Z" level=debug msg="tick"
+I1103 03:00:02.123456    1234 server.go:100] Version: v1.32.5
+I1103 03:00:03.123456    1234 kubelet.go:1000] SyncLoop (UPDATE, "api")
+EOF
+check "filter: info and debug lines are dropped" "0" "$(forwarded)"
+
+# --- redaction: one case per secret shape ---
+redact() { # redact <name> <line> <secret-that-must-not-appear>
+    forward "redact_$1" <<EOF
+time="2026-11-03T03:00:00Z" level=error msg="$2"
+EOF
+    check "redact(${1}): line still forwarded" "1" "$(forwarded)"
+    if grep -qF "$3" "${OUT}"; then
+        check "redact(${1}): secret removed" "absent" "present"
+    else
+        check "redact(${1}): secret removed" "absent" "absent"
+    fi
+    check "redact(${1}): marker present" "1" "$(grep -c 'REDACTED' "${OUT}" || true)"
+}
+redact bearer "request failed Authorization: Bearer abc123SECRETvalue456" "abc123SECRETvalue456"
+redact k3s_token "join failed token K1009f3c2d4e5a6b7c8d9e0f1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2::server:s3cr3tPassw0rd" "s3cr3tPassw0rd"
+redact jwt "bad token eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.c2lnbmF0dXJlLXNlY3JldA" "eyJhbGciOiJIUzI1NiJ9"
+redact akia "credentials AKIAIOSFODNN7EXAMPLE rejected" "AKIAIOSFODNN7EXAMPLE"
+redact asia "credentials ASIAIOSFODNN7EXAMPLE rejected" "ASIAIOSFODNN7EXAMPLE"
+redact amz_signature "GET /obj?X-Amz-Signature=0123456789abcdef0123456789abcdef&X-Amz-Expires=60 failed" "0123456789abcdef0123456789abcdef"
+redact amz_credential "GET /obj?X-Amz-Credential=ZZZsecretscope%2F20261103%2Fap%2Fs3%2Faws4_request failed" "ZZZsecretscope"
+
+# --- truncation: text over 512 bytes is cut ---
+LONG=$(awk 'BEGIN { for (i = 0; i < 2000; i++) printf "a"; }')
+forward truncate <<EOF
+time="2026-11-03T03:00:00Z" level=error msg="${LONG}"
+EOF
+check "truncate: still forwarded" "1" "$(forwarded)"
+BODY=$(sed 's/^\[k3s-agent\] //' "${OUT}" | head -n 1)
+check "truncate: body is exactly 512 bytes" "512" "$(printf '%s' "${BODY}" | wc -c | tr -d ' ')"
+check "truncate: keeps the start of the line" "time=\"2026-11-03T03:00:00Z\"" "$(printf '%s' "${BODY}" | cut -c1-27)"
+
+# --- redaction runs before the cut: a secret over the limit leaves no prefix ---
+# The pad puts the 512-byte cut right after the bare marker (AKIA, eyJ, K10).
+# A cut fragment that long matches no redaction pattern, so redacting after the
+# cut would leave it in the output.
+HEAD='time="2026-11-03T03:00:00Z" level=error msg="'
+straddle() { # straddle <name> <secret> <bytes of the secret kept by the cut>
+    PAD=$(awk -v n=$((512 - ${#HEAD} - 1 - $3)) 'BEGIN { for (i = 0; i < n; i++) printf "x"; }')
+    forward "straddle_$1" <<EOF
+${HEAD}${PAD} $2 tail"
+EOF
+    check "straddle(${1}): line forwarded" "1" "$(forwarded)"
+    check "straddle(${1}): cut keeps the marker length" "$3" "$(sed 's/^\[k3s-agent\] //' "${OUT}" | head -n 1 | awk -v p="${PAD} " '{ print length($0) - index($0, p) - length(p) + 1 }')"
+    if grep -qF "$(printf '%s' "$2" | cut -c1-"$3")" "${OUT}"; then
+        check "straddle(${1}): no fragment survives the cut" "absent" "present"
+    else
+        check "straddle(${1}): no fragment survives the cut" "absent" "absent"
+    fi
+}
+straddle jwt "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0In0.c2ln" 3
+straddle akia "AKIAIOSFODNN7EXAMPLE" 4
+straddle k3s_token "K1009f3c2d4e5a6b7c8d9e0f1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2::server:s3cr3tPassw0rd" 3
+
+# --- rate cap: 60 a minute, then one summary of what was dropped ---
+gen() { # gen <count>
+    awk -v n="$1" 'BEGIN { for (i = 0; i < n; i++) printf "time=\"t\" level=error msg=\"loop %d\"\n", i }'
+}
+gen 100 > "${WORK}/burst"
+forward rate_cap < "${WORK}/burst"
+check "rate cap: 60 lines forwarded plus one summary" "61" "$(forwarded)"
+check "rate cap: one dropped summary" "1" "$(grep -c '40 lines dropped' "${OUT}" || true)"
+check "rate cap: first line kept" "1" "$(grep -c 'loop 0"' "${OUT}" || true)"
+check "rate cap: line past the cap is not forwarded" "0" "$(grep -c 'loop 60"' "${OUT}" || true)"
+
+# --- rate cap: a new minute starts a new window and reports the old drops ---
+# The clock command advances 60 s every 70 calls, so lines 0-69 share a window
+# (10 dropped) and lines 70-74 land in the next one.
+CASE="${WORK}/rate_window"
+mkdir -p "${CASE}"
+OUT="${CASE}/console"
+: > "${OUT}"
+echo 0 > "${CASE}/calls"
+cat > "${CASE}/clock" <<EOF
+#!/bin/sh
+n=\$(cat "${CASE}/calls")
+echo \$((n + 1)) > "${CASE}/calls"
+echo \$((1000 + n / 70 * 60))
+EOF
+chmod +x "${CASE}/clock"
+gen 75 | FORWARD_CONSOLE="${OUT}" FORWARD_PREFIX="[k3s-agent]" FORWARD_CLOCK_CMD="${CASE}/clock" sh "${SCRIPT}" 2>/dev/null || true
+check "window: 60 + 5 forwarded plus one summary" "66" "$(forwarded)"
+check "window: summary names the 10 dropped" "1" "$(grep -c '10 lines dropped' "${OUT}" || true)"
+check "window: new window forwards its lines" "1" "$(grep -c 'loop 74"' "${OUT}" || true)"
+
+# --- blocked console: lines are dropped, the forwarder neither stalls nor fails ---
+CASE="${WORK}/blocked"
+mkdir -p "${CASE}"
+mkfifo "${CASE}/console"
+gen 3 > "${CASE}/in"
+(
+    FORWARD_CONSOLE="${CASE}/console" FORWARD_PREFIX="[k3s-agent]" FORWARD_CLOCK_CMD="echo 1000" \
+        FORWARD_WRITE_TIMEOUT=1 sh "${SCRIPT}" < "${CASE}/in"
+) &
+PID=$!
+i=0
+while kill -0 "${PID}" 2>/dev/null && [ "${i}" -lt 20 ]; do
+    sleep 1
+    i=$((i + 1))
+done
+if kill -0 "${PID}" 2>/dev/null; then
+    kill "${PID}" 2>/dev/null || true
+    check "blocked console: forwarder finished" "finished" "stalled"
+else
+    check "blocked console: forwarder finished" "finished" "finished"
+fi
+rc=0
+wait "${PID}" || rc=$?
+check "blocked console: exit 0" "0" "${rc}"
+
+# --- the service is wired in, and nothing depends on it ---
+INITD="${SCRIPT_DIR}/mulga-eks-log-forward.initd"
+check "initd ships" "yes" "$([ -f "${INITD}" ] && echo yes || echo no)"
+check "manifest installs the initd" "1" "$(grep -c 'mulga-eks-log-forward.initd:/etc/init.d/mulga-eks-log-forward' "${SCRIPT_DIR}/manifest.conf" || true)"
+check "manifest installs the script" "1" "$(grep -c 'mulga-eks-log-forward.sh:/usr/local/sbin/mulga-eks-log-forward' "${SCRIPT_DIR}/manifest.conf" || true)"
+DEPS=0
+for f in "${SCRIPT_DIR}"/*.initd; do
+    [ "${f}" = "${INITD}" ] && continue
+    if grep -E '^[[:space:]]*(need|use|want)[[:space:]].*mulga-eks-log-forward' "${f}" >/dev/null 2>&1; then
+        DEPS=$((DEPS + 1))
+    fi
+done
+check "nothing depends on the forwarder" "0" "${DEPS}"
+if [ -f "${INITD}" ]; then
+    check "initd does not need anything it could block boot on" "0" "$(grep -cE '^[[:space:]]*need[[:space:]]' "${INITD}" || true)"
+fi
+
+echo "mulga-eks-log-forward_test: ${PASS} passed, ${FAIL} failed"
+[ "${FAIL}" -eq 0 ]

scripts/images/eks-node/setup.sh

@@ -45,10 +45,10 @@ ln -sf /usr/local/bin/k3s /usr/local/bin/ctr
 chmod 0755 /etc/init.d/eks-node-role /etc/init.d/k3s /etc/init.d/k3s-agent \
     /etc/init.d/eks-token-webhook /etc/init.d/k3s-first-boot /etc/init.d/mulga-eks-state-report \
     /etc/init.d/mulga-eks-addon-sync /etc/init.d/mulga-eks-node-gc /etc/init.d/konnectivity-server \
-    /etc/init.d/eks-pod-identity-webhook
+    /etc/init.d/eks-pod-identity-webhook /etc/init.d/mulga-eks-log-forward
 chmod 0755 /usr/local/sbin/eks-node-role /usr/local/sbin/k3s-first-boot \
     /usr/local/sbin/mulga-eks-state-report /usr/local/sbin/mulga-eks-addon-sync \
-    /usr/local/sbin/mulga-eks-node-gc
+    /usr/local/sbin/mulga-eks-node-gc /usr/local/sbin/mulga-eks-log-forward
 # start_pre helpers shared with the systemd wrappers that will eventually
 # replace these runscripts.
 chmod 0755 /usr/local/sbin/k3s-prestart /usr/local/sbin/k3s-agent-prestart \

spinifex/domains/ec2/awsapi/volume/AttachVolume.go

@@ -49,6 +49,12 @@ func AttachVolume(ctx context.Context, input *ec2.AttachVolumeInput, natsConn *n
 		device = *input.Device
 	}
 
+	// The shared record decides: a stopped instance has no QMP to plug into, so
+	// its record is edited and the volume claimed there instead.
+	if isStoppedInstance(ctx, instanceID, natsConn, accountID) {
+		return attachToStopped(ctx, natsConn, accountID, input)
+	}
+
 	command := ec2v1.EC2InstanceCommand{
 		ID: instanceID,
 		Attributes: ec2v1.EC2CommandAttributes{
@@ -97,6 +103,24 @@ func AttachVolume(ctx context.Context, input *ec2.AttachVolumeInput, natsConn *n
 	return output, nil
 }
 
+// attachToStoppedSubject is answered by any node and edits the stopped
+// instance record in the shared bucket.
+const attachToStoppedSubject = "ec2.AttachVolumeToStopped"
+
+// attachToStopped asks the instance service to add the volume to the stopped
+// instance's record; the volume is claimed in-use by the same call.
+func attachToStopped(ctx context.Context, natsConn *nats.Conn, accountID string, input *ec2.AttachVolumeInput) (ec2.VolumeAttachment, error) {
+	attachment, err := natsmsg.NATSRequest[ec2.VolumeAttachment](ctx, natsConn, attachToStoppedSubject, input, 30*time.Second, accountID)
+	if err != nil {
+		slog.ErrorContext(ctx, "AttachVolume: stopped-instance attach failed", "volumeId", aws.StringValue(input.VolumeId), "instanceId", aws.StringValue(input.InstanceId), "err", err)
+		return ec2.VolumeAttachment{}, err
+	}
+	if attachment == nil {
+		return ec2.VolumeAttachment{}, nil
+	}
+	return *attachment, nil
+}
+
 // isStoppedInstance checks the shared KV (via ec2.DescribeStoppedInstances) to
 // determine whether instanceID exists as a stopped instance.
 func isStoppedInstance(ctx context.Context, instanceID string, natsConn *nats.Conn, accountID string) bool {

spinifex/domains/ec2/awsapi/volume/AttachVolume_stopped_test.go

@@ -0,0 +1,55 @@
+package volume
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// The record decides the path: a stopped instance has no QMP to plug into, so
+// the attach goes to the instance service and ec2.cmd.<id> is never used.
+func TestAttachVolume_StoppedInstanceUsesTheRecordPath(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	log := &subjectLog{}
+
+	stoppedRecord := &ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{{
+		Instances: []*ec2.Instance{{InstanceId: aws.String("i-stopped")}},
+	}}}
+	handlers := map[string]func(*nats.Msg){
+		"ec2.DescribeStoppedInstances": func(m *nats.Msg) { respondJSON(t, m, stoppedRecord) },
+		attachToStoppedSubject: func(m *nats.Msg) {
+			log.add(attachToStoppedSubject)
+			respondJSON(t, m, &ec2.VolumeAttachment{
+				VolumeId: aws.String("vol-data"), InstanceId: aws.String("i-stopped"),
+				Device: aws.String("/dev/sdf"), State: aws.String("attached"),
+			})
+		},
+		"ec2.cmd.i-stopped": func(m *nats.Msg) {
+			log.add("ec2.cmd.i-stopped")
+			_ = m.Respond(nil)
+		},
+	}
+	for subject, h := range handlers {
+		sub, err := nc.Subscribe(subject, h)
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = sub.Unsubscribe() })
+	}
+	require.NoError(t, nc.Flush())
+
+	out, err := AttachVolume(context.Background(), &ec2.AttachVolumeInput{
+		VolumeId:   aws.String("vol-data"),
+		InstanceId: aws.String("i-stopped"),
+		Device:     aws.String("/dev/sdf"),
+	}, nc, "000000000001")
+	require.NoError(t, err)
+
+	assert.Equal(t, "attached", aws.StringValue(out.State))
+	assert.Equal(t, []string{attachToStoppedSubject}, log.get(),
+		"only the record path runs for a stopped instance")
+}

spinifex/domains/ec2/awsapi/volume/DetachVolume.go

@@ -74,6 +74,12 @@ func DetachVolume(ctx context.Context, input *ec2.DetachVolumeInput, natsConn *n
 
 	force := input.Force != nil && *input.Force
 
+	// The shared record decides: a stopped instance has no QMP to unplug from,
+	// so its record is edited first and the volume released after.
+	if isStoppedInstance(ctx, instanceID, natsConn, accountID) {
+		return detachFromStopped(ctx, natsConn, accountID, volumeID, instanceID, device)
+	}
+
 	command := ec2v1.EC2InstanceCommand{
 		ID: instanceID,
 		Attributes: ec2v1.EC2CommandAttributes{
@@ -131,6 +137,10 @@ func DetachVolume(ctx context.Context, input *ec2.DetachVolumeInput, natsConn *n
 // running the instance and so cannot help when that host is the problem.
 const forceDetachSubject = "ec2.ForceDetachVolume"
 
+// detachFromStoppedSubject is answered by any node and edits the stopped
+// instance record in the shared bucket.
+const detachFromStoppedSubject = "ec2.DetachVolumeFromStopped"
+
 // forceDetach releases a volume whose instance has no responder.
 //
 // A volume outlives the instance it was attached to whenever teardown failed
@@ -153,3 +163,27 @@ func forceDetach(ctx context.Context, natsConn *nats.Conn, accountID, volumeID,
 	}
 	return *attachment, nil
 }
+
+// detachFromStopped removes the volume from the stopped instance's record and
+// then releases the volume, in that order, so a failed release leaves the
+// record clean and the volume still held rather than the reverse.
+func detachFromStopped(ctx context.Context, natsConn *nats.Conn, accountID, volumeID, instanceID, device string) (ec2.VolumeAttachment, error) {
+	in := &ec2.DetachVolumeInput{VolumeId: &volumeID, InstanceId: &instanceID}
+	if device != "" {
+		in.Device = &device
+	}
+	if _, err := natsmsg.NATSRequest[ec2.VolumeAttachment](ctx, natsConn, detachFromStoppedSubject, in, 30*time.Second, accountID); err != nil {
+		slog.ErrorContext(ctx, "DetachVolume: stopped-instance record edit failed", "volumeId", volumeID, "instanceId", instanceID, "err", err)
+		return ec2.VolumeAttachment{}, err
+	}
+	attachment, err := natsmsg.NATSRequest[ec2.VolumeAttachment](ctx, natsConn, forceDetachSubject,
+		&ec2.DetachVolumeInput{VolumeId: &volumeID, Force: aws.Bool(true)}, 30*time.Second, accountID)
+	if err != nil {
+		slog.ErrorContext(ctx, "DetachVolume: release after record edit failed", "volumeId", volumeID, "instanceId", instanceID, "err", err)
+		return ec2.VolumeAttachment{}, err
+	}
+	if attachment == nil {
+		return ec2.VolumeAttachment{}, nil
+	}
+	return *attachment, nil
+}

spinifex/domains/ec2/awsapi/volume/DetachVolume_stopped_test.go

@@ -0,0 +1,115 @@
+package volume
+
+import (
+	"context"
+	"encoding/json"
+	"sync"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type subjectLog struct {
+	mu   sync.Mutex
+	seen []string
+}
+
+func (l *subjectLog) add(s string) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.seen = append(l.seen, s)
+}
+
+func (l *subjectLog) get() []string {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return append([]string(nil), l.seen...)
+}
+
+func respondJSON(t *testing.T, msg *nats.Msg, v any) {
+	t.Helper()
+	data, err := json.Marshal(v)
+	require.NoError(t, err)
+	require.NoError(t, msg.Respond(data))
+}
+
+// The record decides the path. A stopped instance has no QMP to unplug from, so
+// its record is edited and the volume released, and ec2.cmd.<id> is never used.
+func TestDetachVolume_StoppedInstanceEditsTheRecordThenReleasesTheVolume(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	log := &subjectLog{}
+
+	stoppedRecord := &ec2.DescribeInstancesOutput{Reservations: []*ec2.Reservation{{
+		Instances: []*ec2.Instance{{InstanceId: aws.String("i-stopped")}},
+	}}}
+	handlers := map[string]func(*nats.Msg){
+		"ec2.DescribeStoppedInstances": func(m *nats.Msg) { respondJSON(t, m, stoppedRecord) },
+		detachFromStoppedSubject: func(m *nats.Msg) {
+			log.add(detachFromStoppedSubject)
+			respondJSON(t, m, &ec2.VolumeAttachment{
+				VolumeId: aws.String("vol-data"), InstanceId: aws.String("i-stopped"), Device: aws.String("/dev/sdf"),
+			})
+		},
+		forceDetachSubject: func(m *nats.Msg) {
+			log.add(forceDetachSubject)
+			respondJSON(t, m, &ec2.VolumeAttachment{
+				VolumeId: aws.String("vol-data"), InstanceId: aws.String("i-stopped"), State: aws.String("detached"),
+			})
+		},
+		"ec2.cmd.i-stopped": func(m *nats.Msg) {
+			log.add("ec2.cmd.i-stopped")
+			_ = m.Respond(nil)
+		},
+	}
+	for subject, h := range handlers {
+		sub, err := nc.Subscribe(subject, h)
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = sub.Unsubscribe() })
+	}
+	require.NoError(t, nc.Flush())
+
+	out, err := DetachVolume(context.Background(), &ec2.DetachVolumeInput{
+		VolumeId:   aws.String("vol-data"),
+		InstanceId: aws.String("i-stopped"),
+	}, nc, "000000000001")
+	require.NoError(t, err)
+
+	assert.Equal(t, "detached", aws.StringValue(out.State))
+	assert.Equal(t, []string{detachFromStoppedSubject, forceDetachSubject}, log.get(),
+		"record first, then the volume; the host path is never used for a stopped instance")
+}
+
+// NoResponders alone must not be read as "stopped": a dead or partitioned node
+// gives the same answer, and editing the record then would race a live QEMU.
+func TestDetachVolume_NoRespondersAloneDoesNotMeanStopped(t *testing.T) {
+	_, nc := testutil.StartTestNATS(t)
+	log := &subjectLog{}
+
+	for _, subject := range []string{detachFromStoppedSubject, forceDetachSubject} {
+		sub, err := nc.Subscribe(subject, func(m *nats.Msg) {
+			log.add(subject)
+			_ = m.Respond(nil)
+		})
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = sub.Unsubscribe() })
+	}
+	sub, err := nc.Subscribe("ec2.DescribeStoppedInstances", func(m *nats.Msg) {
+		respondJSON(t, m, &ec2.DescribeInstancesOutput{})
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	require.NoError(t, nc.Flush())
+
+	_, err = DetachVolume(context.Background(), &ec2.DetachVolumeInput{
+		VolumeId:   aws.String("vol-data"),
+		InstanceId: aws.String("i-silent-node"),
+	}, nc, "000000000001")
+
+	require.Error(t, err)
+	assert.Empty(t, log.get(), "no record edit and no control-plane release without a stopped record")
+}

spinifex/domains/ec2/instance/service_impl.go

@@ -2700,6 +2700,12 @@ func (s *InstanceServiceImpl) launchClaimedInstance(ctx context.Context, instanc
 	// failure of this launch.
 	instance.FirstLaunch = false
 
+	if err := s.reconcileStartVolumes(ctx, instance); err != nil {
+		slog.ErrorContext(ctx, "launchClaimedInstance: volumes cannot be reconciled", "instanceId", instance.ID, "err", err)
+		undo()
+		return err
+	}
+
 	instanceType, ok := s.resourceMgr.InstanceTypes()[instance.InstanceType]
 	if !ok {
 		slog.ErrorContext(ctx, "launchClaimedInstance: instance type not available on this node",
@@ -3674,3 +3680,153 @@ func (s *InstanceServiceImpl) DescribeInstanceStatus(ctx context.Context, input
 	slog.InfoContext(ctx, "DescribeInstanceStatus completed", "count", len(statuses))
 	return &ec2.DescribeInstanceStatusOutput{InstanceStatuses: statuses}, nil
 }
+
+// DetachVolumeFromStopped removes a data volume from a stopped instance's
+// record, so a later start no longer attaches it. It does not touch the
+// volume's own state; the caller releases that once the record is clean. The
+// device the volume held comes back on the attachment.
+func (s *InstanceServiceImpl) DetachVolumeFromStopped(ctx context.Context, input *ec2.DetachVolumeInput, accountID string) (*ec2.VolumeAttachment, error) {
+	if input == nil || aws.StringValue(input.InstanceId) == "" || aws.StringValue(input.VolumeId) == "" {
+		return nil, errors.New(awserrors.ErrorMissingParameter)
+	}
+	if s.stoppedStore == nil {
+		slog.ErrorContext(ctx, "DetachVolumeFromStopped: stopped store not available")
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	instanceID, volumeID := *input.InstanceId, *input.VolumeId
+
+	stored, err := s.stoppedStore.LoadStoppedInstance(instanceID)
+	if err != nil {
+		slog.ErrorContext(ctx, "DetachVolumeFromStopped: failed to load stopped instance", "instanceId", instanceID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if stored == nil || !IsInstanceVisible(accountID, stored.AccountID) {
+		return nil, errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
+	}
+
+	req, ok := findVolumeRequest(stored, volumeID)
+	if !ok {
+		return nil, errors.New(awserrors.ErrorIncorrectState)
+	}
+	if req.EFI {
+		return nil, errors.New(awserrors.ErrorOperationNotPermitted)
+	}
+
+	var removed bool
+	_, err = s.stoppedStore.UpdateStoppedInstance(instanceID, func(v *vm.VM) {
+		removed = removeVolumeFromRecord(v, volumeID)
+	})
+	if err != nil {
+		if errors.Is(err, kvstore.ErrNotFound) {
+			slog.WarnContext(ctx, "DetachVolumeFromStopped: instance claimed concurrently", "instanceId", instanceID)
+			return nil, errors.New(awserrors.ErrorIncorrectInstanceState)
+		}
+		slog.ErrorContext(ctx, "DetachVolumeFromStopped: failed to update stopped instance", "instanceId", instanceID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if !removed {
+		return nil, errors.New(awserrors.ErrorIncorrectState)
+	}
+
+	return &ec2.VolumeAttachment{
+		VolumeId:   aws.String(volumeID),
+		InstanceId: aws.String(instanceID),
+		Device:     aws.String(req.DeviceName),
+		State:      aws.String(ec2.VolumeAttachmentStateDetaching),
+	}, nil
+}
+
+// AttachVolumeToStopped adds an available volume to a stopped instance's
+// record, so the next start mounts it. The volume is claimed in-use first and
+// released again if the record edit fails, so a failure never leaves the record
+// naming a volume another instance could take.
+func (s *InstanceServiceImpl) AttachVolumeToStopped(ctx context.Context, input *ec2.AttachVolumeInput, accountID string) (*ec2.VolumeAttachment, error) {
+	if input == nil || aws.StringValue(input.InstanceId) == "" || aws.StringValue(input.VolumeId) == "" {
+		return nil, errors.New(awserrors.ErrorMissingParameter)
+	}
+	if s.stoppedStore == nil || s.metadata == nil {
+		slog.ErrorContext(ctx, "AttachVolumeToStopped: stores not available")
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	instanceID, volumeID := *input.InstanceId, *input.VolumeId
+
+	stored, err := s.stoppedStore.LoadStoppedInstance(instanceID)
+	if err != nil {
+		slog.ErrorContext(ctx, "AttachVolumeToStopped: failed to load stopped instance", "instanceId", instanceID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if stored == nil || !IsInstanceVisible(accountID, stored.AccountID) {
+		return nil, errors.New(awserrors.ErrorInvalidInstanceIDNotFound)
+	}
+
+	meta, err := s.metadata.GetVolume(ctx, accountID, volumeID)
+	if err != nil {
+		if objectstore.IsNoSuchKeyError(err) {
+			return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound)
+		}
+		slog.ErrorContext(ctx, "AttachVolumeToStopped: failed to read volume", "volumeId", volumeID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if meta.TenantID != accountID {
+		return nil, errors.New(awserrors.ErrorInvalidVolumeNotFound)
+	}
+	if meta.State != "available" {
+		return nil, errors.New(awserrors.ErrorVolumeInUse)
+	}
+	if meta.AvailabilityZone != "" && s.config != nil && s.config.AZ != "" && meta.AvailabilityZone != s.config.AZ {
+		return nil, errors.New(awserrors.ErrorInvalidVolumeZoneMismatch)
+	}
+
+	device, boot, err := stoppedAttachDevice(stored, aws.StringValue(input.Device))
+	if err != nil {
+		return nil, err
+	}
+
+	meta.State = "in-use"
+	meta.AttachedInstance = instanceID
+	meta.DeviceName = device
+	meta.AttachedAt = time.Now()
+	if err := s.metadata.PutVolume(ctx, meta); err != nil {
+		slog.ErrorContext(ctx, "AttachVolumeToStopped: failed to claim volume", "volumeId", volumeID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+
+	var conflict error
+	_, err = s.stoppedStore.UpdateStoppedInstance(instanceID, func(v *vm.VM) {
+		// The device was chosen against a snapshot, so it is checked again
+		// against the record the edit is applied to.
+		if _, _, cerr := stoppedAttachDevice(v, device); cerr != nil {
+			conflict = cerr
+			return
+		}
+		addVolumeToRecord(v, volumeID, device, boot)
+	})
+	if err == nil && conflict != nil {
+		err = conflict
+	}
+	if err != nil {
+		meta.State = "available"
+		meta.AttachedInstance = ""
+		meta.DeviceName = ""
+		meta.AttachedAt = time.Time{}
+		if rerr := s.metadata.PutVolume(ctx, meta); rerr != nil {
+			slog.ErrorContext(ctx, "AttachVolumeToStopped: failed to release volume after a failed record edit", "volumeId", volumeID, "err", rerr)
+		}
+		if conflict != nil {
+			return nil, conflict
+		}
+		if errors.Is(err, kvstore.ErrNotFound) {
+			slog.WarnContext(ctx, "AttachVolumeToStopped: instance claimed concurrently", "instanceId", instanceID)
+			return nil, errors.New(awserrors.ErrorIncorrectInstanceState)
+		}
+		slog.ErrorContext(ctx, "AttachVolumeToStopped: failed to update stopped instance", "instanceId", instanceID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+
+	return &ec2.VolumeAttachment{
+		VolumeId:   aws.String(volumeID),
+		InstanceId: aws.String(instanceID),
+		Device:     aws.String(device),
+		State:      aws.String(ec2.VolumeAttachmentStateAttaching),
+	}, nil
+}

spinifex/domains/ec2/instance/start_after_detach_test.go

@@ -0,0 +1,54 @@
+//test:in-package — it swaps the manager's volume mounter, an unexported dependency.
+package instance
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"testing"
+
+	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// mountRecorder notes which volumes a launch asked to mount and then fails the
+// launch, which is the earliest point that shows what the start would attach.
+type mountRecorder struct {
+	refusingMounter
+
+	mu      sync.Mutex
+	mounted []string
+}
+
+func (r *mountRecorder) Mount(_ context.Context, v *vm.VM) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	for _, req := range v.EBSRequests.Requests {
+		r.mounted = append(r.mounted, req.Name)
+	}
+	return errors.New("stop after recording")
+}
+
+func (r *mountRecorder) MountOne(context.Context, string, *viperblocklegacyv1.EBSRequest) error {
+	return nil
+}
+
+// A volume detached while the instance was stopped must not come back when the
+// instance is started: the start path reads the same record the detach edited.
+func TestStartStoppedInstance_DoesNotReattachAVolumeDetachedWhileStopped(t *testing.T) {
+	id := "i-detach-then-start"
+	stopped := stoppedWithVolumes("acc", id)
+	stopped.InstanceType = "t3.micro"
+	f := newStartFixture(t, stopped)
+	rec := &mountRecorder{}
+	f.svc.vmMgr.SetDeps(vm.Deps{VolumeMounter: rec})
+
+	_, err := f.svc.DetachVolumeFromStopped(context.Background(), stoppedDetachInput(id, "vol-data"), "acc")
+	require.NoError(t, err)
+	require.Error(t, f.start(t, id), "the recorder fails the launch after the mount")
+
+	assert.Equal(t, []string{"vol-root", "vol-other"}, rec.mounted,
+		"the detached volume is not part of the start")
+}

spinifex/domains/ec2/instance/start_volumes.go

@@ -0,0 +1,86 @@
+package instance
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"slices"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+)
+
+// reconcileStartVolumes drops the data volumes a claimed instance can no longer
+// have, and fails the start when its boot volume is gone.
+func (s *InstanceServiceImpl) reconcileStartVolumes(ctx context.Context, instance *vm.VM) error {
+	if s.metadata == nil {
+		return nil
+	}
+
+	instance.EBSRequests.Mu.Lock()
+	requests := slices.Clone(instance.EBSRequests.Requests)
+	instance.EBSRequests.Mu.Unlock()
+
+	if !slices.ContainsFunc(requests, func(r viperblocklegacyv1.EBSRequest) bool { return r.Boot }) {
+		return noRootVolumeError(instance)
+	}
+
+	var drop []string
+	for _, req := range requests {
+		meta, err := s.metadata.GetVolume(ctx, instance.AccountID, req.Name)
+		missing := err != nil && objectstore.IsNoSuchKeyError(err)
+		if err != nil && !missing {
+			return fmt.Errorf("read volume %s: %w", req.Name, err)
+		}
+		if req.Boot {
+			if missing {
+				slog.ErrorContext(ctx, "start: root volume no longer exists", "instanceId", instance.ID, "volumeId", req.Name)
+				return noRootVolumeError(instance)
+			}
+			continue
+		}
+		if req.EFI {
+			continue
+		}
+		if missing {
+			slog.WarnContext(ctx, "start: dropping deleted data volume", "instanceId", instance.ID, "volumeId", req.Name)
+			drop = append(drop, req.Name)
+			continue
+		}
+		if meta.AttachedInstance != "" && meta.AttachedInstance != instance.ID {
+			slog.WarnContext(ctx, "start: dropping data volume attached to another instance",
+				"instanceId", instance.ID, "volumeId", req.Name, "attachedTo", meta.AttachedInstance)
+			drop = append(drop, req.Name)
+		}
+	}
+	if len(drop) == 0 {
+		return nil
+	}
+
+	instance.EBSRequests.Mu.Lock()
+	instance.EBSRequests.Requests = slices.DeleteFunc(instance.EBSRequests.Requests, func(r viperblocklegacyv1.EBSRequest) bool {
+		return slices.Contains(drop, r.Name)
+	})
+	instance.EBSRequests.Mu.Unlock()
+	if instance.Instance != nil {
+		instance.Instance.BlockDeviceMappings = slices.DeleteFunc(instance.Instance.BlockDeviceMappings, func(m *ec2.InstanceBlockDeviceMapping) bool {
+			return m != nil && m.Ebs != nil && slices.Contains(drop, aws.StringValue(m.Ebs.VolumeId))
+		})
+	}
+	return nil
+}
+
+// noRootVolumeError is what AWS answers when an instance is started with
+// nothing attached at its root device name.
+func noRootVolumeError(instance *vm.VM) error {
+	device := ebsmetadata.RootDeviceName
+	if instance.Instance != nil && aws.StringValue(instance.Instance.RootDeviceName) != "" {
+		device = aws.StringValue(instance.Instance.RootDeviceName)
+	}
+	return awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "Instance does not have a volume attached at root (%s)", device)
+}

spinifex/domains/ec2/instance/start_volumes_test.go

@@ -0,0 +1,125 @@
+package instance
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const startVolumesAccount = "000000000011"
+
+func startVolumesService(t *testing.T, volumes ...ebsmetadata.Volume) *InstanceServiceImpl {
+	t.Helper()
+	store := objectstore.NewMemoryObjectStore()
+	meta := ebsmetadata.NewStore(store, testRootBucket)
+	for _, v := range volumes {
+		v.TenantID = startVolumesAccount
+		require.NoError(t, meta.PutVolume(context.Background(), v))
+	}
+	return &InstanceServiceImpl{metadata: meta}
+}
+
+func startingInstance(id string, names ...string) *vm.VM {
+	v := &vm.VM{ID: id, AccountID: startVolumesAccount, Status: vm.StateStopped, Instance: &ec2.Instance{InstanceId: aws.String(id)}}
+	for i, n := range names {
+		boot := i == 0
+		device := "/dev/sd" + string(rune('f'+i))
+		if boot {
+			device = "/dev/sda1"
+		}
+		v.EBSRequests.Requests = append(v.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{Name: n, Boot: boot, DeviceName: device})
+		v.Instance.BlockDeviceMappings = append(v.Instance.BlockDeviceMappings, &ec2.InstanceBlockDeviceMapping{
+			DeviceName: aws.String(device), Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String(n)},
+		})
+	}
+	return v
+}
+
+func requestedNames(v *vm.VM) []string {
+	var out []string
+	for _, r := range v.EBSRequests.Requests {
+		out = append(out, r.Name)
+	}
+	return out
+}
+
+func mappedNames(v *vm.VM) []string {
+	var out []string
+	for _, m := range v.Instance.BlockDeviceMappings {
+		out = append(out, aws.StringValue(m.Ebs.VolumeId))
+	}
+	return out
+}
+
+// An instance stopped on an older build had its data volumes released, and they
+// may since have been attached elsewhere or deleted; starting must not take a
+// volume that is no longer its own.
+func TestReconcileStartVolumes_DropsVolumesThatAreNoLongerTheInstances(t *testing.T) {
+	svc := startVolumesService(t,
+		ebsmetadata.Volume{VolumeID: "vol-root", State: "in-use", AttachedInstance: "i-old"},
+		ebsmetadata.Volume{VolumeID: "vol-elsewhere", State: "in-use", AttachedInstance: "i-someone-else"},
+		ebsmetadata.Volume{VolumeID: "vol-free", State: "available"},
+		ebsmetadata.Volume{VolumeID: "vol-mine", State: "in-use", AttachedInstance: "i-old"},
+	)
+	inst := startingInstance("i-old", "vol-root", "vol-elsewhere", "vol-gone", "vol-free", "vol-mine")
+
+	require.NoError(t, svc.reconcileStartVolumes(context.Background(), inst))
+
+	want := []string{"vol-root", "vol-free", "vol-mine"}
+	assert.Equal(t, want, requestedNames(inst),
+		"attached-elsewhere and deleted volumes are dropped; free and own ones stay")
+	assert.Equal(t, want, mappedNames(inst), "the API view stops listing them too")
+}
+
+func TestReconcileStartVolumes_MissingBootVolumeFailsTheStart(t *testing.T) {
+	svc := startVolumesService(t, ebsmetadata.Volume{VolumeID: "vol-data", State: "available"})
+	inst := startingInstance("i-old", "vol-root", "vol-data")
+	inst.Instance.RootDeviceName = aws.String("/dev/sda1")
+
+	err := svc.reconcileStartVolumes(context.Background(), inst)
+
+	require.Error(t, err)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidParameterValue), "got %v", err)
+	assert.Contains(t, err.Error(), "Instance does not have a volume attached at root (/dev/sda1)")
+}
+
+// A root volume detached while the instance was stopped leaves no boot request
+// in the record; AWS refuses the start with the same answer.
+func TestReconcileStartVolumes_NoRootVolumeAttachedFailsTheStart(t *testing.T) {
+	svc := startVolumesService(t, ebsmetadata.Volume{VolumeID: "vol-data", State: "in-use", AttachedInstance: "i-old"})
+	inst := startingInstance("i-old", "vol-root", "vol-data")
+	require.True(t, removeVolumeFromRecord(inst, "vol-root"))
+	inst.Instance.RootDeviceName = aws.String("/dev/sda1")
+
+	err := svc.reconcileStartVolumes(context.Background(), inst)
+
+	require.Error(t, err)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidParameterValue), "got %v", err)
+	assert.Contains(t, err.Error(), "Instance does not have a volume attached at root (/dev/sda1)")
+	assert.Equal(t, []string{"vol-data"}, requestedNames(inst), "a refused start changes nothing")
+}
+
+// An instance stopped on an older build had its data volumes released to
+// available with no attachment; those are still the instance's own and a
+// start must bring them back.
+func TestReconcileStartVolumes_ReattachesStillAvailableUnattachedVolume(t *testing.T) {
+	svc := startVolumesService(t,
+		ebsmetadata.Volume{VolumeID: "vol-root", State: "in-use", AttachedInstance: "i-old"},
+		ebsmetadata.Volume{VolumeID: "vol-released", State: "available"},
+	)
+	inst := startingInstance("i-old", "vol-root", "vol-released")
+
+	require.NoError(t, svc.reconcileStartVolumes(context.Background(), inst))
+
+	assert.Equal(t, []string{"vol-root", "vol-released"}, requestedNames(inst))
+	assert.Equal(t, []string{"vol-root", "vol-released"}, mappedNames(inst))
+}

spinifex/domains/ec2/instance/stopped_attach_test.go

@@ -0,0 +1,180 @@
+//test:in-package — it swaps the manager's volume mounter, an unexported dependency.
+package instance
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	vmmock "github.com/mulgadc/spinifex/spinifex/runtime/compute/vm/mock"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const stoppedAttachAZ = "ap-southeast-2a"
+
+type stoppedAttachFixture struct {
+	svc    *InstanceServiceImpl
+	stored *vm.VM
+	id     string
+}
+
+func newStoppedAttachFixture(t *testing.T, volumes ...ebsmetadata.Volume) *stoppedAttachFixture {
+	t.Helper()
+	id := "i-stopped-attach"
+	svc := startVolumesService(t, volumes...)
+	stored := startingInstance(id, "vol-root", "vol-data")
+	svc.config = &config.Config{AZ: stoppedAttachAZ}
+	svc.vmMgr = mgrWith(map[string]*vm.VM{})
+	svc.stoppedStore = &vmmock.StateStore{Stopped: map[string]*vm.VM{id: stored}}
+	return &stoppedAttachFixture{svc: svc, stored: stored, id: id}
+}
+
+func (f *stoppedAttachFixture) attach(volumeID, device string) (*ec2.VolumeAttachment, error) {
+	in := &ec2.AttachVolumeInput{InstanceId: aws.String(f.id), VolumeId: aws.String(volumeID)}
+	if device != "" {
+		in.Device = aws.String(device)
+	}
+	return f.svc.AttachVolumeToStopped(context.Background(), in, startVolumesAccount)
+}
+
+func (f *stoppedAttachFixture) volume(t *testing.T, id string) ebsmetadata.Volume {
+	t.Helper()
+	v, err := f.svc.metadata.GetVolume(context.Background(), startVolumesAccount, id)
+	require.NoError(t, err)
+	return v
+}
+
+func availableVolume(id string) ebsmetadata.Volume {
+	return ebsmetadata.Volume{VolumeID: id, State: "available", AvailabilityZone: stoppedAttachAZ}
+}
+
+// AWS lets a volume be attached to a stopped instance; the record then names it
+// so the next start mounts it, and the volume is held in-use meanwhile.
+func TestAttachVolumeToStopped_AddsTheVolumeAndClaimsIt(t *testing.T) {
+	f := newStoppedAttachFixture(t, availableVolume("vol-new"))
+
+	out, err := f.attach("vol-new", "")
+	require.NoError(t, err)
+
+	assert.Equal(t, "/dev/sdf", aws.StringValue(out.Device), "the first free data device")
+	assert.Equal(t, []string{"vol-root", "vol-data", "vol-new"}, requestedNames(f.stored))
+	assert.Equal(t, []string{"vol-root", "vol-data", "vol-new"}, mappedNames(f.stored))
+	assert.Equal(t, vm.StateStopped, f.stored.Status)
+	got := f.volume(t, "vol-new")
+	assert.Equal(t, "in-use", got.State)
+	assert.Equal(t, f.id, got.AttachedInstance)
+	assert.Equal(t, "/dev/sdf", got.DeviceName)
+	for _, r := range f.stored.EBSRequests.Requests {
+		if r.Name == "vol-new" {
+			assert.False(t, r.Boot, "a data device is not the root")
+		}
+	}
+}
+
+func TestAttachVolumeToStopped_Refusals(t *testing.T) {
+	tests := []struct {
+		name    string
+		volumes []ebsmetadata.Volume
+		volume  string
+		device  string
+		want    string
+	}{
+		{"volume in use elsewhere", []ebsmetadata.Volume{{VolumeID: "vol-busy", State: "in-use", AttachedInstance: "i-other", AvailabilityZone: stoppedAttachAZ}}, "vol-busy", "", awserrors.ErrorVolumeInUse},
+		{"wrong availability zone", []ebsmetadata.Volume{{VolumeID: "vol-far", State: "available", AvailabilityZone: "ap-southeast-2b"}}, "vol-far", "", awserrors.ErrorInvalidVolumeZoneMismatch},
+		{"no such volume", nil, "vol-missing", "", awserrors.ErrorInvalidVolumeNotFound},
+		{"device already taken", []ebsmetadata.Volume{availableVolume("vol-new")}, "vol-new", "/dev/sdg", awserrors.ErrorInvalidParameterValue},
+		{"root device already has a volume", []ebsmetadata.Volume{availableVolume("vol-new")}, "vol-new", "/dev/sda1", awserrors.ErrorInvalidParameterValue},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			f := newStoppedAttachFixture(t, tt.volumes...)
+
+			_, err := f.attach(tt.volume, tt.device)
+
+			require.Error(t, err)
+			assert.True(t, awserrors.IsErrorCode(err, tt.want), "got %v, want %s", err, tt.want)
+			assert.Equal(t, []string{"vol-root", "vol-data"}, requestedNames(f.stored), "a refused attach changes nothing")
+			for _, v := range tt.volumes {
+				got := f.volume(t, v.VolumeID)
+				assert.Equal(t, v.State, got.State, "a refused attach leaves the volume as it was")
+				assert.Equal(t, v.AttachedInstance, got.AttachedInstance)
+			}
+		})
+	}
+}
+
+func TestAttachVolumeToStopped_OtherAccountsInstanceIsNotFound(t *testing.T) {
+	f := newStoppedAttachFixture(t, availableVolume("vol-new"))
+
+	_, err := f.svc.AttachVolumeToStopped(context.Background(),
+		&ec2.AttachVolumeInput{InstanceId: aws.String(f.id), VolumeId: aws.String("vol-new")}, "000000000099")
+
+	require.Error(t, err)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorInvalidInstanceIDNotFound))
+	assert.Equal(t, []string{"vol-root", "vol-data"}, requestedNames(f.stored))
+}
+
+// A record edit that cannot be applied must give the volume back, otherwise it
+// stays in-use against an instance that does not name it.
+func TestAttachVolumeToStopped_FailedRecordEditReleasesTheVolume(t *testing.T) {
+	f := newStoppedAttachFixture(t, availableVolume("vol-new"))
+	f.svc.stoppedStore = &vmmock.StateStore{Stopped: map[string]*vm.VM{f.id: f.stored}, UpdateStoppedErr: assert.AnError}
+
+	_, err := f.attach("vol-new", "")
+
+	require.Error(t, err)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorServerInternal))
+	got := f.volume(t, "vol-new")
+	assert.Equal(t, "available", got.State)
+	assert.Empty(t, got.AttachedInstance)
+}
+
+// Detaching the root volume of a stopped instance makes it unstartable; the way
+// back is attaching a volume at the root device name, which makes it the boot
+// volume again so the start finds a root.
+func TestAttachVolumeToStopped_RootDeviceNameRestoresTheRootAndTheStartWorks(t *testing.T) {
+	id := "i-root-roundtrip"
+	stopped := stoppedWithVolumes(startVolumesAccount, id)
+	stopped.InstanceType = "t3.micro"
+	stopped.Instance.RootDeviceName = aws.String("/dev/sda1")
+	f := newStartFixture(t, stopped)
+	rec := &mountRecorder{}
+	f.svc.vmMgr.SetDeps(vm.Deps{VolumeMounter: rec})
+	f.svc.config = &config.Config{AZ: stoppedAttachAZ}
+	meta := startVolumesService(t,
+		ebsmetadata.Volume{VolumeID: "vol-root", State: "in-use", AttachedInstance: id, AvailabilityZone: stoppedAttachAZ},
+		ebsmetadata.Volume{VolumeID: "vol-data", State: "in-use", AttachedInstance: id, AvailabilityZone: stoppedAttachAZ},
+		ebsmetadata.Volume{VolumeID: "vol-other", State: "in-use", AttachedInstance: id, AvailabilityZone: stoppedAttachAZ},
+	)
+	f.svc.metadata = meta.metadata
+	stopped.AccountID = startVolumesAccount
+
+	_, err := f.svc.DetachVolumeFromStopped(context.Background(), stoppedDetachInput(id, "vol-root"), startVolumesAccount)
+	require.NoError(t, err)
+	vol, err := f.svc.metadata.GetVolume(context.Background(), startVolumesAccount, "vol-root")
+	require.NoError(t, err)
+	vol.State, vol.AttachedInstance = "available", ""
+	require.NoError(t, f.svc.metadata.PutVolume(context.Background(), vol))
+
+	out, err := f.svc.AttachVolumeToStopped(context.Background(),
+		&ec2.AttachVolumeInput{InstanceId: aws.String(id), VolumeId: aws.String("vol-root"), Device: aws.String("/dev/sda1")}, startVolumesAccount)
+	require.NoError(t, err)
+	assert.Equal(t, "/dev/sda1", aws.StringValue(out.Device))
+	var boot bool
+	for _, r := range stopped.EBSRequests.Requests {
+		if r.Name == "vol-root" {
+			boot = r.Boot
+		}
+	}
+	assert.True(t, boot, "the root device name makes it the boot volume")
+
+	_, err = f.svc.StartStoppedInstance(context.Background(), &StartStoppedInstanceInput{InstanceID: id}, startVolumesAccount)
+	require.Error(t, err, "the recorder fails the launch after the mount")
+	assert.Contains(t, rec.mounted, "vol-root", "the start now mounts the restored root")
+}

spinifex/domains/ec2/instance/stopped_detach.go

@@ -0,0 +1,112 @@
+package instance
+
+import (
+	"errors"
+	"fmt"
+	"slices"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+)
+
+func findVolumeRequest(v *vm.VM, volumeID string) (viperblocklegacyv1.EBSRequest, bool) {
+	v.EBSRequests.Mu.Lock()
+	defer v.EBSRequests.Mu.Unlock()
+	i := slices.IndexFunc(v.EBSRequests.Requests, func(r viperblocklegacyv1.EBSRequest) bool { return r.Name == volumeID })
+	if i < 0 {
+		return viperblocklegacyv1.EBSRequest{}, false
+	}
+	return v.EBSRequests.Requests[i], true
+}
+
+// removeVolumeFromRecord drops the volume from the mount list and the API view
+// of the record, and reports whether it was a data volume that was there.
+func removeVolumeFromRecord(v *vm.VM, volumeID string) bool {
+	v.EBSRequests.Mu.Lock()
+	defer v.EBSRequests.Mu.Unlock()
+	i := slices.IndexFunc(v.EBSRequests.Requests, func(r viperblocklegacyv1.EBSRequest) bool { return r.Name == volumeID })
+	if i < 0 || v.EBSRequests.Requests[i].EFI {
+		return false
+	}
+	v.EBSRequests.Requests = slices.Delete(v.EBSRequests.Requests, i, i+1)
+	if v.Instance != nil {
+		v.Instance.BlockDeviceMappings = slices.DeleteFunc(v.Instance.BlockDeviceMappings, func(m *ec2.InstanceBlockDeviceMapping) bool {
+			return m != nil && m.Ebs != nil && aws.StringValue(m.Ebs.VolumeId) == volumeID
+		})
+	}
+	return true
+}
+
+// stoppedAttachDevice picks the device a volume takes on a stopped instance and
+// reports whether it is the root. An empty request takes the next free
+// /dev/sd[f-p]; a named device must be free, and the root device name is free
+// only while no boot volume is attached.
+func stoppedAttachDevice(v *vm.VM, requested string) (string, bool, error) {
+	root := ebsmetadata.RootDeviceName
+	if v.Instance != nil && aws.StringValue(v.Instance.RootDeviceName) != "" {
+		root = aws.StringValue(v.Instance.RootDeviceName)
+	}
+
+	v.EBSRequests.Mu.Lock()
+	defer v.EBSRequests.Mu.Unlock()
+	used := make(map[string]bool, len(v.EBSRequests.Requests))
+	hasBoot := false
+	for _, r := range v.EBSRequests.Requests {
+		if r.DeviceName != "" {
+			used[r.DeviceName] = true
+		}
+		hasBoot = hasBoot || r.Boot
+	}
+	if v.Instance != nil {
+		for _, m := range v.Instance.BlockDeviceMappings {
+			if m != nil && m.DeviceName != nil {
+				used[*m.DeviceName] = true
+			}
+		}
+	}
+
+	if requested == "" {
+		for c := 'f'; c <= 'p'; c++ {
+			if dev := fmt.Sprintf("/dev/sd%c", c); !used[dev] {
+				return dev, false, nil
+			}
+		}
+		return "", false, errors.New(awserrors.ErrorAttachmentLimitExceeded)
+	}
+	if used[requested] {
+		return "", false, errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	boot := requested == root
+	if boot && hasBoot {
+		return "", false, errors.New(awserrors.ErrorInvalidParameterValue)
+	}
+	return requested, boot, nil
+}
+
+// addVolumeToRecord adds the volume to the mount list and the API view of the
+// record. The root device name makes it the boot volume.
+func addVolumeToRecord(v *vm.VM, volumeID, device string, boot bool) {
+	v.EBSRequests.Mu.Lock()
+	defer v.EBSRequests.Mu.Unlock()
+	v.EBSRequests.Requests = append(v.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{
+		Name:       volumeID,
+		DeviceName: device,
+		Boot:       boot,
+	})
+	if v.Instance == nil {
+		return
+	}
+	mapping := &ec2.InstanceBlockDeviceMapping{}
+	mapping.SetDeviceName(device)
+	mapping.Ebs = &ec2.EbsInstanceBlockDevice{}
+	mapping.Ebs.SetVolumeId(volumeID)
+	mapping.Ebs.SetAttachTime(time.Now())
+	mapping.Ebs.SetDeleteOnTermination(false)
+	mapping.Ebs.SetStatus("attached")
+	v.Instance.BlockDeviceMappings = append(v.Instance.BlockDeviceMappings, mapping)
+}

spinifex/domains/ec2/instance/stopped_detach_test.go

@@ -0,0 +1,139 @@
+package instance
+
+import (
+	"context"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	vmmock "github.com/mulgadc/spinifex/spinifex/runtime/compute/vm/mock"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func stoppedWithVolumes(owner, id string) *vm.VM {
+	v := &vm.VM{
+		ID: id, AccountID: owner, Status: vm.StateStopped,
+		Instance: &ec2.Instance{
+			InstanceId: aws.String(id),
+			BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{
+				{DeviceName: aws.String("/dev/sda1"), Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-root")}},
+				{DeviceName: aws.String("/dev/sdf"), Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-data")}},
+				{DeviceName: aws.String("/dev/sdg"), Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-other")}},
+			},
+		},
+	}
+	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+		{Name: "vol-root", Boot: true, DeviceName: "/dev/sda1"},
+		{Name: "vol-data", DeviceName: "/dev/sdf"},
+		{Name: "vol-other", DeviceName: "/dev/sdg"},
+	}
+	return v
+}
+
+func stoppedDetachInput(instanceID, volumeID string) *ec2.DetachVolumeInput {
+	return &ec2.DetachVolumeInput{InstanceId: aws.String(instanceID), VolumeId: aws.String(volumeID)}
+}
+
+func volumeNames(v *vm.VM) []string {
+	var out []string
+	for _, r := range v.EBSRequests.Requests {
+		out = append(out, r.Name)
+	}
+	return out
+}
+
+// AWS lets a stopped instance's data volume be detached; the record must stop
+// claiming it so a later start does not re-attach it.
+func TestDetachVolumeFromStopped_RemovesOnlyThatVolume(t *testing.T) {
+	owner := awsidentifiers.GlobalAccountID
+	id := "i-stopped-detach"
+	stored := stoppedWithVolumes(owner, id)
+	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: stored}}
+	svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), stoppedStore: store}
+
+	out, err := svc.DetachVolumeFromStopped(context.Background(), stoppedDetachInput(id, "vol-data"), owner)
+	require.NoError(t, err)
+
+	assert.Equal(t, "/dev/sdf", aws.StringValue(out.Device))
+	assert.Equal(t, id, aws.StringValue(out.InstanceId))
+	assert.Equal(t, []string{"vol-root", "vol-other"}, volumeNames(stored))
+	var mapped []string
+	for _, m := range stored.Instance.BlockDeviceMappings {
+		mapped = append(mapped, aws.StringValue(m.Ebs.VolumeId))
+	}
+	assert.Equal(t, []string{"vol-root", "vol-other"}, mapped)
+	assert.Equal(t, vm.StateStopped, stored.Status, "the instance stays stopped")
+}
+
+func TestDetachVolumeFromStopped_Refusals(t *testing.T) {
+	owner := awsidentifiers.GlobalAccountID
+	tests := []struct {
+		name    string
+		caller  string
+		volume  string
+		stopped bool
+		want    string
+	}{
+		{"volume not on the instance", owner, "vol-elsewhere", true, awserrors.ErrorIncorrectState},
+		{"another account", "000000000099", "vol-data", true, awserrors.ErrorInvalidInstanceIDNotFound},
+		{"no such stopped instance", owner, "vol-data", false, awserrors.ErrorInvalidInstanceIDNotFound},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			id := "i-stopped-refuse"
+			stored := stoppedWithVolumes(owner, id)
+			store := &vmmock.StateStore{Stopped: map[string]*vm.VM{}}
+			if tt.stopped {
+				store.Stopped[id] = stored
+			}
+			svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), stoppedStore: store}
+
+			_, err := svc.DetachVolumeFromStopped(context.Background(), stoppedDetachInput(id, tt.volume), tt.caller)
+
+			require.Error(t, err)
+			assert.True(t, awserrors.IsErrorCode(err, tt.want), "got %v, want %s", err, tt.want)
+			assert.Equal(t, []string{"vol-root", "vol-data", "vol-other"}, volumeNames(stored),
+				"a refused detach changes nothing")
+		})
+	}
+}
+
+// AWS lets the root volume of a stopped instance be detached; the instance then
+// cannot start until a volume is attached at its root device name again.
+func TestDetachVolumeFromStopped_DetachesTheRootVolume(t *testing.T) {
+	owner := awsidentifiers.GlobalAccountID
+	id := "i-stopped-root"
+	stored := stoppedWithVolumes(owner, id)
+	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: stored}}
+	svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), stoppedStore: store}
+
+	out, err := svc.DetachVolumeFromStopped(context.Background(), stoppedDetachInput(id, "vol-root"), owner)
+	require.NoError(t, err)
+
+	assert.Equal(t, "/dev/sda1", aws.StringValue(out.Device))
+	assert.Equal(t, []string{"vol-data", "vol-other"}, volumeNames(stored))
+	assert.Equal(t, vm.StateStopped, stored.Status)
+}
+
+// The EFI variable store is Spinifex-internal and never shown to the customer,
+// so it cannot be detached even by its name.
+func TestDetachVolumeFromStopped_RefusesTheInternalEFIStore(t *testing.T) {
+	owner := awsidentifiers.GlobalAccountID
+	id := "i-stopped-efi"
+	stored := stoppedWithVolumes(owner, id)
+	stored.EBSRequests.Requests = append(stored.EBSRequests.Requests,
+		viperblocklegacyv1.EBSRequest{Name: "vol-efi", EFI: true})
+	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: stored}}
+	svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), stoppedStore: store}
+
+	_, err := svc.DetachVolumeFromStopped(context.Background(), stoppedDetachInput(id, "vol-efi"), owner)
+
+	require.Error(t, err)
+	assert.True(t, awserrors.IsErrorCode(err, awserrors.ErrorOperationNotPermitted), "got %v", err)
+	assert.Equal(t, []string{"vol-root", "vol-data", "vol-other", "vol-efi"}, volumeNames(stored))
+}

spinifex/domains/ec2/instance/uefistore/uefistore.go

@@ -0,0 +1,185 @@
+// Package uefistore holds the types that describe an instance's UEFI variable
+// store and the bucket of store records that carry its deletion and write-back
+// obligations past the life of the instance record.
+package uefistore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
+	"github.com/nats-io/nats.go/jetstream"
+)
+
+const (
+	// BucketName holds one store record per instance, keyed by instance ID.
+	BucketName = "spinifex-uefi-stores"
+	// BucketVersion is the bucket's schema stamp.
+	BucketVersion = 1
+	// RecordSchemaVersion is the version every record is written at.
+	RecordSchemaVersion uint16 = 1
+
+	// StateLive is a store whose objects belong to a running or stopped instance.
+	StateLive = "live"
+	// StateDeleting is a store whose objects are to be deleted, and then the record.
+	StateDeleting = "deleting"
+
+	objectRoot = "uefi/"
+)
+
+// ErrInvalidRecord is returned for a store record that breaks its invariants,
+// whether it is about to be written or was just read.
+var ErrInvalidRecord = errors.New("uefistore: invalid store record")
+
+// FirmwareIdentity names the firmware a variable store was written under. A
+// store only boots where Arch and VarsSize match the node's firmware, because
+// pflash needs a byte-exact region; CodeSHA256 is recorded, not enforced.
+type FirmwareIdentity struct {
+	Arch       string `json:"arch"`
+	VarsSize   int64  `json:"vars_size"`
+	CodeSHA256 string `json:"code_sha256"`
+}
+
+// UEFIStoreRecord is the durable owner of one instance's variable store
+// objects. It outlives the instance record, so an obligation recorded here is
+// not lost when a terminated record expires.
+type UEFIStoreRecord struct {
+	SchemaVersion uint16         `json:"schema_version"`
+	InstanceID    string         `json:"instance_id"`
+	AccountID     string         `json:"account_id"`
+	State         string         `json:"state"`
+	WriteBack     *UEFIWriteBack `json:"write_back,omitempty"`
+}
+
+// UEFIWriteBack is a write-back that failed and is still owed. Key is the
+// object every retry puts to, so retries leave no extra objects; SeededFrom is
+// the pointer Node seeded from, which a retry must still find on the record.
+type UEFIWriteBack struct {
+	Node       string    `json:"node"`
+	Key        string    `json:"key"`
+	SeededFrom string    `json:"seeded_from"`
+	Since      time.Time `json:"since"`
+}
+
+// KeyPrefix is the object-store prefix every store object of ownerID (an
+// instance or image ID) is written under.
+func KeyPrefix(ownerID string) string {
+	return objectRoot + ownerID + "/"
+}
+
+// ownsKey reports whether key names one object directly under ownerID's prefix.
+func ownsKey(ownerID, key string) bool {
+	name, ok := strings.CutPrefix(key, KeyPrefix(ownerID))
+	return ok && name != "" && !strings.Contains(name, "/")
+}
+
+// validate checks rec against key. A write-back key outside the instance's own
+// prefix is refused because a superseded obligation deletes that key.
+func (rec *UEFIStoreRecord) validate(key string) error {
+	switch {
+	case rec.SchemaVersion > RecordSchemaVersion:
+		return fmt.Errorf("%w: %s is at schema_version %d, newer than this node understands (%d)",
+			ErrInvalidRecord, key, rec.SchemaVersion, RecordSchemaVersion)
+	case rec.SchemaVersion != RecordSchemaVersion:
+		return fmt.Errorf("%w: %s has schema_version %d", ErrInvalidRecord, key, rec.SchemaVersion)
+	case !strings.HasPrefix(rec.InstanceID, "i-"):
+		return fmt.Errorf("%w: instance_id %q is not an instance ID", ErrInvalidRecord, rec.InstanceID)
+	case rec.InstanceID != key:
+		return fmt.Errorf("%w: instance_id %s stored at key %s", ErrInvalidRecord, rec.InstanceID, key)
+	case rec.State != StateLive && rec.State != StateDeleting:
+		return fmt.Errorf("%w: %s has state %q", ErrInvalidRecord, key, rec.State)
+	}
+	if wb := rec.WriteBack; wb != nil {
+		switch {
+		case wb.Node == "":
+			return fmt.Errorf("%w: %s write-back names no node", ErrInvalidRecord, key)
+		case wb.Since.IsZero():
+			return fmt.Errorf("%w: %s write-back has no since", ErrInvalidRecord, key)
+		case !ownsKey(rec.InstanceID, wb.Key):
+			return fmt.Errorf("%w: %s write-back key %q is not under %s", ErrInvalidRecord, key, wb.Key, KeyPrefix(rec.InstanceID))
+		case !ownsKey(rec.InstanceID, wb.SeededFrom):
+			return fmt.Errorf("%w: %s write-back seeded_from %q is not under %s", ErrInvalidRecord, key, wb.SeededFrom, KeyPrefix(rec.InstanceID))
+		}
+	}
+	return nil
+}
+
+func bucketConfig() kvstore.Config {
+	return kvstore.Config{
+		Name:        BucketName,
+		Description: "Spinifex UEFI variable store records (deletion and write-back obligations)",
+		History:     1,
+		// Left false: a lost bucket has lost obligations, and an empty
+		// recreated one would report every store as having none.
+		RecreateIfMissing: false,
+		OnOpen: func(ctx context.Context, kv jetstream.KeyValue) error {
+			return migrate.DefaultRegistry.RunKV(ctx, BucketName, kv, BucketVersion)
+		},
+		Missing: "UEFI store KV bucket not initialized",
+	}
+}
+
+// Store reads and writes store records. Every change is guarded by the
+// revision the caller read, so a lost race is reported rather than retried.
+type Store struct {
+	kv *kvstore.Store[UEFIStoreRecord]
+}
+
+// NewStore returns a Store over js. The bucket is created on Open or first use.
+func NewStore(js jetstream.JetStream) *Store {
+	return &Store{kv: kvstore.New[UEFIStoreRecord](js, bucketConfig())}
+}
+
+// Open creates the bucket if it is missing and stamps its schema version.
+func (s *Store) Open(ctx context.Context) error {
+	_, err := s.kv.KV(ctx)
+	return err
+}
+
+// Get returns instanceID's record and the revision it was read at. A missing
+// record is kvstore.ErrNotFound; one that breaks its invariants is
+// ErrInvalidRecord, so a newer or corrupt record is never acted on.
+func (s *Store) Get(ctx context.Context, instanceID string) (*UEFIStoreRecord, uint64, error) {
+	rec, rev, err := s.kv.Get(ctx, instanceID)
+	if err != nil {
+		return nil, 0, err
+	}
+	if err := rec.validate(instanceID); err != nil {
+		return nil, 0, err
+	}
+	return rec, rev, nil
+}
+
+// Create writes a new record, which must be live: a store is recorded before
+// its first object is written. An existing record is kvstore.ErrExists. Every
+// write stamps rec with RecordSchemaVersion.
+func (s *Store) Create(ctx context.Context, rec *UEFIStoreRecord) (uint64, error) {
+	rec.SchemaVersion = RecordSchemaVersion
+	if rec.State != StateLive {
+		return 0, fmt.Errorf("%w: %s created in state %q, not %q", ErrInvalidRecord, rec.InstanceID, rec.State, StateLive)
+	}
+	if err := rec.validate(rec.InstanceID); err != nil {
+		return 0, err
+	}
+	return s.kv.Create(ctx, rec.InstanceID, rec)
+}
+
+// CompareAndSet replaces the record only if it is still at rev, returning
+// kvstore.ErrConflict when it is not, and the revision written otherwise.
+func (s *Store) CompareAndSet(ctx context.Context, rec *UEFIStoreRecord, rev uint64) (uint64, error) {
+	rec.SchemaVersion = RecordSchemaVersion
+	if err := rec.validate(rec.InstanceID); err != nil {
+		return 0, err
+	}
+	return s.kv.CompareAndSet(ctx, rec.InstanceID, rec, rev)
+}
+
+// CompareAndDelete removes instanceID's record only if it is still at rev,
+// returning kvstore.ErrConflict when it is not.
+func (s *Store) CompareAndDelete(ctx context.Context, instanceID string, rev uint64) error {
+	return s.kv.CompareAndDelete(ctx, instanceID, rev)
+}

spinifex/domains/ec2/instance/uefistore/uefistore_test.go

@@ -0,0 +1,285 @@
+package uefistore_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/nats-io/nats.go/jetstream"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const instanceID = "i-0123456789abcdef0"
+
+func openStore(t *testing.T) (*uefistore.Store, jetstream.JetStream) {
+	t.Helper()
+	_, _, js := testutil.StartTestJetStream(t)
+	s := uefistore.NewStore(js)
+	require.NoError(t, s.Open(t.Context()))
+	return s, js
+}
+
+func liveRecord() *uefistore.UEFIStoreRecord {
+	return &uefistore.UEFIStoreRecord{InstanceID: instanceID, AccountID: "111122223333", State: uefistore.StateLive}
+}
+
+func writeBack() *uefistore.UEFIWriteBack {
+	return &uefistore.UEFIWriteBack{
+		Node:       "node-2",
+		Key:        uefistore.KeyPrefix(instanceID) + "b",
+		SeededFrom: uefistore.KeyPrefix(instanceID) + "a",
+		Since:      time.Date(2026, 11, 3, 1, 2, 3, 0, time.UTC),
+	}
+}
+
+func rawPut(t *testing.T, js jetstream.JetStream, key, body string) {
+	t.Helper()
+	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
+	require.NoError(t, err)
+	_, err = kv.Put(t.Context(), key, []byte(body))
+	require.NoError(t, err)
+}
+
+// The records carry obligations past the 1 h terminated-record TTL, so the
+// bucket must not expire them, and it is stamped so a newer schema is refused.
+func TestOpen_CreatesBucketWithNoTTLAndStampsVersion(t *testing.T) {
+	_, js := openStore(t)
+
+	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
+	require.NoError(t, err)
+	status, err := kv.Status(t.Context())
+	require.NoError(t, err)
+	assert.Equal(t, int64(1), status.History())
+	assert.Zero(t, status.TTL(), "a TTL would expire deletion obligations before they are settled")
+
+	v, err := kvutil.ReadVersion(t.Context(), kv)
+	require.NoError(t, err)
+	assert.Equal(t, uefistore.BucketVersion, v)
+}
+
+func TestCreate_RoundTripsAndStampsSchemaVersion(t *testing.T) {
+	s, _ := openStore(t)
+	rec := liveRecord()
+	rec.WriteBack = writeBack()
+
+	rev, err := s.Create(t.Context(), rec)
+	require.NoError(t, err)
+
+	got, gotRev, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, rev, gotRev)
+	assert.Equal(t, uefistore.RecordSchemaVersion, got.SchemaVersion)
+	assert.Equal(t, rec, got)
+}
+
+// The wire names are what every release reads, so they are pinned.
+func TestRecord_WireShape(t *testing.T) {
+	s, js := openStore(t)
+	rec := liveRecord()
+	rec.WriteBack = writeBack()
+	_, err := s.Create(t.Context(), rec)
+	require.NoError(t, err)
+
+	kv, err := js.KeyValue(t.Context(), uefistore.BucketName)
+	require.NoError(t, err)
+	entry, err := kv.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.JSONEq(t, `{
+		"schema_version": 1,
+		"instance_id": "i-0123456789abcdef0",
+		"account_id": "111122223333",
+		"state": "live",
+		"write_back": {
+			"node": "node-2",
+			"key": "uefi/i-0123456789abcdef0/b",
+			"seeded_from": "uefi/i-0123456789abcdef0/a",
+			"since": "2026-11-03T01:02:03Z"
+		}
+	}`, string(entry.Value()))
+}
+
+func TestGet_MissingIsNotFound(t *testing.T) {
+	s, _ := openStore(t)
+	_, _, err := s.Get(t.Context(), instanceID)
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+}
+
+func TestCreate_ExistingRecordIsErrExists(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+
+	dup := liveRecord()
+	dup.AccountID = "999999999999"
+	_, err = s.Create(t.Context(), dup)
+	require.ErrorIs(t, err, kvstore.ErrExists)
+
+	got, _, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, "111122223333", got.AccountID, "the second create must not overwrite the first")
+}
+
+// A store is recorded live before its first object, so a record that starts
+// out deleting has no object it could be describing.
+func TestCreate_RefusesDeleting(t *testing.T) {
+	s, _ := openStore(t)
+	rec := liveRecord()
+	rec.State = uefistore.StateDeleting
+
+	_, err := s.Create(t.Context(), rec)
+	require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
+	_, _, err = s.Get(t.Context(), instanceID)
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+}
+
+func TestCompareAndSet_StaleRevisionConflictsAndWritesNothing(t *testing.T) {
+	s, _ := openStore(t)
+	first, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+
+	moved := liveRecord()
+	moved.WriteBack = writeBack()
+	second, err := s.CompareAndSet(t.Context(), moved, first)
+	require.NoError(t, err)
+	assert.Greater(t, second, first)
+
+	stale := liveRecord()
+	stale.State = uefistore.StateDeleting
+	_, err = s.CompareAndSet(t.Context(), stale, first)
+	require.ErrorIs(t, err, kvstore.ErrConflict)
+
+	got, rev, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, second, rev)
+	assert.Equal(t, uefistore.StateLive, got.State)
+	assert.Equal(t, writeBack(), got.WriteBack)
+}
+
+// Two actors that read the same revision race to change it; exactly one may
+// win, so neither can overwrite an obligation the other recorded.
+func TestCompareAndSet_ConcurrentWritersFromOneReadHaveOneWinner(t *testing.T) {
+	s, _ := openStore(t)
+	_, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+	_, rev, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+
+	const writers = 8
+	var (
+		wg   sync.WaitGroup
+		mu   sync.Mutex
+		wins []string
+	)
+	for i := range writers {
+		wg.Go(func() {
+			rec := liveRecord()
+			rec.WriteBack = writeBack()
+			rec.WriteBack.Node = fmt.Sprintf("node-%d", i)
+			if _, err := s.CompareAndSet(context.Background(), rec, rev); err == nil {
+				mu.Lock()
+				wins = append(wins, rec.WriteBack.Node)
+				mu.Unlock()
+			} else {
+				assert.ErrorIs(t, err, kvstore.ErrConflict)
+			}
+		})
+	}
+	wg.Wait()
+
+	require.Len(t, wins, 1)
+	got, _, err := s.Get(t.Context(), instanceID)
+	require.NoError(t, err)
+	assert.Equal(t, wins[0], got.WriteBack.Node)
+}
+
+func TestCompareAndDelete_StaleRevisionKeepsRecord(t *testing.T) {
+	s, _ := openStore(t)
+	first, err := s.Create(t.Context(), liveRecord())
+	require.NoError(t, err)
+	deleting := liveRecord()
+	deleting.State = uefistore.StateDeleting
+	second, err := s.CompareAndSet(t.Context(), deleting, first)
+	require.NoError(t, err)
+
+	require.ErrorIs(t, s.CompareAndDelete(t.Context(), instanceID, first), kvstore.ErrConflict)
+	_, _, err = s.Get(t.Context(), instanceID)
+	require.NoError(t, err, "a delete guarded on a stale revision must leave the record")
+
+	require.NoError(t, s.CompareAndDelete(t.Context(), instanceID, second))
+	_, _, err = s.Get(t.Context(), instanceID)
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+}
+
+// A superseded obligation deletes its recorded key, so a key outside the
+// instance's own prefix must never reach the bucket.
+func TestCompareAndSet_RefusesInvalidRecords(t *testing.T) {
+	cases := map[string]func(*uefistore.UEFIStoreRecord){
+		"unknown state":            func(r *uefistore.UEFIStoreRecord) { r.State = "gone" },
+		"not an instance ID":       func(r *uefistore.UEFIStoreRecord) { r.InstanceID = "ami-0123456789abcdef0" },
+		"write-back with no node":  func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Node = "" },
+		"write-back with no since": func(r *uefistore.UEFIStoreRecord) { r.WriteBack.Since = time.Time{} },
+		"write-back key under another instance": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix("i-0fedcba9876543210") + "b"
+		},
+		"write-back key under an image": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix("ami-0123456789abcdef0") + "b"
+		},
+		"write-back key is the prefix itself": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix(instanceID)
+		},
+		"write-back key nested below the prefix": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.Key = uefistore.KeyPrefix(instanceID) + "x/y"
+		},
+		"write-back seeded from another instance": func(r *uefistore.UEFIStoreRecord) {
+			r.WriteBack.SeededFrom = uefistore.KeyPrefix("i-0fedcba9876543210") + "a"
+		},
+	}
+	for name, mutate := range cases {
+		t.Run(name, func(t *testing.T) {
+			s, _ := openStore(t)
+			rev, err := s.Create(t.Context(), liveRecord())
+			require.NoError(t, err)
+
+			rec := liveRecord()
+			rec.WriteBack = writeBack()
+			mutate(rec)
+			_, err = s.CompareAndSet(t.Context(), rec, rev)
+			require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
+
+			got, gotRev, err := s.Get(t.Context(), instanceID)
+			require.NoError(t, err)
+			assert.Equal(t, rev, gotRev, "an invalid record must not be written")
+			assert.Nil(t, got.WriteBack)
+		})
+	}
+}
+
+// A record a newer release wrote, or one that does not describe its own key,
+// is reported rather than decoded into something a reaper would act on.
+func TestGet_RefusesRecordsItCannotTrust(t *testing.T) {
+	cases := map[string]string{
+		"newer schema":       `{"schema_version":2,"instance_id":"i-0123456789abcdef0","state":"live"}`,
+		"no schema":          `{"instance_id":"i-0123456789abcdef0","state":"live"}`,
+		"another instance":   `{"schema_version":1,"instance_id":"i-0fedcba9876543210","state":"deleting"}`,
+		"unknown state":      `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"purged"}`,
+		"foreign write-back": `{"schema_version":1,"instance_id":"i-0123456789abcdef0","state":"live","write_back":{"node":"n","key":"uefi/i-0fedcba9876543210/b","seeded_from":"uefi/i-0123456789abcdef0/a","since":"2026-11-03T00:00:00Z"}}`,
+	}
+	for name, body := range cases {
+		t.Run(name, func(t *testing.T) {
+			s, js := openStore(t)
+			require.True(t, json.Valid([]byte(body)))
+			rawPut(t, js, instanceID, body)
+
+			_, _, err := s.Get(t.Context(), instanceID)
+			require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
+		})
+	}
+}

spinifex/domains/ec2/volume/service_impl_test.go

@@ -1725,6 +1725,28 @@ func TestDeleteVolume_VolumeAttachedButAvailable(t *testing.T) {
 	assert.Equal(t, awserrors.ErrorVolumeInUse, err.Error())
 }
 
+// A stopped instance keeps its volumes in use, so deleting one is refused
+// until it is detached; AWS answers the same.
+func TestDeleteVolume_VolumeHeldByStoppedInstanceIsRefused(t *testing.T) {
+	kv := setupTestVolumeKV(t)
+	store := objectstore.NewMemoryObjectStore()
+	svc := newTestVolumeServiceWithStore("ap-southeast-2a", store)
+	svc.snapshotKV = kv
+
+	createVolumeInStoreWithMeta(t, svc, store, "vol-held", ebsmetadata.Volume{
+		VolumeID:         "vol-held",
+		CapacityGiB:      10,
+		State:            "in-use",
+		AttachedInstance: "i-stopped",
+	})
+
+	_, err := svc.DeleteVolume(context.Background(), &ec2.DeleteVolumeInput{
+		VolumeId: aws.String("vol-held"),
+	}, testVolAccountID)
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorVolumeInUse, err.Error())
+}
+
 func TestDeleteVolume_EmptyStateUnattachedDeletable(t *testing.T) {
 	kv := setupTestVolumeKV(t)
 	store := objectstore.NewMemoryObjectStore()

spinifex/operator/upgrade/VERSIONS.md

@@ -23,6 +23,7 @@ Single source of truth for the current schema version of every config file Spini
 | `spinifex-cluster-state` | `1` | `daemon.ClusterStateBucketVersion` |
 | `spinifex-cluster-config` | `1` | `daemon.ClusterConfigBucketVersion` |
 | `spinifex-terminated-instances` | `3` | `daemon.TerminatedInstanceBucketVersion` |
+| `spinifex-uefi-stores` | `1` | `uefistore.BucketVersion` |
 
 ## Registered migrations
 

spinifex/providers/ebs/conformance/conformance.go

@@ -150,6 +150,11 @@ type SuiteConfig struct {
 	// a node other than NodeID: what a request meets when the volume's owner
 	// does not answer. Nil skips the cases that need it.
 	QueueOnOtherNode func(t *testing.T) ebsprovider.EBSProvider
+
+	// RevokeExport makes the provider revoke a live export as it does when the
+	// volume is lost to another node, and returns the FenceNotice it sent; see
+	// CaptureFenceNotice. Nil skips the case, for a provider that never revokes.
+	RevokeExport func(t *testing.T, published *ebsprovider.PublishedVolume) ebsprovider.FenceNotice
 }
 
 func (c SuiteConfig) node() string {
@@ -988,6 +993,24 @@ func RunSuiteWithConfig(t *testing.T, newRawProvider func(t *testing.T) ebsprovi
 			nbdInfoRetry(t, dialURI(t, publishedNBDURI(t, second)))
 		})
 
+		t.Run("revoking a live export sends a fence notice to its node", func(t *testing.T) {
+			if cfg.RevokeExport == nil {
+				t.Skip("this provider never revokes a live export")
+			}
+			provider := newProvider(t)
+			ctx := context.Background()
+			volumeID := cfg.id("vol-pub-revoked")
+			_, err := provider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30}})
+			require.NoError(t, err)
+			published, err := provider.PublishVolume(ctx, ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: cfg.node()})
+			require.NoError(t, err)
+
+			notice := cfg.RevokeExport(t, published)
+			assert.Equal(t, volumeID, notice.VolumeID, "the daemon stops the guest on the volume the notice names")
+			assert.Equal(t, published.NodeID, notice.NodeID, "the notice must name the node whose export was revoked")
+			assert.True(t, ebsprovider.SupportsSchemaVersion(notice.SchemaVersion), "a daemon refuses a notice in a version it cannot read")
+		})
+
 		t.Run("not_found on absent volume", func(t *testing.T) {
 			provider := newProvider(t)
 			_, err := provider.PublishVolume(context.Background(), ebsprovider.PublishVolumeRequest{Versioned: ebsprovider.NewVersioned(), VolumeID: cfg.id("vol-never-existed"), NodeID: cfg.node()})

spinifex/providers/ebs/conformance/conformance_test.go

@@ -75,6 +75,28 @@ func TestMemoryProviderNamePrefixConformance(t *testing.T) {
 	}
 }
 
+// TestMemoryProviderFenceNoticeConformance runs the suite with the revocation
+// case enabled, so the contract's fence notice is held to the same check here
+// as on viperblockd.
+func TestMemoryProviderFenceNoticeConformance(t *testing.T) {
+	_, conn := testutil.StartTestNATS(t)
+	shared := ebsprovider.NewMemoryProvider(conformance.ReferenceCapabilities)
+	shared.NotifyFencesOn(conn)
+
+	conformance.RunSuiteWithConfig(t, func(t *testing.T) ebsprovider.EBSProvider {
+		t.Helper()
+		return shared
+	}, conformance.SuiteConfig{
+		RevokeExport: func(t *testing.T, published *ebsprovider.PublishedVolume) ebsprovider.FenceNotice {
+			t.Helper()
+			return conformance.CaptureFenceNotice(t, conn, published.NodeID, func() error {
+				_, err := shared.RevokePublication(t.Context(), published.VolumeID, "node-elsewhere")
+				return err
+			})
+		},
+	})
+}
+
 // TestNATSProviderConformance runs the suite against a NATSProvider backed
 // by natsserve.Serve, the production-shaped neutral server, rather than a
 // test-only twin: this exercises the transport this package's implementors actually run in production, not a hand-written stand-in for it.

spinifex/providers/ebs/conformance/fence_notice.go

@@ -0,0 +1,60 @@
+package conformance
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/require"
+)
+
+// fenceNoticeWait bounds how long CaptureFenceNotice waits for the notice.
+const fenceNoticeWait = 10 * time.Second
+
+// CaptureFenceNotice answers nodeID's fence notices on conn as a daemon with no
+// guest on the volume does, starts revoke, and returns the first notice heard.
+// revoke runs on its own goroutine, since a sender may retry until answered.
+func CaptureFenceNotice(t *testing.T, conn *nats.Conn, nodeID string, revoke func() error) ebsprovider.FenceNotice {
+	t.Helper()
+	subject, err := ebsprovider.FenceNoticeSubject(nodeID)
+	require.NoError(t, err)
+
+	heard := make(chan ebsprovider.FenceNotice, 1)
+	sub, err := conn.Subscribe(subject, func(msg *nats.Msg) {
+		var notice ebsprovider.FenceNotice
+		ack := ebsprovider.FenceNoticeAck{Versioned: ebsprovider.NewVersioned()}
+		if err := json.Unmarshal(msg.Data, &notice); err != nil {
+			ack.Error = &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeInvalidArgument, Message: err.Error()}
+		} else {
+			ack.Versioned = ebsprovider.ReplyVersioned(notice.SchemaVersion)
+			select {
+			case heard <- notice:
+			default:
+			}
+		}
+		if payload, err := json.Marshal(ack); err == nil {
+			_ = msg.Respond(payload)
+		}
+	})
+	require.NoError(t, err)
+	defer func() { _ = sub.Unsubscribe() }()
+	require.NoError(t, conn.Flush())
+
+	revoked := make(chan error, 1)
+	go func() { revoked <- revoke() }()
+
+	timeout := time.After(fenceNoticeWait)
+	for {
+		select {
+		case notice := <-heard:
+			return notice
+		case err := <-revoked:
+			require.NoError(t, err, "revoking the export")
+			revoked = nil
+		case <-timeout:
+			t.Fatalf("no fence notice on %s within %s of revoking the export; the daemon would leave its guest running on a revoked disk", subject, fenceNoticeWait)
+		}
+	}
+}

spinifex/providers/ebs/fence_notice.go

@@ -0,0 +1,53 @@
+package ebsprovider
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/nats-io/nats.go"
+)
+
+// NotifyFenced sends notice once and returns the instance the daemon stopped.
+// An ErrInvalidArgument error is terminal: the notice can never be sent. A
+// *ProviderError (internal for an undecodable ack) was heard; others were not.
+func NotifyFenced(ctx context.Context, conn *nats.Conn, notice FenceNotice, timeout time.Duration) (string, error) {
+	if notice.VolumeID == "" {
+		return "", fmt.Errorf("%w: fence notice names no volume", ErrInvalidArgument)
+	}
+	subject, err := FenceNoticeSubject(notice.NodeID)
+	if err != nil {
+		return "", err
+	}
+	if conn == nil {
+		return "", fmt.Errorf("%w: no connection to send the fence notice on", ErrInvalidArgument)
+	}
+
+	reply, err := requestReply(ctx, conn, subject, timeout, notice)
+	if err != nil {
+		return "", err
+	}
+	var ack FenceNoticeAck
+	if err := json.Unmarshal(reply, &ack); err != nil {
+		// The daemon answered, so repeating the notice would only repeat this.
+		return "", &ProviderError{Code: ErrorCodeInternal, Message: fmt.Sprintf("decode %s ack: %v", subject, err)}
+	}
+	if ack.Error != nil {
+		return ack.InstanceID, ack.Error
+	}
+	return ack.InstanceID, nil
+}
+
+// FenceNoticeRetryable reports whether a NotifyFenced error leaves the notice
+// to be sent again: it was not heard, or the daemon refused it as unavailable.
+func FenceNoticeRetryable(err error) bool {
+	if err == nil || errors.Is(err, ErrInvalidArgument) {
+		return false
+	}
+	if _, refused := errors.AsType[*ProviderError](err); refused {
+		return errors.Is(err, ErrUnavailable)
+	}
+	return true
+}

spinifex/providers/ebs/fence_notice_test.go

@@ -0,0 +1,205 @@
+package ebsprovider
+
+import (
+	"context"
+	"encoding/json"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// answerFenceNotices replies to node-a's fence notices with reply.
+func answerFenceNotices(t *testing.T, conn *nats.Conn, reply []byte) {
+	t.Helper()
+	subject, err := FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	sub, err := conn.Subscribe(subject, func(msg *nats.Msg) { _ = msg.Respond(reply) })
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	require.NoError(t, conn.Flush())
+}
+
+func fenceNoticeFor(volumeID string) FenceNotice {
+	return FenceNotice{Versioned: NewVersioned(), VolumeID: volumeID, NodeID: "node-a", Winner: "node-b"}
+}
+
+// TestNotifyFenced_SeparatesHeardFromUnheard pins what a sender's retry turns
+// on: an answer of any kind ends the notice, and only silence repeats it.
+func TestNotifyFenced_SeparatesHeardFromUnheard(t *testing.T) {
+	t.Run("acknowledged", func(t *testing.T) {
+		_, conn := testutil.StartTestNATS(t)
+		ack, err := json.Marshal(FenceNoticeAck{Versioned: NewVersioned(), InstanceID: "i-stopped"})
+		require.NoError(t, err)
+		answerFenceNotices(t, conn, ack)
+
+		instanceID, err := NotifyFenced(t.Context(), conn, fenceNoticeFor("vol-1"), time.Second)
+		require.NoError(t, err)
+		assert.Equal(t, "i-stopped", instanceID)
+	})
+
+	t.Run("refused", func(t *testing.T) {
+		_, conn := testutil.StartTestNATS(t)
+		ack, err := json.Marshal(FenceNoticeAck{Versioned: NewVersioned(), Error: &ProviderError{Code: ErrorCodeInvalidArgument, Message: "no"}})
+		require.NoError(t, err)
+		answerFenceNotices(t, conn, ack)
+
+		_, err = NotifyFenced(t.Context(), conn, fenceNoticeFor("vol-1"), time.Second)
+		var refused *ProviderError
+		require.ErrorAs(t, err, &refused, "a refusal was heard, and repeating it would only be refused again")
+		assert.ErrorIs(t, err, ErrInvalidArgument)
+		assert.False(t, FenceNoticeRetryable(err))
+	})
+
+	t.Run("refused as unavailable", func(t *testing.T) {
+		_, conn := testutil.StartTestNATS(t)
+		ack, err := json.Marshal(FenceNoticeAck{Versioned: NewVersioned(), Error: &ProviderError{Code: ErrorCodeUnavailable, Message: "later"}})
+		require.NoError(t, err)
+		answerFenceNotices(t, conn, ack)
+
+		_, err = NotifyFenced(t.Context(), conn, fenceNoticeFor("vol-1"), time.Second)
+		require.ErrorIs(t, err, ErrUnavailable)
+		assert.True(t, FenceNoticeRetryable(err), "a daemon that cannot act yet may act on the next notice")
+	})
+
+	t.Run("undecodable answer", func(t *testing.T) {
+		_, conn := testutil.StartTestNATS(t)
+		answerFenceNotices(t, conn, []byte("{"))
+
+		_, err := NotifyFenced(t.Context(), conn, fenceNoticeFor("vol-1"), time.Second)
+		var refused *ProviderError
+		require.ErrorAs(t, err, &refused, "the daemon answered, so the notice was heard")
+		assert.Equal(t, ErrorCodeInternal, refused.Code)
+		assert.False(t, FenceNoticeRetryable(err))
+	})
+
+	t.Run("nobody listening", func(t *testing.T) {
+		_, conn := testutil.StartTestNATS(t)
+		_, err := NotifyFenced(t.Context(), conn, fenceNoticeFor("vol-1"), time.Second)
+		require.Error(t, err)
+		var refused *ProviderError
+		assert.NotErrorAs(t, err, &refused, "an unheard notice must read as unheard, or the sender stops retrying")
+		assert.True(t, FenceNoticeRetryable(err))
+	})
+
+	t.Run("connection closed", func(t *testing.T) {
+		_, conn := testutil.StartTestNATS(t)
+		conn.Close()
+		_, err := NotifyFenced(t.Context(), conn, fenceNoticeFor("vol-1"), time.Second)
+		require.ErrorIs(t, err, nats.ErrConnectionClosed)
+		assert.True(t, FenceNoticeRetryable(err), "a connection that comes back can still carry the notice")
+	})
+}
+
+// TestNotifyFenced_AttributesTheNoticeToItsAccount holds the notice to the
+// contract's request path: the daemon's span names the account it acts for.
+func TestNotifyFenced_AttributesTheNoticeToItsAccount(t *testing.T) {
+	_, conn := testutil.StartTestNATS(t)
+	subject, err := FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	accounts := make(chan string, 1)
+	ack, err := json.Marshal(FenceNoticeAck{Versioned: NewVersioned()})
+	require.NoError(t, err)
+	sub, err := conn.Subscribe(subject, func(msg *nats.Msg) {
+		accounts <- msg.Header.Get(AccountIDHeader)
+		_ = msg.Respond(ack)
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	require.NoError(t, conn.Flush())
+
+	_, err = NotifyFenced(WithAccountID(t.Context(), "123456789012"), conn, fenceNoticeFor("vol-1"), time.Second)
+	require.NoError(t, err)
+	assert.Equal(t, "123456789012", <-accounts)
+}
+
+// TestNotifyFenced_RefusesAnUnaddressableNotice keeps a notice that can never
+// be sent from reading as unheard, which a retrying sender would repeat forever.
+func TestNotifyFenced_RefusesAnUnaddressableNotice(t *testing.T) {
+	_, conn := testutil.StartTestNATS(t)
+	cases := map[string]struct {
+		conn   *nats.Conn
+		notice FenceNotice
+	}{
+		"no volume": {conn, FenceNotice{Versioned: NewVersioned(), NodeID: "node-a"}},
+		"no node":   {conn, FenceNotice{Versioned: NewVersioned(), VolumeID: "vol-1"}},
+		"bad node":  {conn, FenceNotice{Versioned: NewVersioned(), VolumeID: "vol-1", NodeID: "node.a"}},
+		"no conn":   {nil, fenceNoticeFor("vol-1")},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			_, err := NotifyFenced(context.Background(), tc.conn, tc.notice, time.Second)
+			require.ErrorIs(t, err, ErrInvalidArgument)
+			assert.False(t, FenceNoticeRetryable(err), "a notice that can never be sent must not be retried")
+		})
+	}
+}
+
+// TestMemoryProviderRevokePublication_RetriesAnUnheardNotice keeps the memory
+// provider to the fence's delivery rule: a notice nobody answered is sent
+// again until one does, not dropped after the first try.
+func TestMemoryProviderRevokePublication_RetriesAnUnheardNotice(t *testing.T) {
+	_, conn := testutil.StartTestNATS(t)
+	ctx := t.Context()
+	provider := NewMemoryProvider(Capabilities{})
+	provider.fenceNoticeTimeout = 100 * time.Millisecond
+	provider.fenceNoticeEvery = 10 * time.Millisecond
+	provider.NotifyFencesOn(conn)
+	_, err := provider.CreateVolume(ctx, CreateVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1", CapacityRange: CapacityRange{RequiredBytes: 1 << 30}})
+	require.NoError(t, err)
+	_, err = provider.PublishVolume(ctx, PublishVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1", NodeID: "node-a"})
+	require.NoError(t, err)
+
+	subject, err := FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	ack, err := json.Marshal(FenceNoticeAck{Versioned: NewVersioned(), InstanceID: "i-stopped"})
+	require.NoError(t, err)
+	var heard atomic.Int32
+	sub, err := conn.Subscribe(subject, func(msg *nats.Msg) {
+		if heard.Add(1) == 1 {
+			return // the first notice is lost, as on a daemon still reconnecting
+		}
+		_ = msg.Respond(ack)
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = sub.Unsubscribe() })
+	require.NoError(t, conn.Flush())
+
+	stopped, err := provider.RevokePublication(ctx, "vol-1", "node-b")
+	require.NoError(t, err, "a lost first notice must be sent again")
+	assert.Equal(t, "i-stopped", stopped)
+	assert.Equal(t, int32(2), heard.Load())
+}
+
+func TestMemoryProviderRevokePublication(t *testing.T) {
+	_, conn := testutil.StartTestNATS(t)
+	ctx := context.Background()
+	provider := NewMemoryProvider(Capabilities{})
+	_, err := provider.CreateVolume(ctx, CreateVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1", CapacityRange: CapacityRange{RequiredBytes: 1 << 30}})
+	require.NoError(t, err)
+	_, err = provider.PublishVolume(ctx, PublishVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1", NodeID: "node-a"})
+	require.NoError(t, err)
+
+	_, err = provider.RevokePublication(ctx, "vol-1", "node-b")
+	require.ErrorIs(t, err, ErrUnavailable, "with nowhere to announce it, a revocation would leave the guest unstopped")
+	stillPublished, err := provider.PublishVolume(ctx, PublishVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1", NodeID: "node-b"})
+	require.ErrorIs(t, err, ErrVolumeInUse, "a refused revocation must leave the publication in place")
+	assert.Nil(t, stillPublished)
+
+	ack, err := json.Marshal(FenceNoticeAck{Versioned: NewVersioned()})
+	require.NoError(t, err)
+	answerFenceNotices(t, conn, ack)
+	provider.NotifyFencesOn(conn)
+
+	_, err = provider.RevokePublication(ctx, "vol-1", "node-b")
+	require.NoError(t, err)
+	_, err = provider.PublishVolume(ctx, PublishVolumeRequest{Versioned: NewVersioned(), VolumeID: "vol-1", NodeID: "node-b"})
+	require.NoError(t, err, "a revoked publication no longer holds the volume")
+
+	_, err = provider.RevokePublication(ctx, "vol-never", "node-b")
+	require.ErrorIs(t, err, ErrNotFound)
+}

spinifex/providers/ebs/memory.go

@@ -2,12 +2,15 @@ package ebsprovider
 
 import (
 	"bytes"
+	"cmp"
 	"context"
 	"fmt"
 	"maps"
 	"slices"
 	"sync"
 	"time"
+
+	"github.com/nats-io/nats.go"
 )
 
 // MemoryProvider is a deterministic, concurrency-safe Provider used by
@@ -19,6 +22,10 @@ type MemoryProvider struct {
 	volumes      map[string]*memoryVolume
 	snapshots    map[string]*Snapshot
 	now          func() time.Time
+	fenceConn    *nats.Conn
+	// Zero takes the defaults; tests shorten them.
+	fenceNoticeTimeout time.Duration
+	fenceNoticeEvery   time.Duration
 }
 
 type memoryVolume struct {
@@ -406,13 +413,73 @@ func (m *MemoryProvider) UnpublishVolume(_ context.Context, req UnpublishVolumeR
 	if volume == nil || volume.published == nil || volume.published.NodeID != req.NodeID {
 		return result, nil
 	}
-	volume.published = nil
-	volume.readOnly = false
-	volume.volume.State = VolumeStateAvailable
+	volume.unpublish()
 	result.TornDown = req.SchemaVersion >= NoSealSchemaVersion
 	return result, nil
 }
 
+// NotifyFencesOn sets the connection RevokePublication announces fences on.
+func (m *MemoryProvider) NotifyFencesOn(conn *nats.Conn) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.fenceConn = conn
+}
+
+// unpublish drops the volume's publication. The caller holds the provider lock.
+func (v *memoryVolume) unpublish() {
+	v.published = nil
+	v.readOnly = false
+	v.volume.State = VolumeStateAvailable
+}
+
+// Defaults for RevokePublication's notice: each attempt's wait for an answer,
+// and the pause before an unanswered notice is sent again.
+const (
+	memoryFenceNoticeTimeout = 2 * time.Second
+	memoryFenceNoticeEvery   = time.Second
+)
+
+// RevokePublication tears volumeID's publication down as a provider that lost
+// the volume to winner does, then sends a FenceNotice to the node it was
+// published on until heard or ctx ends. It returns the instance stopped there.
+func (m *MemoryProvider) RevokePublication(ctx context.Context, volumeID, winner string) (string, error) {
+	m.mu.Lock()
+	conn := m.fenceConn
+	volume := m.volumes[volumeID]
+	timeout := cmp.Or(m.fenceNoticeTimeout, memoryFenceNoticeTimeout)
+	every := cmp.Or(m.fenceNoticeEvery, memoryFenceNoticeEvery)
+	if conn == nil {
+		m.mu.Unlock()
+		return "", fmt.Errorf("%w: no connection to announce the fence on", ErrUnavailable)
+	}
+	if volume == nil || volume.published == nil {
+		m.mu.Unlock()
+		return "", fmt.Errorf("%w: volume %s is not published", ErrNotFound, volumeID)
+	}
+	nodeID := volume.published.NodeID
+	volume.unpublish()
+	m.mu.Unlock()
+
+	notice := FenceNotice{
+		Versioned: NewVersioned(),
+		VolumeID:  volumeID,
+		NodeID:    nodeID,
+		Winner:    winner,
+		Reason:    "volume revoked in favour of " + winner,
+	}
+	for {
+		instanceID, err := NotifyFenced(ctx, conn, notice, timeout)
+		if !FenceNoticeRetryable(err) || conn.IsClosed() {
+			return instanceID, err
+		}
+		select {
+		case <-ctx.Done():
+			return "", fmt.Errorf("fence notice for %s never acknowledged: %w: %w", volumeID, ctx.Err(), err)
+		case <-time.After(every):
+		}
+	}
+}
+
 // HoldsUnsealedWrites answers that none are held: every write lands in this
 // process's map, which is the backend, before it is acknowledged.
 func (m *MemoryProvider) HoldsUnsealedWrites(_ context.Context, nodeID string, volumeIDs []string) ([]string, error) {

spinifex/providers/ebs/nats.go

@@ -569,17 +569,33 @@ func (p *NATSProvider) HoldsUnsealedWrites(ctx context.Context, nodeID string, v
 // connected reports whether a request can be sent now. A reconnecting
 // connection would buffer it and send it on reconnect, after its caller may
 // have given up, so the provider would act with nobody waiting on the result.
-func (p *NATSProvider) connected() bool {
-	return p.conn != nil && p.conn.IsConnected()
+func connected(conn *nats.Conn) bool {
+	return conn != nil && conn.IsConnected()
 }
 
+func (p *NATSProvider) connected() bool { return connected(p.conn) }
+
 func (p *NATSProvider) request(ctx context.Context, subject string, input, output any) error {
-	if !p.connected() {
-		return nats.ErrConnectionClosed
+	reply, err := requestReply(ctx, p.conn, subject, p.timeoutFor(subject), input)
+	if err != nil {
+		return err
+	}
+	if err := json.Unmarshal(reply, output); err != nil {
+		return fmt.Errorf("decode %s response: %w: %w", subject, ErrUnreadableReply, err)
+	}
+	return nil
+}
+
+// requestReply sends input on subject as ctx's account, waits up to timeout
+// and returns the raw reply. Its client span records a transport error, an
+// unreadable reply, or the error the reply carries; decoding is the caller's.
+func requestReply(ctx context.Context, conn *nats.Conn, subject string, timeout time.Duration, input any) ([]byte, error) {
+	if !connected(conn) {
+		return nil, nats.ErrConnectionClosed
 	}
 	payload, err := json.Marshal(input)
 	if err != nil {
-		return fmt.Errorf("encode %s request: %w", subject, err)
+		return nil, fmt.Errorf("encode %s request: %w", subject, err)
 	}
 
 	request := &nats.Msg{Subject: subject, Data: payload, Header: nats.Header{}}
@@ -590,18 +606,24 @@ func (p *NATSProvider) request(ctx context.Context, subject string, input, outpu
 	ctx, span := startClientSpan(ctx, subject, accountID, request.Header)
 	defer span.End()
 
-	requestCtx, cancel := context.WithTimeout(ctx, p.timeoutFor(subject))
+	requestCtx, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
-	msg, err := p.conn.RequestMsgWithContext(requestCtx, request)
+	msg, err := conn.RequestMsgWithContext(requestCtx, request)
 	if err != nil {
 		RecordSpanError(span, err)
-		return fmt.Errorf("request %s: %w", subject, err)
-	}
-	if err := json.Unmarshal(msg.Data, output); err != nil {
-		RecordSpanError(span, err)
-		return fmt.Errorf("decode %s response: %w: %w", subject, ErrUnreadableReply, err)
+		return nil, fmt.Errorf("request %s: %w", subject, err)
 	}
 	RecordResponseError(span, msg.Data)
+	return msg.Data, nil
+}
+
+// ValidateNodeID reports whether nodeID can address this node's contract
+// subjects. A node that fails it has no fence-notice subject, so a revoked
+// export there would leave its guest running.
+func ValidateNodeID(nodeID string) error {
+	if err := validateSubjectToken(nodeID); err != nil {
+		return fmt.Errorf("node ID cannot address EBS provider subjects: %w", err)
+	}
 	return nil
 }
 

spinifex/providers/ebs/telemetry.go

@@ -3,6 +3,7 @@ package ebsprovider
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"maps"
 	"slices"
 	"strings"
@@ -218,9 +219,9 @@ func RecordProviderError(span trace.Span, providerErr *ProviderError) {
 	span.SetStatus(codes.Error, providerErr.Message)
 }
 
-// RecordResponseError tags span with the error an encoded response carries.
-// The response types share no accessor for it, so this re-reads the one field
-// rather than making every verb implement one; it runs only when recording.
+// RecordResponseError tags span with the error an encoded response carries, or
+// fails it for a response that is not readable. The response types share no
+// accessor, so this re-reads the one field; it runs only when recording.
 func RecordResponseError(span trace.Span, data []byte) {
 	if !span.IsRecording() {
 		return
@@ -229,6 +230,7 @@ func RecordResponseError(span trace.Span, data []byte) {
 		Error *ProviderError `json:"error"`
 	}
 	if err := json.Unmarshal(data, &envelope); err != nil {
+		RecordSpanError(span, fmt.Errorf("%w: %w", ErrUnreadableReply, err))
 		return
 	}
 	RecordProviderError(span, envelope.Error)

spinifex/providers/ebs/viperblock/conformance_test.go

@@ -75,6 +75,7 @@ type providerHarness struct {
 	nodeID           string
 	otherNodeID      string
 	killExport       func(*testing.T, *ebsprovider.PublishedVolume)
+	revokeExport     func(*testing.T, *ebsprovider.PublishedVolume) ebsprovider.FenceNotice
 	queueOnOtherNode func(t *testing.T) ebsprovider.EBSProvider
 }
 
@@ -86,6 +87,8 @@ type providerHarness struct {
 // would go to a subject nobody serves and time out instead of being refused.
 // Both share the NATS the volume leases live in and the bucket the volumes
 // live in, so a second opener is refused by the lease, not by the harness.
+// The hooks kill a published export's nbdkit behind the handlers' back, and
+// revoke one as a lost lease does.
 func inProcessProvider(t *testing.T) providerHarness {
 	t.Helper()
 	pluginPath := requireExportTools(t)
@@ -123,6 +126,17 @@ func inProcessProvider(t *testing.T) providerHarness {
 	_, err = js.CreateKeyValue(t.Context(), jetstream.KeyValueConfig{Bucket: clusterv1.ClusterStateBucket})
 	require.NoError(t, err)
 
+	holder := func(t *testing.T, published *ebsprovider.PublishedVolume) (*Config, MountedVolume) {
+		t.Helper()
+		for _, node := range []*Config{first, second} {
+			if mv, ok := findMountedVolume(node, published.VolumeID); ok && node.NodeName == published.NodeID {
+				return node, mv
+			}
+		}
+		t.Fatalf("no mounted entry for %s on %s", published.VolumeID, published.NodeID)
+		return nil, MountedVolume{}
+	}
+
 	return providerHarness{
 		newProvider: func(t *testing.T) ebsprovider.EBSProvider {
 			t.Helper()
@@ -130,16 +144,18 @@ func inProcessProvider(t *testing.T) providerHarness {
 		},
 		nodeID:      first.NodeName,
 		otherNodeID: second.NodeName,
-		// Kills a published export's nbdkit behind the handlers' back.
 		killExport: func(t *testing.T, published *ebsprovider.PublishedVolume) {
 			t.Helper()
-			for _, node := range []*Config{first, second} {
-				if mv, ok := findMountedVolume(node, published.VolumeID); ok && node.NodeName == published.NodeID {
-					require.NoError(t, hostprocess.ForceKillProcess(mv.PID, 5*time.Second))
-					return
-				}
-			}
-			t.Fatalf("no mounted entry for %s on %s", published.VolumeID, published.NodeID)
+			_, mv := holder(t, published)
+			require.NoError(t, hostprocess.ForceKillProcess(mv.PID, 5*time.Second))
+		},
+		revokeExport: func(t *testing.T, published *ebsprovider.PublishedVolume) ebsprovider.FenceNotice {
+			t.Helper()
+			node, _ := holder(t, published)
+			return conformance.CaptureFenceNotice(t, nc, published.NodeID, func() error {
+				node.onVolumeLeaseLost(t.Context(), published.VolumeID, leaseLostStalled)
+				return nil
+			})
 		},
 		queueOnOtherNode: func(t *testing.T) ebsprovider.EBSProvider {
 			t.Helper()
@@ -163,6 +179,7 @@ func TestViperblockdConformance(t *testing.T) {
 		NodeID:           h.nodeID,
 		OtherNodeID:      h.otherNodeID,
 		KillExport:       h.killExport,
+		RevokeExport:     h.revokeExport,
 		QueueOnOtherNode: h.queueOnOtherNode,
 	})
 }

spinifex/providers/ebs/viperblock/provider_handlers.go

@@ -53,12 +53,12 @@ var providerObjectStoreFactory = func(cfg *Config) objectstore.ObjectStore {
 // viperblock engine construction out of the EC2 control-plane handlers and
 // into the storage daemon that owns BaseDir and the mounted-volume registry.
 //
-// PublishVolume/UnpublishVolume are node-addressed (ebsprovider.PublishSubject
-// / UnpublishSubject already route to one node), so they are only registered
-// when cfg.NodeName is set; there is no queue-group fallback the way the
-// legacy ebs.mount/ebs.unmount subjects have.
+// It refuses a cfg.NodeName that cannot address the node-addressed subjects.
 // Exported so harnesses can put a real provider behind the control plane.
 func RegisterProviderSubjects(cfg *Config, nc *nats.Conn) error {
+	if err := cfg.validateNodeName(); err != nil {
+		return err
+	}
 	// The lease store has to exist before any subject is served: a handler
 	// that reaches an engine open without one refuses, and refusing every
 	// publish is a worse failure than not starting.
@@ -114,11 +114,6 @@ func RegisterProviderSubjects(cfg *Config, nc *nats.Conn) error {
 		return fmt.Errorf("subscribe to %s: %w", snapshotCreateWildcard, err)
 	}
 
-	if cfg.NodeName == "" {
-		slog.Warn("ebs.provider: NodeName is empty, this node cannot serve PublishVolume/UnpublishVolume attachments")
-		return nil
-	}
-
 	publishSubject, err := ebsprovider.PublishSubject(cfg.NodeName)
 	if err != nil {
 		return fmt.Errorf("build publish subject: %w", err)
@@ -1809,8 +1804,8 @@ func unmountVolume(ctx context.Context, cfg *Config, volumeName string) (viperbl
 
 // handlePublishVolume serves ebs.provider.v1.<node>.mount, the provider-neutral
 // front for the same nbdkit mount path ebs.mount uses. It is node-addressed
-// (RegisterProviderSubjects only subscribes it when cfg.NodeName is set), so
-// unlike the legacy handler there is no queue-group fallback to reason about.
+// (subscribed on cfg.NodeName's subject), so unlike the legacy handler there
+// is no queue-group fallback to reason about.
 func handlePublishVolume(ctx context.Context, cfg *Config, nc *nats.Conn, msg *nats.Msg) {
 	var req ebsprovider.PublishVolumeRequest
 	err := json.Unmarshal(msg.Data, &req)

spinifex/providers/ebs/viperblock/provider_handlers_test.go

@@ -1027,29 +1027,29 @@ func TestProviderHandlers_UnpublishVolume_AbsentIsIdempotent(t *testing.T) {
 	assert.Nil(t, resp.Error, "unpublishing a volume this node never mounted must succeed")
 }
 
-// TestProviderHandlers_PublishUnpublish_RequireNodeName covers
-// RegisterProviderSubjects' NodeName gate: PublishVolume/UnpublishVolume must
-// not be registered when cfg.NodeName is empty (PublishSubject has no node to
-// address), while every other ebs.provider.v1.* subject still is.
-func TestProviderHandlers_PublishUnpublish_RequireNodeName(t *testing.T) {
+// TestProviderHandlers_RefuseAnUnaddressableNodeName keeps viperblockd from
+// serving where the node-addressed subjects, the fence notice among them, have
+// no subject: a fence there would leave its guest running.
+func TestProviderHandlers_RefuseAnUnaddressableNodeName(t *testing.T) {
 	_, natsURL := setupEmbeddedNATS(t)
-	cfgNoNode := setupTestConfig(t, natsURL)
-	cfgNoNode.NodeName = ""
-	ncNoNode := startProviderSubjects(t, cfgNoNode, natsURL)
+	for _, node := range []string{"", "node.a", "node-*"} {
+		t.Run(node, func(t *testing.T) {
+			nc, err := nats.Connect(natsURL)
+			require.NoError(t, err)
+			t.Cleanup(nc.Close)
 
-	_, natsURL2 := setupEmbeddedNATS(t)
-	cfgNamed := setupTestConfig(t, natsURL2)
-	ncNamed := startProviderSubjects(t, cfgNamed, natsURL2)
+			register := setupTestConfig(t, natsURL)
+			register.NodeName = node
+			err = RegisterProviderSubjects(register, nc)
+			require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument)
+			assert.Contains(t, err.Error(), "viperblockd node name")
 
-	assert.Equal(t, ncNamed.NumSubscriptions()-3, ncNoNode.NumSubscriptions(),
-		"an empty NodeName must skip exactly the publish, unpublish and unsealed-writes subscriptions")
-
-	// The other provider subjects must still work without a node name.
-	body := marshalRequest(t, map[string]any{"schema_version": ebsprovider.SchemaVersion})
-	msg := requestProvider(t, ncNoNode, ebsprovider.CapabilitiesSubject, body)
-	var resp ebsprovider.GetCapabilitiesResponse
-	require.NoError(t, json.Unmarshal(msg.Data, &resp))
-	assert.Nil(t, resp.Error)
+			run := setupTestConfig(t, natsURL)
+			run.NodeName = node
+			require.ErrorIs(t, run.Run(nc), ebsprovider.ErrInvalidArgument)
+			assert.Zero(t, nc.NumSubscriptions(), "a refused start must serve nothing")
+		})
+	}
 }
 
 // TestProviderHandlers_CreateVolume_SeedValidation covers the guards that must

spinifex/providers/ebs/viperblock/viperblock.go

@@ -24,6 +24,7 @@ import (
 	"github.com/mulgadc/bluebottle/pkg/masterkey"
 	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
 	"github.com/mulgadc/viperblock/viperblock"
 	"github.com/mulgadc/viperblock/viperblock/backends/s3"
@@ -826,6 +827,9 @@ func (cfg *Config) RecoverMountedVolumes(ctx context.Context, nc *nats.Conn) {
 // readiness: without the round trip a caller can act on the signal and still
 // have its request arrive before the interest does.
 func (cfg *Config) Run(nc *nats.Conn) error {
+	if err := cfg.validateNodeName(); err != nil {
+		return err
+	}
 	if err := cfg.LoadEncryptionKey(); err != nil {
 		return err
 	}
@@ -839,11 +843,7 @@ func (cfg *Config) Run(nc *nats.Conn) error {
 
 	cfg.RecoverMountedVolumes(context.Background(), nc)
 
-	if cfg.NodeName != "" {
-		slog.Info("Waiting for EBS events", "node", cfg.NodeName)
-	} else {
-		slog.Info("Waiting for EBS events (single-node mode)")
-	}
+	slog.Info("Waiting for EBS events", "node", cfg.NodeName)
 
 	if err := RegisterVolumeLifecycleSubjects(cfg, nc); err != nil {
 		return err
@@ -861,6 +861,15 @@ func (cfg *Config) Run(nc *nats.Conn) error {
 	return nil
 }
 
+// validateNodeName refuses a node name the contract cannot address. Without
+// one the fence notice has no subject, and a fenced guest would keep running.
+func (cfg *Config) validateNodeName() error {
+	if err := ebsprovider.ValidateNodeID(cfg.NodeName); err != nil {
+		return fmt.Errorf("viperblockd node name %q (cluster config node): %w", cfg.NodeName, err)
+	}
+	return nil
+}
+
 // Shutdown flushes each mounted volume's WAL and reaps idle nbdkit processes.
 // It snapshots MountedVolumes and clears it while holding the lock, then
 // flushes/kills outside the lock since VB.Close does heavy I/O.

spinifex/providers/ebs/viperblock/viperblockd_integration_test.go

@@ -839,42 +839,6 @@ func TestIntegration_ServiceGracefulShutdown(t *testing.T) {
 	t.Log("Service is running and responsive")
 }
 
-// TestIntegration_GenericTopicRouting tests that NodeName="" uses generic ebs.unmount with queue group.
-func TestIntegration_GenericTopicRouting(t *testing.T) {
-	t.Parallel()
-	if testing.Short() {
-		t.Skip("Skipping integration test in short mode")
-	}
-
-	ns, natsURL := setupEmbeddedNATS(t)
-	defer ns.Shutdown()
-
-	cfg := setupTestConfig(t, natsURL)
-	cfg.NodeName = "" // Single-node / generic mode
-	cfg.MountedVolumes = []MountedVolume{
-		{Name: "vol-generic", PID: deadpid.PID},
-	}
-
-	startTestService(t, cfg)
-
-	nc, err := nats.Connect(natsURL)
-	assert.NoError(t, err)
-	defer nc.Close()
-
-	// Send to generic topic (not ebs.{node}.unmount)
-	request := viperblocklegacyv1.EBSRequest{Name: "vol-generic"}
-	requestData, _ := json.Marshal(request)
-
-	msg, err := nc.Request("ebs.unmount", requestData, 3*time.Second)
-	assert.NoError(t, err)
-
-	var response viperblocklegacyv1.EBSUnMountResponse
-	assert.NoError(t, json.Unmarshal(msg.Data, &response))
-	assert.Equal(t, "vol-generic", response.Volume)
-	assert.False(t, response.Mounted)
-	assert.Empty(t, response.Error)
-}
-
 // TestIntegration_MountAuxiliaryVolumeSuffix tests that volumes with -cloudinit suffix
 // reach the mount handler and trigger the auxiliary volume code path.
 func TestIntegration_MountAuxiliaryVolumeSuffix(t *testing.T) {

spinifex/providers/ebs/viperblock/volume_fence.go

@@ -9,8 +9,8 @@ import (
 	"os"
 	"time"
 
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/telemetry"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
 	"github.com/nats-io/nats.go/jetstream"
 )
@@ -319,46 +319,40 @@ func (cfg *Config) fenceNoticeRetry() time.Duration {
 }
 
 // publishVolumeFenced tells the local daemon a guest is running against a disk
-// that is gone, until the daemon acknowledges. It gives up on a closed
-// connection, or once the volume is exported here again by a newer mount.
+// that is gone, until the daemon acknowledges. It gives up on a notice that can
+// never be acted on, a closed connection, or the volume being exported again.
 func (cfg *Config) publishVolumeFenced(ctx context.Context, volumeName, winner string) {
 	if cfg.nc == nil {
 		return
 	}
-	event := viperblocklegacyv1.VolumeFencedEvent{
-		Volume: volumeName,
-		Node:   cfg.leaseOwner(),
-		Winner: winner,
-		Reason: fmt.Sprintf("volume lease moved to %s while this node had the volume open", winner),
+	notice := ebsprovider.FenceNotice{
+		Versioned: ebsprovider.NewVersioned(),
+		VolumeID:  volumeName,
+		NodeID:    cfg.NodeName,
+		Winner:    winner,
+		Reason:    fmt.Sprintf("volume lease moved to %s while this node had the volume open", winner),
 	}
-	payload, err := json.Marshal(event)
-	if err != nil {
-		slog.ErrorContext(ctx, "fence: marshal fenced event", "volume", volumeName, "err", err)
-		return
-	}
-	subject := viperblocklegacyv1.VolumeFencedSubject(cfg.NodeName)
 	began := time.Now()
 
 	for attempt := 1; ; attempt++ {
-		reqCtx, cancel := context.WithTimeout(ctx, fenceNoticeTimeout)
-		reply, err := cfg.nc.RequestWithContext(reqCtx, subject, payload)
-		cancel()
+		instanceID, err := ebsprovider.NotifyFenced(ctx, cfg.nc, notice, fenceNoticeTimeout)
 		if err == nil {
-			var ack viperblocklegacyv1.VolumeFencedAck
-			if jerr := json.Unmarshal(reply.Data, &ack); jerr == nil && ack.Error != "" {
-				slog.ErrorContext(ctx, "fence: the daemon could not act on the fenced volume",
-					"volume", volumeName, "subject", subject, "err", ack.Error)
-			} else if attempt > 1 {
+			if attempt > 1 {
 				slog.WarnContext(ctx, "fence: the daemon acknowledged the fenced volume after retrying",
-					"volume", volumeName, "instance", ack.Instance, "attempts", attempt,
+					"volume", volumeName, "instance", instanceID, "attempts", attempt,
 					"unacknowledged_ms", otelsetup.Millis(time.Since(began)))
 			}
 			return
 		}
+		if !ebsprovider.FenceNoticeRetryable(err) {
+			slog.ErrorContext(ctx, "fence: the daemon cannot act on the fenced volume, the guest is left with dead I/O",
+				"volume", volumeName, "node", notice.NodeID, "attempts", attempt, "err", err)
+			return
+		}
 
 		if cfg.nc.IsClosed() {
 			slog.ErrorContext(ctx, "fence: connection closed before the daemon acknowledged the fenced volume, the guest may be left with dead I/O",
-				"volume", volumeName, "subject", subject, "attempts", attempt)
+				"volume", volumeName, "node", notice.NodeID, "attempts", attempt)
 			return
 		}
 		if _, again := cfg.lookupMountedVolume(volumeName); again {
@@ -368,7 +362,7 @@ func (cfg *Config) publishVolumeFenced(ctx context.Context, volumeName, winner s
 		}
 		if attempt == 1 || attempt%fenceNoticeLogEveryTries == 0 {
 			slog.ErrorContext(ctx, "fence: the daemon has not acknowledged the fenced volume, retrying; until it does the guest keeps running with dead I/O",
-				"volume", volumeName, "subject", subject, "attempts", attempt,
+				"volume", volumeName, "node", notice.NodeID, "attempts", attempt,
 				"unacknowledged_ms", otelsetup.Millis(time.Since(began)), "err", err)
 		}
 

spinifex/providers/ebs/viperblock/volume_fence_invariants_test.go

@@ -11,7 +11,7 @@ import (
 	"testing"
 	"time"
 
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -133,16 +133,16 @@ func TestS4_FenceHolderReadIsBounded(t *testing.T) {
 // fenceResponder answers fence notices as the daemon does and records them.
 type fenceResponder struct {
 	mu     sync.Mutex
-	events []viperblocklegacyv1.VolumeFencedEvent
+	events []ebsprovider.FenceNotice
 }
 
 func (r *fenceResponder) handle(msg *nats.Msg) {
-	var event viperblocklegacyv1.VolumeFencedEvent
-	_ = json.Unmarshal(msg.Data, &event)
+	var notice ebsprovider.FenceNotice
+	_ = json.Unmarshal(msg.Data, &notice)
 	r.mu.Lock()
-	r.events = append(r.events, event)
+	r.events = append(r.events, notice)
 	r.mu.Unlock()
-	payload, _ := json.Marshal(viperblocklegacyv1.VolumeFencedAck{Instance: "i-fenced"})
+	payload, _ := json.Marshal(ebsprovider.FenceNoticeAck{Versioned: ebsprovider.ReplyVersioned(notice.SchemaVersion), InstanceID: "i-fenced"})
 	_ = msg.Respond(payload)
 }
 
@@ -185,7 +185,9 @@ func TestS4_FenceNoticeSurvivesAMissedDelivery(t *testing.T) {
 	require.NoError(t, err)
 	t.Cleanup(daemonConn.Close)
 	responder := &fenceResponder{}
-	_, err = daemonConn.Subscribe(viperblocklegacyv1.VolumeFencedSubject("node-a"), responder.handle)
+	subject, err := ebsprovider.FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	_, err = daemonConn.Subscribe(subject, responder.handle)
 	require.NoError(t, err)
 	require.NoError(t, daemonConn.Flush())
 
@@ -197,7 +199,10 @@ func TestS4_FenceNoticeSurvivesAMissedDelivery(t *testing.T) {
 	require.GreaterOrEqualf(t, responder.count(), 1,
 		"ADR-0007:S4: a daemon that subscribed after the first notice never heard the fence, so its guest runs on with no disk. "+
 			"Fix: retry publishVolumeFenced (volume_fence.go) until the daemon acknowledges")
-	assert.Equal(t, volumeName, responder.events[0].Volume)
+	first := responder.events[0]
+	assert.Equal(t, volumeName, first.VolumeID)
+	assert.Equal(t, "node-a", first.NodeID)
+	assert.True(t, ebsprovider.SupportsSchemaVersion(first.SchemaVersion), "the daemon refuses a notice in a version it cannot read")
 
 	seen := responder.count()
 	time.Sleep(5 * cfg.fenceNoticeEvery)
@@ -252,3 +257,93 @@ func TestS4_FenceNoticeStopsWithTheConnection(t *testing.T) {
 		t.Fatal("a fence notice kept retrying on a closed connection")
 	}
 }
+
+// TestFenceNotice_UnavailableDaemonIsAskedAgain keeps a refusal that says
+// "not now" from ending delivery: the guest is still running on a dead disk.
+func TestFenceNotice_UnavailableDaemonIsAskedAgain(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+
+	const volumeName = "vol-fencenoticeunavailable"
+	cfg, nc := noticeConfig(t, natsURL, volumeName)
+
+	subject, err := ebsprovider.FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	unavailable, err := json.Marshal(ebsprovider.FenceNoticeAck{
+		Versioned: ebsprovider.NewVersioned(),
+		Error:     &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnavailable, Message: "not yet"},
+	})
+	require.NoError(t, err)
+	ack, err := json.Marshal(ebsprovider.FenceNoticeAck{Versioned: ebsprovider.NewVersioned(), InstanceID: "i-fenced"})
+	require.NoError(t, err)
+	var heard atomic.Int32
+	_, err = nc.Subscribe(subject, func(msg *nats.Msg) {
+		if heard.Add(1) == 1 {
+			_ = msg.Respond(unavailable)
+			return
+		}
+		_ = msg.Respond(ack)
+	})
+	require.NoError(t, err)
+	require.NoError(t, nc.Flush())
+
+	done := make(chan struct{})
+	go func() { defer close(done); cfg.onVolumeLeaseLost(context.Background(), volumeName, leaseLostStalled) }()
+	select {
+	case <-done:
+	case <-time.After(fenceNoticeTimeout + 2*time.Second):
+		t.Fatal("a fence notice was still being sent after the daemon acknowledged it")
+	}
+	assert.Equal(t, int32(2), heard.Load(), "ADR-0007:S4: a daemon that was unavailable was never asked again")
+}
+
+// TestFenceNotice_UnsendableNoticeIsNotRetried stops a notice that can never
+// be addressed from spinning until shutdown on a goroutine nobody watches.
+func TestFenceNotice_UnsendableNoticeIsNotRetried(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+
+	const volumeName = "vol-fencenoticeunsendable"
+	cfg, _ := noticeConfig(t, natsURL, volumeName)
+	cfg.NodeName = ""
+	cfg.MountedVolumes = nil // gone, as after the teardown, so only the error can end the retry
+
+	done := make(chan struct{})
+	go func() { defer close(done); cfg.publishVolumeFenced(t.Context(), volumeName, "node-b") }()
+	select {
+	case <-done:
+	case <-time.After(fenceNoticeTimeout + 2*time.Second):
+		t.Fatal("a fence notice with no subject to send it on kept retrying")
+	}
+}
+
+// TestFenceNotice_RefusedNoticeIsNotRepeated keeps a notice the daemon heard and
+// could not act on from being resent forever: a refusal is an answer.
+func TestFenceNotice_RefusedNoticeIsNotRepeated(t *testing.T) {
+	_, natsURL := setupEmbeddedNATS(t)
+
+	const volumeName = "vol-fencenoticerefused"
+	cfg, nc := noticeConfig(t, natsURL, volumeName)
+
+	subject, err := ebsprovider.FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	var heard atomic.Int32
+	refusal, err := json.Marshal(ebsprovider.FenceNoticeAck{
+		Versioned: ebsprovider.NewVersioned(),
+		Error:     &ebsprovider.ProviderError{Code: ebsprovider.ErrorCodeUnsupportedVersion, Message: "no"},
+	})
+	require.NoError(t, err)
+	_, err = nc.Subscribe(subject, func(msg *nats.Msg) {
+		heard.Add(1)
+		_ = msg.Respond(refusal)
+	})
+	require.NoError(t, err)
+	require.NoError(t, nc.Flush())
+
+	done := make(chan struct{})
+	go func() { defer close(done); cfg.onVolumeLeaseLost(context.Background(), volumeName, leaseLostStalled) }()
+	select {
+	case <-done:
+	case <-time.After(fenceNoticeTimeout + 2*time.Second):
+		t.Fatal("a fence notice kept retrying after the daemon refused it")
+	}
+	assert.Equal(t, int32(1), heard.Load())
+}

spinifex/providers/ebs/viperblock/volume_fence_test.go

@@ -12,6 +12,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -171,15 +172,25 @@ func TestVolumeFence_CompletesOnceTheWriterFinallyExits(t *testing.T) {
 
 	const volumeName = "vol-fencekilllate"
 
-	cfg := fencedConfig(t, natsURL, "node-a", volumeName)
+	cfg, nc := noticeConfig(t, natsURL, volumeName)
 	cfg.MountedVolumes[0].PID = 0 // rejected by ForceKillProcess, so the kill cannot succeed
 	// Alive for the guard and one watch pass, gone on the next: the fence has to
 	// see the failure persist and then resolve.
 	var probes atomic.Int32
 	cfg.processAlive = func(int) bool { return probes.Add(1) <= 2 }
 	cfg.fenceWatchEvery = 10 * time.Millisecond
+	responder := &fenceResponder{}
+	subject, err := ebsprovider.FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	_, err = nc.Subscribe(subject, responder.handle)
+	require.NoError(t, err)
+	require.NoError(t, nc.Flush())
 
 	cfg.fenceVolume(t.Context(), volumeName, leaseLostToPeer)
+	require.Equalf(t, 1, responder.count(),
+		"ADR-0007:S4: a fence that completed late never told the daemon, so its guest runs on with no disk. "+
+			"Fix: publishVolumeFenced once awaitFencedWriterExit sees the writer gone (volume_fence.go)")
+	assert.Equal(t, volumeName, responder.events[0].VolumeID)
 
 	cfg.mu.Lock()
 	mounted := len(cfg.MountedVolumes)

spinifex/runtime/compute/vm/console_log_reaper.go

@@ -0,0 +1,228 @@
+package vm
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	hostprocess "github.com/mulgadc/spinifex/spinifex/runtime/host/process"
+)
+
+const (
+	// consoleTrimAbove is the size past which a console log is cut back.
+	consoleTrimAbove = 4 << 20
+	// consoleKeepBytes is how much of the newest output survives a cut.
+	consoleKeepBytes = 1 << 20
+	// consoleRemoveAfter is how long a retired console file lingers so a late
+	// GetConsoleOutput still sees it.
+	consoleRemoveAfter = time.Hour
+	// consoleIdleAfter is how long a live console file must sit unchanged, with
+	// no QEMU on this node holding it open, before it is removed.
+	consoleIdleAfter = time.Hour
+)
+
+// consoleTerminatedSuffix marks the console file of an instance that is gone
+// for good. The log shipper only follows console-*.log, so a renamed file is
+// no longer shipped, and the reaper removes it once it is old enough.
+const consoleTerminatedSuffix = ".terminated"
+
+// RetiredConsolePath is where the console file at path moves when its instance
+// is terminated.
+func RetiredConsolePath(path string) string { return path + consoleTerminatedSuffix }
+
+// retireConsoleLog renames an ended instance's console file so the reaper can
+// age it out. The file is final by now: QEMU is down or being force-finalized.
+func retireConsoleLog(path string) {
+	if path == "" {
+		return
+	}
+	if err := os.Rename(path, RetiredConsolePath(path)); err != nil && !os.IsNotExist(err) {
+		slog.Warn("vm: retire console log failed", "path", path, "err", err)
+	}
+}
+
+// ConsoleLogReaper caps the serial console files under dir and removes the
+// retired ones. A live console-<id>.log goes only after an idle hour with no
+// QEMU on this node holding it open, decided from /proc alone and never from
+// instance state.
+type ConsoleLogReaper struct {
+	dir      string
+	procRoot string
+	now      func() time.Time
+}
+
+var _ Reaper = (*ConsoleLogReaper)(nil)
+
+// NewConsoleLogReaper builds a reaper over the console files in dir.
+func NewConsoleLogReaper(dir string) *ConsoleLogReaper {
+	return &ConsoleLogReaper{dir: dir, procRoot: qemuProcRoot, now: time.Now}
+}
+
+func (r *ConsoleLogReaper) Class() string      { return "console-log" }
+func (r *ConsoleLogReaper) Scope() ReaperScope { return ScopeNodeLocal }
+
+// NewLocalConsoleLogReaper builds the reaper over this node's runtime directory.
+func (m *Manager) NewLocalConsoleLogReaper() *ConsoleLogReaper {
+	return NewConsoleLogReaper(hostprocess.RuntimeDir())
+}
+
+// Sweep removes idle live console files nothing holds open, cuts oversized ones
+// back to their newest consoleKeepBytes, and removes retired ones that have sat
+// for over an hour.
+func (r *ConsoleLogReaper) Sweep(context.Context) (int, error) {
+	reaped := 0
+	live, err := filepath.Glob(filepath.Join(r.dir, "console-*.log"))
+	if err != nil {
+		return 0, err
+	}
+	now := r.now()
+	warned := false
+	for _, p := range live {
+		info, err := os.Stat(p)
+		if err != nil || !info.Mode().IsRegular() {
+			continue
+		}
+		if now.Sub(info.ModTime()) > consoleIdleAfter && r.removeIfUnheld(p, &warned) {
+			reaped++
+			continue
+		}
+		if info.Size() <= consoleTrimAbove {
+			continue
+		}
+		if err := trimConsoleLog(p); err != nil {
+			slog.Warn("vm/gc: trim console log failed", "path", p, "err", err)
+			continue
+		}
+		reaped++
+	}
+	retired, err := filepath.Glob(filepath.Join(r.dir, "console-*.log"+consoleTerminatedSuffix))
+	if err != nil {
+		return reaped, err
+	}
+	for _, p := range retired {
+		info, err := os.Stat(p)
+		if err != nil || !info.Mode().IsRegular() || now.Sub(info.ModTime()) <= consoleRemoveAfter {
+			continue
+		}
+		if err := os.Remove(p); err != nil && !os.IsNotExist(err) {
+			slog.Warn("vm/gc: remove retired console log failed", "path", p, "err", err)
+			continue
+		}
+		reaped++
+	}
+	return reaped, nil
+}
+
+// removeIfUnheld removes path when no QEMU process on this node has it open. If
+// the process table cannot be read in full it removes nothing, since a guest
+// that is merely quiet must keep its log.
+func (r *ConsoleLogReaper) removeIfUnheld(path string, warned *bool) bool {
+	held, err := qemuOpenFiles(r.procRoot, r.dir)
+	if err != nil {
+		if !*warned {
+			slog.Warn("vm/gc: idle console logs kept, cannot tell whether QEMU holds them", "err", err)
+			*warned = true
+		}
+		return false
+	}
+	if resolved, err := filepath.EvalSymlinks(path); err == nil {
+		path = resolved
+	}
+	if held[path] {
+		return false
+	}
+	if err := os.Remove(path); err != nil {
+		if !os.IsNotExist(err) {
+			slog.Warn("vm/gc: remove idle console log failed", "path", path, "err", err)
+		}
+		return false
+	}
+	return true
+}
+
+// qemuOpenFiles returns the files every QEMU process under procRoot has open.
+// Any unreadable descriptor table is an error, never a partial answer. The scan
+// is complete only while the daemon keeps CAP_SYS_ADMIN under ProtectProc=invisible,
+// so an instance pidfile naming a live pid the scan missed is an error too.
+func qemuOpenFiles(procRoot, runtimeDir string) (map[string]bool, error) {
+	pids, err := scanLiveQEMUPIDs(procRoot)
+	if err != nil {
+		return nil, err
+	}
+	if err := checkPidFilesVisible(runtimeDir, pids); err != nil {
+		return nil, err
+	}
+	held := make(map[string]bool)
+	for _, pid := range pids {
+		fdDir := filepath.Join(procRoot, strconv.Itoa(pid), "fd")
+		fds, err := os.ReadDir(fdDir)
+		if err != nil {
+			if os.IsNotExist(err) {
+				continue
+			}
+			return nil, err
+		}
+		for _, fd := range fds {
+			target, err := os.Readlink(filepath.Join(fdDir, fd.Name()))
+			if err != nil {
+				if os.IsNotExist(err) {
+					continue
+				}
+				return nil, err
+			}
+			held[target] = true
+		}
+	}
+	return held, nil
+}
+
+// checkPidFilesVisible fails when an instance pidfile names a process that is
+// alive but absent from seen. Signal 0 answers even when /proc hides the pid.
+func checkPidFilesVisible(runtimeDir string, seen []int) error {
+	visible := make(map[int]bool, len(seen))
+	for _, pid := range seen {
+		visible[pid] = true
+	}
+	for pid, id := range pidFileOwners(runtimeDir) {
+		if !strings.HasPrefix(id, "i-") || visible[pid] {
+			continue
+		}
+		if err := syscall.Kill(pid, 0); err == nil || err == syscall.EPERM {
+			return fmt.Errorf("pidfile for %s names live pid %d that the process scan cannot see", id, pid)
+		}
+	}
+	return nil
+}
+
+// trimConsoleLog rewrites the file in place as its newest consoleKeepBytes. The
+// guest's writer keeps its descriptor (O_APPEND), so output written while the
+// copy runs may be lost but the writer is never blocked.
+func trimConsoleLog(path string) error {
+	f, err := os.OpenFile(path, os.O_RDWR, 0)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	info, err := f.Stat()
+	if err != nil {
+		return err
+	}
+	if info.Size() <= consoleTrimAbove {
+		return nil
+	}
+	tail := make([]byte, consoleKeepBytes)
+	if _, err := f.ReadAt(tail, info.Size()-consoleKeepBytes); err != nil && err != io.EOF {
+		return err
+	}
+	if _, err := f.WriteAt(tail, 0); err != nil {
+		return err
+	}
+	return f.Truncate(consoleKeepBytes)
+}

spinifex/runtime/compute/vm/console_log_reaper_test.go

@@ -0,0 +1,298 @@
+package vm
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// consoleBody returns n bytes of 16-byte numbered lines, so any slice of it
+// identifies which part of the original it came from.
+func consoleBody(n int) []byte {
+	var b bytes.Buffer
+	for i := 0; b.Len() < n; i++ {
+		fmt.Fprintf(&b, "l%014d\n", i)
+	}
+	return b.Bytes()[:n]
+}
+
+func newConsoleReaper(t *testing.T, now time.Time) (*ConsoleLogReaper, string) {
+	t.Helper()
+	dir := t.TempDir()
+	r := NewConsoleLogReaper(dir)
+	r.procRoot = t.TempDir()
+	r.now = func() time.Time { return now }
+	return r, dir
+}
+
+func writeConsole(t *testing.T, dir, id string, body []byte, mtime time.Time) string {
+	t.Helper()
+	p := filepath.Join(dir, "console-"+id+".log")
+	require.NoError(t, os.WriteFile(p, body, 0o644))
+	require.NoError(t, os.Chtimes(p, mtime, mtime))
+	return p
+}
+
+func TestConsoleLogReaper_TrimsToNewestMiB(t *testing.T) {
+	now := time.Now()
+	r, dir := newConsoleReaper(t, now)
+	body := consoleBody(5 << 20)
+	p := writeConsole(t, dir, "i-big", body, now)
+
+	reaped, err := r.Sweep(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, 1, reaped)
+
+	got, err := os.ReadFile(p)
+	require.NoError(t, err)
+	gotSize := len(got)
+	require.Equal(t, consoleKeepBytes, gotSize, "a trim keeps exactly the newest MiB")
+	assert.True(t, bytes.Equal(body[len(body)-consoleKeepBytes:], got), "the newest MiB must survive, in order")
+}
+
+func TestConsoleLogReaper_LeavesFilesAtOrBelowTheLimit(t *testing.T) {
+	now := time.Now()
+	r, dir := newConsoleReaper(t, now)
+	small := consoleBody(3 << 20)
+	limit := consoleBody(consoleTrimAbove)
+	ps := writeConsole(t, dir, "i-small", small, now)
+	pl := writeConsole(t, dir, "i-limit", limit, now)
+
+	reaped, err := r.Sweep(context.Background())
+	require.NoError(t, err)
+	assert.Zero(t, reaped)
+
+	for p, want := range map[string][]byte{ps: small, pl: limit} {
+		got, err := os.ReadFile(p)
+		require.NoError(t, err)
+		assert.True(t, bytes.Equal(want, got), "%s must not change", filepath.Base(p))
+	}
+}
+
+// QEMU writes through a descriptor it keeps open, so the trim must leave the
+// file usable for that writer: no hole of NULs, and later output at the end.
+func TestConsoleLogReaper_AppendWriterKeepsWritingAfterTrim(t *testing.T) {
+	now := time.Now()
+	r, dir := newConsoleReaper(t, now)
+	p := filepath.Join(dir, "console-i-live.log")
+	w, err := os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o644)
+	require.NoError(t, err)
+	defer w.Close()
+	_, err = w.Write(consoleBody(5 << 20))
+	require.NoError(t, err)
+
+	_, err = r.Sweep(context.Background())
+	require.NoError(t, err)
+
+	marker := []byte("after-trim-marker\n")
+	_, err = w.Write(marker)
+	require.NoError(t, err)
+
+	got, err := os.ReadFile(p)
+	require.NoError(t, err)
+	assert.Zero(t, bytes.Count(got, []byte{0}), "no NUL hole from a stale write offset")
+	assert.True(t, bytes.HasSuffix(got, marker), "the writer's next line lands at the end")
+	assert.LessOrEqual(t, len(got), consoleKeepBytes+len(marker))
+}
+
+func TestConsoleLogReaper_RemovesOnlyAgedRetiredFiles(t *testing.T) {
+	now := time.Now()
+	r, dir := newConsoleReaper(t, now)
+	old := now.Add(-consoleRemoveAfter - time.Minute)
+	recent := now.Add(-consoleRemoveAfter + time.Minute)
+	retiredOld := writeConsole(t, dir, "i-gone-old", []byte("x\n"), old)
+	retiredNew := writeConsole(t, dir, "i-gone-new", []byte("x\n"), recent)
+	require.NoError(t, os.Rename(retiredOld, RetiredConsolePath(retiredOld)))
+	require.NoError(t, os.Rename(retiredNew, RetiredConsolePath(retiredNew)))
+	require.NoError(t, os.Chtimes(RetiredConsolePath(retiredOld), old, old))
+	require.NoError(t, os.Chtimes(RetiredConsolePath(retiredNew), recent, recent))
+	// A live file that is not yet idle for an hour is kept.
+	stopped := writeConsole(t, dir, "i-stopped", []byte("boot\n"), recent)
+	other := filepath.Join(dir, "serial-i-x.sock.log.terminated")
+	require.NoError(t, os.WriteFile(other, []byte("x"), 0o644))
+	require.NoError(t, os.Chtimes(other, old, old))
+
+	reaped, err := r.Sweep(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, 1, reaped, "only the retired file older than an hour")
+
+	_, err = os.Stat(RetiredConsolePath(retiredOld))
+	assert.ErrorIs(t, err, os.ErrNotExist, "retired over an hour ago: removed")
+	_, err = os.Stat(RetiredConsolePath(retiredNew))
+	assert.NoError(t, err, "retired recently: kept for a late GetConsoleOutput")
+	_, err = os.Stat(stopped)
+	assert.NoError(t, err, "a live file idle for under an hour is kept")
+	_, err = os.Stat(other)
+	assert.NoError(t, err, "files that are not console logs are never touched")
+}
+
+func TestRetireConsoleLog(t *testing.T) {
+	dir := t.TempDir()
+	p := writeConsole(t, dir, "i-end", []byte("last words\n"), time.Now())
+
+	retireConsoleLog(p)
+	_, err := os.Stat(p)
+	assert.ErrorIs(t, err, os.ErrNotExist, "the live name no longer matches the shipper's glob")
+	got, err := os.ReadFile(RetiredConsolePath(p))
+	require.NoError(t, err)
+	assert.Equal(t, "last words\n", string(got))
+
+	assert.NotPanics(t, func() { retireConsoleLog(p) }, "a repeat finalize is a no-op")
+	assert.NotPanics(t, func() { retireConsoleLog("") }, "an instance that never got a console path is a no-op")
+}
+
+func TestConsoleLogReaper_TrimsOnlyLiveFiles(t *testing.T) {
+	now := time.Now()
+	r, dir := newConsoleReaper(t, now)
+	big := writeConsole(t, dir, "i-big", consoleBody(5<<20), now)
+	retired := writeConsole(t, dir, "i-gone", consoleBody(5<<20), now)
+	require.NoError(t, os.Rename(retired, RetiredConsolePath(retired)))
+
+	reaped, err := r.Sweep(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, 1, reaped)
+	info, err := os.Stat(big)
+	require.NoError(t, err)
+	assert.Equal(t, int64(consoleKeepBytes), info.Size())
+	info, err = os.Stat(RetiredConsolePath(retired))
+	require.NoError(t, err)
+	assert.Equal(t, int64(5<<20), info.Size(), "a retired file is final and is only aged out")
+}
+
+func TestExecute_ConsoleAlwaysAppends(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		cfg  Config
+		want string
+	}{
+		{"socket first boot", Config{SerialSocket: "/s", ConsoleLogPath: "/c"}, "logfile=/c,logappend=on"},
+		{"microvm first boot", Config{MachineType: "microvm", ConsoleLogPath: "/c"}, "file,id=console0,path=/c,append=on"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			cfg := minimalConfig()
+			cfg.MachineType, cfg.SerialSocket = tc.cfg.MachineType, tc.cfg.SerialSocket
+			cfg.ConsoleLogPath = tc.cfg.ConsoleLogPath
+			cmd, err := cfg.Execute()
+			require.NoError(t, err)
+			var chardev string
+			for i, a := range cmd.Args {
+				if a == "-chardev" && i+1 < len(cmd.Args) {
+					chardev = cmd.Args[i+1]
+				}
+			}
+			assert.Contains(t, chardev, tc.want, "a truncating open would let an in-place trim leave a NUL hole")
+		})
+	}
+}
+
+// fakeQEMUProc fabricates procRoot/<pid> with a QEMU comm and the given open files.
+func fakeQEMUProc(t *testing.T, procRoot string, pid int, open ...string) {
+	t.Helper()
+	pdir := filepath.Join(procRoot, strconv.Itoa(pid))
+	require.NoError(t, os.MkdirAll(filepath.Join(pdir, "fd"), 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(pdir, "comm"), []byte("qemu-system-x86\n"), 0o644))
+	for i, f := range open {
+		require.NoError(t, os.Symlink(f, filepath.Join(pdir, "fd", strconv.Itoa(i+3))))
+	}
+}
+
+func TestConsoleLogReaper_RemovesIdleLiveFileNothingHolds(t *testing.T) {
+	now := time.Now()
+	r, dir := newConsoleReaper(t, now)
+	idle := writeConsole(t, dir, "i-stopped-then-terminated", []byte("boot\n"), now.Add(-consoleIdleAfter-time.Minute))
+	fresh := writeConsole(t, dir, "i-fresh", []byte("boot\n"), now.Add(-consoleIdleAfter+time.Minute))
+	fakeQEMUProc(t, r.procRoot, 4242)
+
+	reaped, err := r.Sweep(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, 1, reaped)
+	_, err = os.Stat(idle)
+	assert.ErrorIs(t, err, os.ErrNotExist, "idle over an hour and unopened: removed")
+	_, err = os.Stat(fresh)
+	assert.NoError(t, err, "written within the hour: kept")
+}
+
+func TestConsoleLogReaper_KeepsIdleFileHeldByRealProcess(t *testing.T) {
+	now := time.Now()
+	r, dir := newConsoleReaper(t, now)
+	held := writeConsole(t, dir, "i-quiet-guest", []byte("boot\n"), now.Add(-48*time.Hour))
+	// A real child holds the file open, as QEMU holds its chardev log.
+	cmd := exec.Command("sh", "-c", "exec 3<"+held+"; echo ready; read _")
+	stdin, err := cmd.StdinPipe()
+	require.NoError(t, err)
+	out, err := cmd.StdoutPipe()
+	require.NoError(t, err)
+	require.NoError(t, cmd.Start())
+	t.Cleanup(func() { _ = stdin.Close(); _ = cmd.Wait() })
+	line := make([]byte, 6)
+	_, err = io.ReadFull(out, line)
+	require.NoError(t, err)
+	// Present the child to the reaper as a QEMU process using its real fd table.
+	r.procRoot = t.TempDir()
+	pdir := filepath.Join(r.procRoot, strconv.Itoa(cmd.Process.Pid))
+	require.NoError(t, os.MkdirAll(pdir, 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(pdir, "comm"), []byte("qemu-system-x86\n"), 0o644))
+	require.NoError(t, os.Symlink(fmt.Sprintf("/proc/%d/fd", cmd.Process.Pid), filepath.Join(pdir, "fd")))
+
+	reaped, err := r.Sweep(context.Background())
+	require.NoError(t, err)
+	assert.Zero(t, reaped)
+	_, err = os.Stat(held)
+	assert.NoError(t, err, "a quiet running guest keeps its log")
+}
+
+func TestConsoleLogReaper_KeepsIdleFileWhenProcessTableUnreadable(t *testing.T) {
+	now := time.Now()
+	r, dir := newConsoleReaper(t, now)
+	idle := writeConsole(t, dir, "i-unprovable", []byte("boot\n"), now.Add(-48*time.Hour))
+	pdir := filepath.Join(r.procRoot, "4242")
+	require.NoError(t, os.MkdirAll(pdir, 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(pdir, "comm"), []byte("qemu-system-x86\n"), 0o644))
+	// A QEMU whose fd table cannot be read: a dangling fd entry that is a file, not a directory.
+	require.NoError(t, os.WriteFile(filepath.Join(pdir, "fd"), nil, 0o644))
+
+	reaped, err := r.Sweep(context.Background())
+	require.NoError(t, err)
+	assert.Zero(t, reaped)
+	_, err = os.Stat(idle)
+	assert.NoError(t, err, "cannot prove it is unused: kept")
+}
+
+func TestConsoleLogReaper_KeepsIdleFileWhenPidFileNamesLivePidTheScanMissed(t *testing.T) {
+	now := time.Now()
+	r, dir := newConsoleReaper(t, now)
+	idle := writeConsole(t, dir, "i-quiet", []byte("boot\n"), now.Add(-48*time.Hour))
+	// This test process is alive but is no QEMU, so the fabricated scan never lists it.
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "i-quiet.pid"), []byte(strconv.Itoa(os.Getpid())), 0o644))
+
+	reaped, err := r.Sweep(context.Background())
+	require.NoError(t, err)
+	assert.Zero(t, reaped)
+	_, err = os.Stat(idle)
+	assert.NoError(t, err, "a live pid the scan cannot see means the scan is incomplete: nothing removed")
+}
+
+func TestConsoleLogReaper_IgnoresPidFileOfDeadProcess(t *testing.T) {
+	now := time.Now()
+	r, dir := newConsoleReaper(t, now)
+	idle := writeConsole(t, dir, "i-gone", []byte("boot\n"), now.Add(-48*time.Hour))
+	cmd := exec.Command("true")
+	require.NoError(t, cmd.Run())
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "i-gone.pid"), []byte(strconv.Itoa(cmd.Process.Pid)), 0o644))
+
+	reaped, err := r.Sweep(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, 1, reaped)
+	_, err = os.Stat(idle)
+	assert.ErrorIs(t, err, os.ErrNotExist, "a pidfile for a dead process does not block removal")
+}

spinifex/runtime/compute/vm/console_retire_test.go

@@ -0,0 +1,85 @@
+package vm
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func consoleFixture(t *testing.T, id string) (*VM, string) {
+	t.Helper()
+	path := filepath.Join(t.TempDir(), "console-"+id+".log")
+	require.NoError(t, os.WriteFile(path, []byte("last output\n"), 0o644))
+	return &VM{
+		ID:           id,
+		Status:       StateRunning,
+		InstanceType: "t3.micro",
+		Instance:     &ec2.Instance{},
+		Config:       Config{ConsoleLogPath: path},
+	}, path
+}
+
+func requireRetired(t *testing.T, path string) {
+	t.Helper()
+	_, err := os.Stat(path)
+	require.ErrorIs(t, err, os.ErrNotExist, "the shipper's console-*.log glob must no longer match")
+	got, err := os.ReadFile(RetiredConsolePath(path))
+	require.NoError(t, err)
+	assert.Equal(t, "last output\n", string(got), "the content survives the rename")
+}
+
+func requireLive(t *testing.T, path string) {
+	t.Helper()
+	_, err := os.Stat(path)
+	require.NoError(t, err)
+	_, err = os.Stat(RetiredConsolePath(path))
+	require.ErrorIs(t, err, os.ErrNotExist)
+}
+
+func TestTerminate_RetiresConsoleLog(t *testing.T) {
+	m, _, _, _, _ := shutdownTestManager(t)
+	instance, path := consoleFixture(t, "i-term-console")
+	m.Insert(instance)
+
+	require.NoError(t, m.Terminate(instance.ID))
+
+	requireRetired(t, path)
+}
+
+func TestMarkFailed_RetiresConsoleLog(t *testing.T) {
+	m, _, _, _, _ := shutdownTestManager(t)
+	instance, path := consoleFixture(t, "i-failed-console")
+	instance.FirstLaunch = true
+	instance.Status = StatePending
+	m.Insert(instance)
+
+	m.MarkFailed(context.Background(), instance, "launch failed")
+	m.goroutineWg.Wait()
+
+	requireRetired(t, path)
+}
+
+func TestStop_KeepsConsoleLog(t *testing.T) {
+	m, _, _, _, _ := shutdownTestManager(t)
+	instance, path := consoleFixture(t, "i-stop-console")
+	m.Insert(instance)
+
+	require.NoError(t, m.Stop(instance.ID))
+
+	requireLive(t, path)
+}
+
+func TestStopAll_DrainKeepsConsoleLog(t *testing.T) {
+	m, _, _, _, _ := shutdownTestManager(t)
+	instance, path := consoleFixture(t, "i-drain-console")
+	m.Insert(instance)
+
+	require.NoError(t, m.StopAll())
+
+	requireLive(t, path)
+}

spinifex/runtime/compute/vm/guest_poweroff.go

@@ -0,0 +1,27 @@
+package vm
+
+import "log/slog"
+
+// HandleQEMUExit decides what a finished QEMU process means once startup was
+// confirmed. A wait error is a crash. A clean exit after the heartbeat saw the
+// guest power itself off is a customer stop, kept in the stopped bucket so
+// StartInstances can bring it back. Any other clean exit is left to the
+// heartbeat's crash detection.
+func (m *Manager) HandleQEMUExit(instance *VM, waitErr error) {
+	if waitErr != nil {
+		if m.deps.CrashHandler != nil {
+			m.deps.CrashHandler(instance, waitErr)
+		}
+		return
+	}
+	if !instance.guestPoweroff.Load() {
+		slog.Info("VM process exited cleanly", "instance", instance.ID)
+		return
+	}
+
+	slog.Info("Guest powered itself off, stopping instance", "instance", instance.ID)
+	m.UpdateState(instance.ID, func(v *VM) { v.DesiredState = DesiredStopped })
+	if err := m.Stop(instance.ID); err != nil {
+		slog.Error("Stop after guest poweroff failed", "instance", instance.ID, "err", err)
+	}
+}

spinifex/runtime/compute/vm/guest_poweroff_test.go

@@ -0,0 +1,155 @@
+package vm
+
+import (
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func guestPoweroffManager(t *testing.T) (*Manager, *fakeStateStore, *recordedTransitions, *[]string) {
+	t.Helper()
+	m, store, _, _, rt := shutdownTestManager(t)
+	var (
+		mu   sync.Mutex
+		down []string
+	)
+	m.SetDeps(Deps{
+		NodeID:          "test-node",
+		StateStore:      store,
+		VolumeMounter:   &fakeVolumeMounter{},
+		InstanceCleaner: &recordingInstanceCleaner{},
+		TransitionState: rt.apply,
+		ShutdownSignal:  func() bool { return false },
+		CrashHandler:    m.HandleCrash,
+		Hooks: ManagerHooks{
+			OnInstanceDown: func(id string) {
+				mu.Lock()
+				down = append(down, id)
+				mu.Unlock()
+			},
+		},
+	})
+	return m, store, rt, &down
+}
+
+func TestQMPHeartbeatPoll_MarksGuestPoweroff(t *testing.T) {
+	for name, tc := range map[string]struct {
+		status  string
+		running bool
+		want    bool
+	}{
+		"guest shut down": {qmpStatusShutdown, false, true},
+		"guest running":   {"running", true, false},
+	} {
+		t.Run(name, func(t *testing.T) {
+			qmpClient, cancel := newMockQMPClient(t, func(cmd qmp.QMPCommand) map[string]any {
+				if cmd.Execute == "query-status" {
+					return qmpStatusResponse(tc.status, tc.running)
+				}
+				return nil
+			})
+			defer cancel()
+
+			m := NewManager()
+			instance := &VM{ID: "i-1", Status: StateRunning, QMPClient: qmpClient}
+			m.Insert(instance)
+
+			require.True(t, m.qmpHeartbeatPoll(instance))
+			assert.Equal(t, tc.want, instance.guestPoweroff.Load())
+		})
+	}
+}
+
+// A guest that powered itself off is a stop: stopping then stopped, released
+// to the stopped bucket so StartInstances can bring it back, and never a crash.
+func TestHandleQEMUExit_GuestPoweroffStopsInstance(t *testing.T) {
+	m, store, rt, down := guestPoweroffManager(t)
+	scheduled := countScheduledRestarts(t)
+
+	instance := &VM{
+		ID:           "i-poweroff",
+		Status:       StateRunning,
+		InstanceType: "t3.micro",
+		Instance:     &ec2.Instance{},
+	}
+	instance.guestPoweroff.Store(true)
+	m.Insert(instance)
+
+	started := time.Now()
+	m.HandleQEMUExit(instance, nil)
+
+	assert.Less(t, time.Since(started), qmpHeartbeatInterval,
+		"the stop must not wait for the next heartbeat")
+	assert.Equal(t, []InstanceState{StateStopping, StateStopped}, rt.targets(instance.ID),
+		"a guest poweroff goes stopping then stopped and never through error")
+	assert.Zero(t, scheduled.Load(), "a guest poweroff must not be restarted")
+	assert.Equal(t, []string{instance.ID}, *down)
+	stored, err := store.LoadStoppedInstance(instance.ID)
+	require.NoError(t, err)
+	require.NotNil(t, stored, "the instance must be startable from the stopped bucket")
+	assert.Equal(t, StateStopped, stored.Status)
+}
+
+// Without the guest-poweroff mark a clean exit is not ours to call a stop.
+func TestHandleQEMUExit_CleanExitWithoutMarkIsLeftToCrashRecovery(t *testing.T) {
+	m, _, rt, _ := guestPoweroffManager(t)
+
+	instance := &VM{ID: "i-clean", Status: StateRunning, InstanceType: "t3.micro", Instance: &ec2.Instance{}}
+	m.Insert(instance)
+
+	m.HandleQEMUExit(instance, nil)
+
+	assert.Empty(t, rt.targets(instance.ID), "no stop without the mark")
+	assert.Equal(t, StateRunning, m.Status(instance))
+}
+
+// A wait error is a crash even when the mark is set: the mark only vouches for
+// the quit we sent, not for a process that died some other way.
+func TestHandleQEMUExit_WaitErrorIsACrashEvenWithMark(t *testing.T) {
+	m, _, rt, _ := guestPoweroffManager(t)
+	countScheduledRestarts(t)
+
+	instance := &VM{ID: "i-killed", Status: StateRunning, InstanceType: "t3.micro", Instance: &ec2.Instance{}}
+	instance.guestPoweroff.Store(true)
+	m.Insert(instance)
+
+	m.HandleQEMUExit(instance, errors.New("signal: killed"))
+
+	assert.Contains(t, rt.targets(instance.ID), StateError)
+	assert.NotContains(t, rt.targets(instance.ID), StateStopped)
+}
+
+// One heartbeat poll is enough to see the poweroff and end QEMU, and the exit
+// then stops the instance. It never goes through the failure count that
+// precedes crash recovery, which takes three polls to reach. The interval is a
+// constant, so "within one interval" is a single poll with no waiting.
+func TestGuestPoweroff_ReachesStoppedFromOnePollWithoutCrashCounting(t *testing.T) {
+	m, _, rt, _ := guestPoweroffManager(t)
+	scheduled := countScheduledRestarts(t)
+
+	qmpClient, cancel := newMockQMPClient(t, func(cmd qmp.QMPCommand) map[string]any {
+		if cmd.Execute == "query-status" {
+			return qmpStatusResponse(qmpStatusShutdown, false)
+		}
+		return nil
+	})
+	defer cancel()
+	instance := &VM{
+		ID: "i-one-poll", Status: StateRunning, InstanceType: "t3.micro",
+		Instance: &ec2.Instance{}, QMPClient: qmpClient,
+	}
+	m.Insert(instance)
+
+	require.True(t, m.qmpHeartbeatPoll(instance), "the poll keeps running until QEMU exits")
+	m.HandleQEMUExit(instance, nil)
+
+	assert.Equal(t, []InstanceState{StateStopping, StateStopped}, rt.targets(instance.ID))
+	assert.Zero(t, instance.Health.QMPConsecutiveFailures, "a powered-off guest is not a failed heartbeat")
+	assert.Zero(t, scheduled.Load(), "no crash restart")
+}

spinifex/runtime/compute/vm/lifecycle.go

@@ -176,7 +176,7 @@ func (m *Manager) launch(ctx context.Context, instance *VM) (err error) {
 	}
 
 	_, qemuSpan := otel.Tracer(vmTracerName).Start(ctx, "vm.launch.start_qemu")
-	qemuErr := m.startQEMU(instance, false)
+	qemuErr := m.startQEMU(instance)
 	endSpanWithError(qemuSpan, qemuErr)
 	if qemuErr != nil {
 		slog.ErrorContext(ctx, "Failed to launch instance", "err", qemuErr)
@@ -282,9 +282,9 @@ func qemuStderrLevel(line string) slog.Level {
 }
 
 // startQEMU launches the QEMU process for instance and waits for startup to
-// confirm. appendConsole keeps the console log of the process it replaces, so a
-// reboot does not erase what the guest printed before it.
-func (m *Manager) startQEMU(instance *VM, appendConsole bool) error {
+// confirm. The console log is opened in append mode, so a replacement process
+// keeps what the guest printed before it.
+func (m *Manager) startQEMU(instance *VM) error {
 	pidFile, err := hostprocess.GeneratePidFile(instance.ID)
 	if err != nil {
 		slog.Error("Failed to generate PID file", "err", err)
@@ -504,7 +504,6 @@ func (m *Manager) startQEMU(instance *VM, appendConsole bool) error {
 		}
 		launchCfg = &composed
 	}
-	launchCfg.ConsoleLogAppend = appendConsole
 
 	// An NBD export is waited on until it accepts connections; a block export
 	// is local to the host and only has to exist. An invalid request has no
@@ -607,13 +606,7 @@ func (m *Manager) startQEMU(instance *VM, appendConsole bool) error {
 			slog.Info("Replaced VM process exited", "instance", instance.ID, "pid", cmd.Process.Pid, "err", waitErr)
 			return
 		}
-		if waitErr != nil {
-			if m.deps.CrashHandler != nil {
-				m.deps.CrashHandler(instance, waitErr)
-			}
-		} else {
-			slog.Info("VM process exited cleanly", "instance", instance.ID)
-		}
+		m.HandleQEMUExit(instance, waitErr)
 	}()
 
 	pid := <-processChan
@@ -968,6 +961,7 @@ func (m *Manager) qmpHeartbeatPoll(instance *VM) bool {
 	if err == nil && qmpStatus.Status == qmpStatusShutdown &&
 		m.Status(instance) == StateRunning {
 		slog.InfoContext(ctx, "Guest powered itself off, ending QEMU", "instance", instance.ID)
+		instance.guestPoweroff.Store(true)
 		if _, quitErr := sendQMPCommand(ctx, instance.QMPClient,
 			qmp.QMPCommand{Execute: "quit"}, instance.ID); quitErr != nil {
 			slog.WarnContext(ctx, "QMP quit failed after guest shutdown",

spinifex/runtime/compute/vm/lifecycle_test.go

@@ -1042,7 +1042,7 @@ func TestStartQEMU_DirectBoot_PrimaryTapError(t *testing.T) {
 		ENIMac:       "02:aa:bb:cc:dd:01",
 	}
 
-	err := m.startQEMU(instance, false)
+	err := m.startQEMU(instance)
 
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "setup tap device")
@@ -1070,7 +1070,7 @@ func TestStartQEMU_DirectBoot_BridgeEnsureError(t *testing.T) {
 		ENIMac:       "02:aa:bb:cc:dd:04",
 	}
 
-	err := m.startQEMU(instance, false)
+	err := m.startQEMU(instance)
 
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "ensure IMDS bridge")
@@ -1105,7 +1105,7 @@ func TestStartQEMU_DirectBoot_ExtraENITapError(t *testing.T) {
 		},
 	}
 
-	err := m.startQEMU(instance, false)
+	err := m.startQEMU(instance)
 
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "extra ENI")
@@ -1126,7 +1126,7 @@ func TestStartQEMU_DirectBoot_MgmtTapError(t *testing.T) {
 		MgmtMAC:      "02:aa:bb:cc:dd:ff",
 	}
 
-	err := m.startQEMU(instance, false)
+	err := m.startQEMU(instance)
 
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "mgmt tap")
@@ -1152,7 +1152,7 @@ func TestStartQEMU_DiskBoot_MgmtTapCreated(t *testing.T) {
 	}
 	instance.EBSRequests.Requests = []EBSRequest{{Name: "vol-root", NBDURI: "nbd:unix:/tmp/root.sock", Boot: true}}
 
-	err := m.startQEMU(instance, false)
+	err := m.startQEMU(instance)
 
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "mgmt tap")
@@ -1174,7 +1174,7 @@ func TestStartQEMU_DirectBoot_InstanceTypeNotFound(t *testing.T) {
 		DirectBoot:   true,
 	}
 
-	err := m.startQEMU(instance, false)
+	err := m.startQEMU(instance)
 
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "x9.enormous")
@@ -1190,7 +1190,7 @@ func TestStartQEMU_BlockDeviceMissing(t *testing.T) {
 	instance := directBootInstance("i-blk-missing",
 		EBSRequest{Name: "vol-blk", BlockDevice: missing, BlockSizeBytes: 1 << 20})
 
-	err := m.startQEMU(instance, false)
+	err := m.startQEMU(instance)
 
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "block device not present for vol-blk")
@@ -2089,12 +2089,12 @@ func TestStartQEMU_DirectBoot_LaunchesMountedDataVolumeWithoutSavingIt(t *testin
 	second := nbdListener(t)
 
 	instance := directBootInstance("i-db-vol", EBSRequest{Name: "vol-data-a", NBDURI: first})
-	require.NoError(t, m.startQEMU(instance, false))
+	require.NoError(t, m.startQEMU(instance))
 
 	// A later launch from the same saved base, after the volume was remounted
 	// on a different endpoint.
 	instance.EBSRequests.Requests[0].NBDURI = second
-	require.NoError(t, m.startQEMU(instance, false))
+	require.NoError(t, m.startQEMU(instance))
 
 	got := launches()
 	require.Len(t, got, 2)
@@ -2136,7 +2136,7 @@ func TestStartQEMU_DirectBoot_LaunchesTheEndpointItProbed(t *testing.T) {
 		instance.EBSRequests.Mu.Unlock()
 		return orig(uri, timeout)
 	}
-	require.NoError(t, m.startQEMU(instance, false))
+	require.NoError(t, m.startQEMU(instance))
 
 	assert.Equal(t, []string{probedEP}, probed)
 	got := launches()
@@ -2151,7 +2151,7 @@ func TestStartQEMU_DirectBoot_NoVolumesLaunchesTheBase(t *testing.T) {
 	launches := withFakeQEMU(t)
 
 	instance := directBootInstance("i-db-novol")
-	require.NoError(t, m.startQEMU(instance, false))
+	require.NoError(t, m.startQEMU(instance))
 
 	got := launches()
 	require.Len(t, got, 1)
@@ -2176,7 +2176,7 @@ func TestStartQEMU_DirectBoot_InvalidDataRequestIsNotLaunched(t *testing.T) {
 			launches := withFakeQEMU(t)
 
 			instance := directBootInstance("i-db-bad", tt.req)
-			err := m.startQEMU(instance, false)
+			err := m.startQEMU(instance)
 
 			require.Error(t, err)
 			assert.Contains(t, err.Error(), tt.wantErr)
@@ -2197,7 +2197,7 @@ func TestStartQEMU_DiskBoot_DataVolumeStaysOnThePCIPath(t *testing.T) {
 		{Name: "vol-data-a", NBDURI: ep},
 	}
 	require.NoError(t, os.WriteFile(strings.TrimPrefix(instance.EBSRequests.Requests[0].NBDURI, "nbd:unix:"), nil, 0o600))
-	require.NoError(t, m.startQEMU(instance, false))
+	require.NoError(t, m.startQEMU(instance))
 
 	got := launches()
 	require.Len(t, got, 1)
@@ -2252,7 +2252,7 @@ func TestStartQEMU_DiskBoot_BootVolumeIsABlockdevNode(t *testing.T) {
 			req := tt.req(t)
 			instance := &VM{ID: "i-boot-blockdev", InstanceType: "t3.nano"}
 			instance.EBSRequests.Requests = []EBSRequest{req}
-			require.NoError(t, m.startQEMU(instance, false))
+			require.NoError(t, m.startQEMU(instance))
 
 			got := launches()
 			require.Len(t, got, 1)
@@ -2283,7 +2283,7 @@ func TestStartQEMU_DiskBoot_NoBootVolumeIsNotLaunched(t *testing.T) {
 
 			instance := &VM{ID: "i-no-boot", InstanceType: "t3.nano"}
 			instance.EBSRequests.Requests = tt.requests
-			err := m.startQEMU(instance, false)
+			err := m.startQEMU(instance)
 
 			require.Error(t, err)
 			assert.Contains(t, err.Error(), "no boot volume among the instance's EBS requests")

spinifex/runtime/compute/vm/reboot.go

@@ -52,7 +52,7 @@ const (
 // startRelaunchedQEMU starts the replacement process. A var so tests can stand
 // in for QEMU.
 var startRelaunchedQEMU = func(m *Manager, instance *VM) error {
-	return m.startQEMU(instance, true)
+	return m.startQEMU(instance)
 }
 
 // errRelaunchFailed marks a reboot that ended the old QEMU and could not start

spinifex/runtime/compute/vm/reboot_test.go

@@ -465,21 +465,18 @@ func TestThreadStatFields(t *testing.T) {
 	assert.Empty(t, threadStatFields("garbage"))
 }
 
-func TestExecute_ConsoleLogAppendOnlyWhenAsked(t *testing.T) {
+func TestExecute_ConsoleLogAppendOnEveryBoot(t *testing.T) {
 	for _, tc := range []struct {
-		name   string
-		cfg    Config
-		want   string
-		absent string
+		name string
+		cfg  Config
+		want string
 	}{
-		{"socket fresh", Config{SerialSocket: "/s", ConsoleLogPath: "/c"}, "logfile=/c", "logappend"},
-		{"socket reboot", Config{SerialSocket: "/s", ConsoleLogPath: "/c", ConsoleLogAppend: true}, "logfile=/c,logappend=on", ""},
-		{"file reboot", Config{MachineType: "microvm", ConsoleLogPath: "/c", ConsoleLogAppend: true}, "file,id=console0,path=/c,append=on", ""},
+		{"socket", Config{SerialSocket: "/s", ConsoleLogPath: "/c"}, "logfile=/c,logappend=on"},
+		{"file", Config{MachineType: "microvm", ConsoleLogPath: "/c"}, "file,id=console0,path=/c,append=on"},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			cfg := minimalConfig()
-			cfg.MachineType, cfg.SerialSocket = tc.cfg.MachineType, tc.cfg.SerialSocket
-			cfg.ConsoleLogPath, cfg.ConsoleLogAppend = tc.cfg.ConsoleLogPath, tc.cfg.ConsoleLogAppend
+			cfg.MachineType, cfg.SerialSocket, cfg.ConsoleLogPath = tc.cfg.MachineType, tc.cfg.SerialSocket, tc.cfg.ConsoleLogPath
 			cmd, err := cfg.Execute()
 			require.NoError(t, err)
 			var chardev string
@@ -489,9 +486,6 @@ func TestExecute_ConsoleLogAppendOnlyWhenAsked(t *testing.T) {
 				}
 			}
 			assert.Contains(t, chardev, tc.want)
-			if tc.absent != "" {
-				assert.NotContains(t, chardev, tc.absent)
-			}
 		})
 	}
 }

spinifex/runtime/compute/vm/record.go

@@ -106,6 +106,10 @@ type InstanceStatus struct {
 	EBSRequests       []EBSRequest   `json:"ebs_requests,omitempty"`
 	ENIAvailableSlots []int          `json:"eni_available_slots,omitempty"`
 	ENIAttachedByID   map[string]int `json:"eni_attached_by_id,omitempty"`
+
+	// UEFIVarStore is optional, so records written without it still decode
+	// and the bucket schema does not change.
+	UEFIVarStore *UEFIVarStoreRef `json:"uefi_var_store,omitempty"`
 }
 
 // Record projects a VM onto the persisted record. QMPClient and the request
@@ -166,6 +170,7 @@ func (v *VM) Record() *InstanceRecord {
 			EBSRequests:                     v.EBSRequests.Requests,
 			ENIAvailableSlots:               v.ENIRequests.AvailableSlots,
 			ENIAttachedByID:                 v.ENIRequests.AttachedByENIID,
+			UEFIVarStore:                    v.UEFIVarStore,
 		},
 	}
 }
@@ -222,6 +227,7 @@ func VMFromRecord(r *InstanceRecord) *VM {
 		TerminatedAt:                    r.Status.TerminatedAt,
 		ShuttingDownAt:                  r.Status.ShuttingDownAt,
 		FirstLaunch:                     r.Status.FirstLaunch,
+		UEFIVarStore:                    r.Status.UEFIVarStore,
 	}
 	v.EBSRequests.Requests = r.Status.EBSRequests
 	v.ENIRequests.AvailableSlots = r.Status.ENIAvailableSlots

spinifex/runtime/compute/vm/record_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/gpu"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
@@ -114,6 +115,37 @@ func TestRecord_RoundTripThroughJSON(t *testing.T) {
 	assertVMFieldsEqual(t, original, vm.VMFromRecord(&decoded))
 }
 
+// The pointer is optional: a record written before it existed decodes with
+// none, and a record without one writes no key, so the bucket schema holds.
+func TestRecord_UEFIVarStoreIsOptional(t *testing.T) {
+	var decoded vm.InstanceRecord
+	require.NoError(t, json.Unmarshal([]byte(`{"metadata":{"name":"i-old"},"spec":{"config":{}},"status":{"status":"stopped"}}`), &decoded))
+	assert.Nil(t, decoded.Status.UEFIVarStore)
+
+	encoded, err := json.Marshal((&vm.VM{ID: "i-bios"}).Record())
+	require.NoError(t, err)
+	assert.NotContains(t, string(encoded), "uefi_var_store")
+}
+
+// The pointer's wire names are what the record carries across releases, so
+// they are pinned rather than left to follow the Go field names.
+func TestRecord_UEFIVarStoreWireShape(t *testing.T) {
+	encoded, err := json.Marshal(populatedVM().Record())
+	require.NoError(t, err)
+
+	var wire struct {
+		Status struct {
+			UEFIVarStore map[string]json.RawMessage `json:"uefi_var_store"`
+		} `json:"status"`
+	}
+	require.NoError(t, json.Unmarshal(encoded, &wire))
+	ref := wire.Status.UEFIVarStore
+	assert.JSONEq(t, `"uefi/i-round-trip/0d6f3c1e-5b7a-4f43-9a8e-2c1d4e5f6a7b"`, string(ref["key"]))
+	assert.JSONEq(t, `{"arch":"arm64","vars_size":67108864,"code_sha256":"ab12"}`, string(ref["firmware"]))
+	assert.JSONEq(t, `"node-3"`, string(ref["writer_node"]))
+	assert.JSONEq(t, `"2026-08-28T04:00:00Z"`, string(ref["written_at"]))
+}
+
 // The mutexes are dropped on the way out, so the rebuilt instance must arrive
 // with usable zero-value locks rather than ones somebody else left held.
 func TestVMFromRecord_RequestWrappersAreUsable(t *testing.T) {
@@ -206,6 +238,12 @@ func populatedVM() *vm.VM {
 		CapacityReservationId: "cr-1",
 		InstanceLifecycle:     "spot",
 		SpotInstanceRequestId: "sir-1",
+		UEFIVarStore: &vm.UEFIVarStoreRef{
+			Key:        "uefi/i-round-trip/0d6f3c1e-5b7a-4f43-9a8e-2c1d4e5f6a7b",
+			Firmware:   uefistore.FirmwareIdentity{Arch: "arm64", VarsSize: 67108864, CodeSHA256: "ab12"},
+			WriterNode: "node-3",
+			WrittenAt:  time.Date(2026, 8, 28, 4, 0, 0, 0, time.UTC),
+		},
 
 		Status:                          vm.StateRunning,
 		Instance:                        &ec2.Instance{InstanceId: aws.String("i-round-trip")},

spinifex/runtime/compute/vm/shutdown.go

@@ -389,6 +389,8 @@ func (m *Manager) finalizeTerminated(instance *VM) error {
 		instance.TerminatedAt = time.Now()
 	}
 
+	retireConsoleLog(instance.Config.ConsoleLogPath)
+
 	if m.deps.StateStore != nil {
 		if err := m.deps.StateStore.WriteTerminatedInstance(instance.ID, instance); err != nil {
 			slog.Error("Failed to write terminated instance to KV, keeping in local state for retry",

spinifex/runtime/compute/vm/vm.go

@@ -19,6 +19,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/contracts/ec2/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/gpu"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
 	"github.com/mulgadc/spinifex/spinifex/runtime/host/hostos"
@@ -136,6 +137,10 @@ type VM struct {
 	// claiming the flag serialises reboots of one instance.
 	rebooting atomic.Bool `json:"-"`
 
+	// guestPoweroff is set once the heartbeat has seen the guest power itself off
+	// and asked QEMU to quit, so the exit that follows is a stop, not a crash.
+	guestPoweroff atomic.Bool `json:"-"`
+
 	// relaunchMu is held while a reboot swaps this guest's QEMU process, so a
 	// stop or terminate never unmounts volumes under a process being started.
 	relaunchMu sync.Mutex `json:"-"`
@@ -308,6 +313,20 @@ type VM struct {
 	// tolerated by migrate.
 	InstanceLifecycle     string `json:"instance_lifecycle,omitempty"`
 	SpotInstanceRequestId string `json:"spot_instance_request_id,omitempty"`
+
+	// UEFIVarStore points at the object holding this instance's UEFI variable
+	// store. Status, so written only by the node; nil for a BIOS instance.
+	UEFIVarStore *UEFIVarStoreRef `json:"uefi_var_store,omitempty"`
+}
+
+// UEFIVarStoreRef names the store object an instance launches from, with the
+// firmware it was written under, so a launch can check it without reading the
+// object's metadata.
+type UEFIVarStoreRef struct {
+	Key        string                     `json:"key"`
+	Firmware   uefistore.FirmwareIdentity `json:"firmware"`
+	WriterNode string                     `json:"writer_node"`
+	WrittenAt  time.Time                  `json:"written_at"`
 }
 
 // UnmarshalJSON folds the fields of legacy records that mixed a request with an
@@ -417,13 +436,10 @@ type Config struct {
 	NoGraphic          bool   `json:"no_graphic"`
 	MachineType        string `json:"machine_type"`
 	ConsoleLogPath     string `json:"console_log_path,omitempty"`
-	// ConsoleLogAppend keeps the existing console log rather than truncating
-	// it, for a reboot that replaces the process.
-	ConsoleLogAppend bool   `json:"-"`
-	SerialSocket     string `json:"serial_socket,omitempty"`
-	CPUType          string `json:"cpu_type"`
-	CPUCount         int    `json:"cpu_count"`
-	Memory           int    `json:"memory"`
+	SerialSocket       string `json:"serial_socket,omitempty"`
+	CPUType            string `json:"cpu_type"`
+	CPUCount           int    `json:"cpu_count"`
+	Memory             int    `json:"memory"`
 
 	Drives    []Drive    `json:"drives"`
 	IOThreads []IOThread `json:"io_threads,omitempty"`
@@ -537,11 +553,11 @@ func (cfg *Config) Execute() (*exec.Cmd, error) {
 		// -serial chardev:console0 wires it to the existing device (not a new one).
 		// Do NOT use -device isa-serial here — that adds a second device (ttyS1)
 		// while the kernel talks to ttyS0.
-		args = append(args, "-chardev", fmt.Sprintf("file,id=console0,path=%s%s", cfg.ConsoleLogPath, appendOpt(",append", cfg.ConsoleLogAppend)))
+		args = append(args, "-chardev", fmt.Sprintf("file,id=console0,path=%s,append=on", cfg.ConsoleLogPath))
 		args = append(args, "-serial", "chardev:console0")
 	} else if cfg.SerialSocket != "" && cfg.ConsoleLogPath != "" {
-		chardevOpts := fmt.Sprintf("socket,id=console0,path=%s,server=on,wait=off,logfile=%s%s",
-			cfg.SerialSocket, cfg.ConsoleLogPath, appendOpt(",logappend", cfg.ConsoleLogAppend))
+		chardevOpts := fmt.Sprintf("socket,id=console0,path=%s,server=on,wait=off,logfile=%s,logappend=on",
+			cfg.SerialSocket, cfg.ConsoleLogPath)
 		args = append(args, "-chardev", chardevOpts)
 		args = append(args, "-serial", "chardev:console0")
 	}
@@ -678,12 +694,3 @@ func (cfg *Config) Execute() (*exec.Cmd, error) {
 
 	return cmd, nil
 }
-
-// appendOpt renders a chardev append option, or nothing when off, so a
-// fresh launch keeps the command line it always had.
-func appendOpt(key string, on bool) string {
-	if !on {
-		return ""
-	}
-	return key + "=on"
-}

spinifex/runtime/compute/vm/vm_test.go

@@ -463,7 +463,7 @@ func TestExecute_MicrovmFileChardev(t *testing.T) {
 
 	args := cmd.Args[1:]
 	chardev := argValue(args, "-chardev")
-	assert.Equal(t, "file,id=console0,path=/var/log/console.log", chardev)
+	assert.Equal(t, "file,id=console0,path=/var/log/console.log,append=on", chardev)
 	// -serial chardev:console0 attaches to the isa-serial=on device (ttyS0).
 	assert.Equal(t, "chardev:console0", argValue(args, "-serial"))
 	// No explicit isa-serial device addition (would create ttyS1, not ttyS0).
@@ -699,6 +699,6 @@ func TestExecute_MicrovmWithPCIEFileChardev(t *testing.T) {
 	require.NoError(t, err)
 
 	args := cmd.Args[1:]
-	assert.Equal(t, "file,id=console0,path=/var/log/console.log", argValue(args, "-chardev"))
+	assert.Equal(t, "file,id=console0,path=/var/log/console.log,append=on", argValue(args, "-chardev"))
 	assert.Equal(t, "chardev:console0", argValue(args, "-serial"))
 }

spinifex/runtime/daemon/daemon.go

@@ -41,7 +41,6 @@ import (
 	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	operatorv1 "github.com/mulgadc/spinifex/contracts/operator/v1"
 	telemetryv1 "github.com/mulgadc/spinifex/contracts/telemetry/v1"
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/preflight"
 	acmdomain "github.com/mulgadc/spinifex/spinifex/domains/acm"
@@ -733,6 +732,11 @@ func (d *Daemon) SetConfigPath(path string) {
 
 // NewDaemon creates a new daemon instance.
 func NewDaemon(cfg *config.ClusterConfig) (*Daemon, error) {
+	// The fence notice is node-addressed: a daemon that cannot subscribe to it
+	// would leave a guest running on a volume whose export was revoked.
+	if err := ebsprovider.ValidateNodeID(cfg.Node); err != nil {
+		return nil, fmt.Errorf("cluster config node %q: %w", cfg.Node, err)
+	}
 	ctx, cancel := context.WithCancel(context.Background())
 
 	// If WalDir is not set, use BaseDir
@@ -936,6 +940,8 @@ func (d *Daemon) subscribeAll() error {
 		// Cluster-wide on purpose: the QMP detach is instance-scoped, so a
 		// volume held by an unreachable host has no other way out.
 		{"ec2.ForceDetachVolume", handleNATSRequest(d.node, d.volumeService.ForceDetachVolume), "spinifex-workers"},
+		{"ec2.DetachVolumeFromStopped", handleNATSRequest(d.node, d.instanceService.DetachVolumeFromStopped), "spinifex-workers"},
+		{"ec2.AttachVolumeToStopped", handleNATSRequest(d.node, d.instanceService.AttachVolumeToStopped), "spinifex-workers"},
 		{"ec2.DescribeVolumeStatus", handleNATSRequest(d.node, d.volumeService.DescribeVolumeStatus), "spinifex-workers"},
 		{"ec2.DescribeVolumesModifications", handleNATSRequest(d.node, d.volumeService.DescribeVolumesModifications), "spinifex-workers"},
 		{"ec2.CreateSnapshot", handleNATSRequest(d.node, d.snapshotService.CreateSnapshot), "spinifex-workers"},
@@ -1353,10 +1359,8 @@ func (d *Daemon) subscribeAll() error {
 		// here, so the reconcile request flows daemon-ward.
 		natsSub{dhcp.TopicLeaseChanged, d.handleDHCPLeaseChanged, "spinifex-workers"},
 		natsSub{dhcp.TopicOwnerCheck, d.handleDHCPOwnerCheck, "spinifex-workers"},
-		// Node-addressed with no queue group: the fenced guest is on this host,
-		// so a worker on another node would find nothing to stop.
-		natsSub{viperblocklegacyv1.VolumeFencedSubject(d.node), d.handleVolumeFenced, ""},
 	)
+	subs = append(subs, d.fenceNoticeSubs()...)
 
 	// Per-node, no queue group: a second daemon claiming this node ID also
 	// answers, so the teardown probe sees a duplicate and refuses to seal. An
@@ -2571,6 +2575,7 @@ func (d *Daemon) startCluster() error {
 			d.vmMgr.NewTerminatedTeardownReaper(),
 			d.vmMgr.NewOrphanQEMUReaper(),
 			d.vmMgr.NewStuckTerminateReaper(),
+			d.vmMgr.NewLocalConsoleLogReaper(),
 		}
 		if eniRec := d.newENIReconciler(); eniRec != nil {
 			reapers = append(reapers, eniRec)
@@ -2754,6 +2759,10 @@ func (d *Daemon) initJetStream() error {
 			err = d.jsManager.InitClusterConfigBucket()
 		}
 
+		if err == nil {
+			err = d.jsManager.InitUEFIStoreBucket()
+		}
+
 		if err == nil {
 			d.jsManager.SetSyncObserver(d)
 			// Replicas cannot be undeclared here: bucket creation refuses

spinifex/runtime/daemon/daemon_handlers_console.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
 	"github.com/nats-io/nats.go"
 )
 
@@ -49,6 +50,11 @@ func (d *Daemon) handleEC2GetConsoleOutput(msg *nats.Msg) {
 
 	if logPath != "" {
 		info, err := os.Stat(logPath)
+		if err != nil && os.IsNotExist(err) {
+			// A terminated instance's file is renamed, not removed, until it ages out.
+			logPath = vm.RetiredConsolePath(logPath)
+			info, err = os.Stat(logPath)
+		}
 		if err == nil {
 			modTime = info.ModTime()
 

spinifex/runtime/daemon/daemon_handlers_fence.go

@@ -3,56 +3,77 @@ package daemon
 import (
 	"context"
 	"encoding/json"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"fmt"
 	"log/slog"
 
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/nats-io/nats.go"
 )
 
-// handleVolumeFenced stops the local guest whose volume viperblockd has just
-// torn the export out from under.
+// fenceNoticeSubs is this node's fence-notice subscription, whatever the EBS
+// provider. Node-addressed with no queue group: the fenced guest is on this
+// host, so a worker on another node would find nothing to stop.
+func (d *Daemon) fenceNoticeSubs() []natsSub {
+	subject, err := ebsprovider.FenceNoticeSubject(d.node)
+	if err != nil {
+		slog.Error("Not serving EBS fence notices: node ID is not a single NATS subject token", "node", d.node, "err", err)
+		return nil
+	}
+	return []natsSub{{subject, d.handleFenceNotice, ""}}
+}
+
+// handleFenceNotice stops the local guest whose volume the EBS provider has
+// just revoked the export of.
 //
 // By the time this arrives the disk is already gone, so the guest is doing no
 // useful work and its writes cannot reach the backend. What is left is a QEMU
 // process holding memory and taps, and an instance record claiming this node
 // runs something that runs somewhere else now.
-func (d *Daemon) handleVolumeFenced(msg *nats.Msg) string {
-	ctx, span := natsmsg.StartConsumerSpan(msg)
+func (d *Daemon) handleFenceNotice(msg *nats.Msg) string {
+	ctx, span := ebsprovider.StartServerSpan(context.Background(), msg)
 	defer span.End()
 
-	var event viperblocklegacyv1.VolumeFencedEvent
-	if err := json.Unmarshal(msg.Data, &event); err != nil {
-		slog.ErrorContext(ctx, "volume fenced: bad event", "err", err)
-		ackVolumeFenced(ctx, msg, viperblocklegacyv1.VolumeFencedAck{Error: "bad event: " + err.Error()})
+	var notice ebsprovider.FenceNotice
+	decodeErr := json.Unmarshal(msg.Data, &notice)
+	ack := ebsprovider.FenceNoticeAck{Versioned: ebsprovider.ReplyVersioned(notice.SchemaVersion)}
+	refuse := func(code ebsprovider.ErrorCode, message string) string {
+		slog.ErrorContext(ctx, "volume fenced: refused the notice", "volumeId", notice.VolumeID, "code", code, "err", message)
+		ack.Error = &ebsprovider.ProviderError{Code: code, Message: message}
+		ebsprovider.RecordProviderError(span, ack.Error)
+		ackFenceNotice(ctx, msg, ack)
 		return outcomeError
 	}
-	if event.Volume == "" {
-		slog.ErrorContext(ctx, "volume fenced: event named no volume")
-		ackVolumeFenced(ctx, msg, viperblocklegacyv1.VolumeFencedAck{Error: "event named no volume"})
-		return outcomeError
+	if decodeErr != nil {
+		return refuse(ebsprovider.ErrorCodeInvalidArgument, "bad notice: "+decodeErr.Error())
+	}
+	if !ebsprovider.SupportsSchemaVersion(notice.SchemaVersion) {
+		return refuse(ebsprovider.ErrorCodeUnsupportedVersion,
+			fmt.Sprintf("unsupported schema version %d; ", notice.SchemaVersion)+ebsprovider.SupportedVersionsText())
+	}
+	if notice.VolumeID == "" {
+		return refuse(ebsprovider.ErrorCodeInvalidArgument, "notice named no volume")
 	}
 
 	// FenceVolume moves the guest out of running before it returns, so the ack
 	// below is sent only once nothing here will resume it.
-	instanceID := d.vmMgr.FenceVolume(ctx, event.Volume, event.Reason)
-	ackVolumeFenced(ctx, msg, viperblocklegacyv1.VolumeFencedAck{Instance: instanceID})
-	if instanceID == "" {
+	ack.InstanceID = d.vmMgr.FenceVolume(ctx, notice.VolumeID, notice.Reason)
+	ackFenceNotice(ctx, msg, ack)
+	if ack.InstanceID == "" {
 		// The usual case: the volume outlived the guest that was using it, so
 		// there is nothing local left to stop.
 		slog.InfoContext(ctx, "volume fenced with no local guest using it",
-			"volumeId", event.Volume, "winner", event.Winner)
+			"volumeId", notice.VolumeID, "winner", notice.Winner)
 		return outcomeSuccess
 	}
 
 	slog.ErrorContext(ctx, "volume fenced, stopped the guest that was using it",
-		"volumeId", event.Volume, "instanceId", instanceID, "winner", event.Winner)
+		"volumeId", notice.VolumeID, "instanceId", ack.InstanceID, "winner", notice.Winner)
 	return outcomeSuccess
 }
 
-// ackVolumeFenced answers viperblockd, which repeats the notice until it hears
+// ackFenceNotice answers the provider, which repeats the notice until it hears
 // back. A notice with no reply subject was published once and needs no answer.
-func ackVolumeFenced(ctx context.Context, msg *nats.Msg, ack viperblocklegacyv1.VolumeFencedAck) {
+func ackFenceNotice(ctx context.Context, msg *nats.Msg, ack ebsprovider.FenceNoticeAck) {
 	if msg.Reply == "" {
 		return
 	}
@@ -62,6 +83,6 @@ func ackVolumeFenced(ctx context.Context, msg *nats.Msg, ack viperblocklegacyv1.
 		return
 	}
 	if err := msg.Respond(payload); err != nil {
-		slog.WarnContext(ctx, "volume fenced: could not acknowledge, viperblockd will repeat the notice", "err", err)
+		slog.WarnContext(ctx, "volume fenced: could not acknowledge, the provider will repeat the notice", "err", err)
 	}
 }

spinifex/runtime/daemon/daemon_handlers_fence_invariants_test.go

@@ -2,92 +2,313 @@ package daemon
 
 import (
 	"encoding/json"
+	"fmt"
+	"sync"
 	"testing"
 	"time"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/codes"
 )
 
-// fenceDaemon is a daemon running one guest on volumeID, answering fence
-// notices on a real NATS server as it does in production.
+// fenceDaemon is a daemon running one guest on volumeID, subscribed to fence
+// notices on a real NATS server through the subscription it makes in production.
 func fenceDaemon(t *testing.T, volumeID string) (*Daemon, *vm.VM, *nats.Conn) {
+	t.Helper()
+	return fenceDaemonWithManager(t, volumeID, vm.NewManagerWithDeps(vm.Deps{NodeID: "node-a"}))
+}
+
+func fenceDaemonWithManager(t *testing.T, volumeID string, mgr *vm.Manager) (*Daemon, *vm.VM, *nats.Conn) {
 	t.Helper()
 	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
 
 	_, nc := testutil.StartTestNATS(t)
-	d := &Daemon{node: "node-a", vmMgr: vm.NewManagerWithDeps(vm.Deps{NodeID: "node-a"})}
-	guest := &vm.VM{
+	d := &Daemon{
+		node:              "node-a",
+		vmMgr:             mgr,
+		natsConn:          nc,
+		natsSubscriptions: map[string]*nats.Subscription{},
+	}
+	guest := fencedGuest(volumeID)
+	d.vmMgr.Insert(guest)
+
+	require.NoError(t, d.registerNatsSubs(d.fenceNoticeSubs()))
+	require.NoError(t, nc.Flush())
+	return d, guest, nc
+}
+
+func fencedGuest(volumeID string) *vm.VM {
+	return &vm.VM{
 		ID:          "i-fencedguest",
 		Status:      vm.StateRunning,
 		Instance:    &ec2.Instance{},
 		EBSRequests: vm.EBSRequests{Requests: []vm.EBSRequest{{Name: volumeID}}},
 	}
-	d.vmMgr.Insert(guest)
+}
 
-	_, err := nc.Subscribe(viperblocklegacyv1.VolumeFencedSubject(d.node), func(msg *nats.Msg) { d.handleVolumeFenced(msg) })
+// requestFenceNotice sends raw to node-a's fence-notice subject and decodes the ack.
+func requestFenceNotice(t *testing.T, nc *nats.Conn, raw []byte) ebsprovider.FenceNoticeAck {
+	t.Helper()
+	subject, err := ebsprovider.FenceNoticeSubject("node-a")
 	require.NoError(t, err)
-	require.NoError(t, nc.Flush())
-	return d, guest, nc
+	reply, err := nc.Request(subject, raw, 2*time.Second)
+	require.NoErrorf(t, err,
+		"ADR-0007:S4: the daemon did not answer a fence notice, so the provider can never tell it was heard. "+
+			"Fix: subscribe fenceNoticeSubs and respond in handleFenceNotice (daemon_handlers_fence.go)")
+	var ack ebsprovider.FenceNoticeAck
+	require.NoError(t, json.Unmarshal(reply.Data, &ack))
+	return ack
+}
+
+func fenceNoticePayload(t *testing.T, notice ebsprovider.FenceNotice) []byte {
+	t.Helper()
+	payload, err := json.Marshal(notice)
+	require.NoError(t, err)
+	return payload
+}
+
+// memoryProviderPublishing is a non-viperblock provider holding volumeID
+// published to node-a, announcing fences on nc.
+func memoryProviderPublishing(t *testing.T, nc *nats.Conn, volumeID string) *ebsprovider.MemoryProvider {
+	t.Helper()
+	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{})
+	provider.NotifyFencesOn(nc)
+	_, err := provider.CreateVolume(t.Context(), ebsprovider.CreateVolumeRequest{
+		Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, CapacityRange: ebsprovider.CapacityRange{RequiredBytes: 1 << 30},
+	})
+	require.NoError(t, err)
+	_, err = provider.PublishVolume(t.Context(), ebsprovider.PublishVolumeRequest{
+		Versioned: ebsprovider.NewVersioned(), VolumeID: volumeID, NodeID: "node-a",
+	})
+	require.NoError(t, err)
+	return provider
 }
 
 // TestS4_FenceNoticeIsAcknowledgedOnlyOnceTheGuestIsStopped enforces ADR-0007:S4
-// on the daemon's half of the fence. viperblockd repeats the notice until it is
+// on the daemon's half of the fence. The provider repeats the notice until it is
 // answered, so the answer is what ends delivery, and it must not come before the
 // guest is out of running, or a lost notice is acknowledged for a guest left up.
 func TestS4_FenceNoticeIsAcknowledgedOnlyOnceTheGuestIsStopped(t *testing.T) {
 	d, guest, nc := fenceDaemon(t, "vol-fencedack")
 
-	payload, err := json.Marshal(viperblocklegacyv1.VolumeFencedEvent{
-		Volume: "vol-fencedack", Node: "node-a", Winner: "node-b", Reason: "volume lease moved to node-b",
-	})
-	require.NoError(t, err)
-
-	reply, err := nc.Request(viperblocklegacyv1.VolumeFencedSubject(d.node), payload, 2*time.Second)
-	require.NoErrorf(t, err,
-		"ADR-0007:S4: the daemon did not answer a fence notice, so viperblockd can never tell it was heard. "+
-			"Fix: respond in handleVolumeFenced (daemon_handlers_fence.go)")
-
-	var ack viperblocklegacyv1.VolumeFencedAck
-	require.NoError(t, json.Unmarshal(reply.Data, &ack))
-	assert.Equal(t, guest.ID, ack.Instance, "the ack names the guest the fence stopped")
-	assert.Empty(t, ack.Error)
+	ack := requestFenceNotice(t, nc, fenceNoticePayload(t, ebsprovider.FenceNotice{
+		Versioned: ebsprovider.NewVersioned(),
+		VolumeID:  "vol-fencedack", NodeID: "node-a", Winner: "node-b", Reason: "volume lease moved to node-b",
+	}))
+	assert.Equal(t, guest.ID, ack.InstanceID, "the ack names the guest the fence stopped")
+	assert.Nil(t, ack.Error)
 	assert.NotEqual(t, vm.StateRunning, d.vmMgr.Status(guest),
 		"ADR-0007:S4: the fence was acknowledged while the guest was still running")
 }
 
+// TestS4_FenceNoticeIsNotAcknowledgedWhileTheStopIsInProgress enforces
+// ADR-0007:S4 on the order of the stop and the ack: with the stop held open,
+// no answer may reach the provider until it completes.
+func TestS4_FenceNoticeIsNotAcknowledgedWhileTheStopIsInProgress(t *testing.T) {
+	entered := make(chan struct{})
+	release := make(chan struct{})
+	var releaseOnce sync.Once
+	t.Cleanup(func() { releaseOnce.Do(func() { close(release) }) })
+
+	var mgr *vm.Manager
+	mgr = vm.NewManagerWithDeps(vm.Deps{NodeID: "node-a", TransitionState: func(v *vm.VM, target vm.InstanceState) error {
+		close(entered)
+		<-release
+		mgr.Inspect(v, func(v *vm.VM) { v.Status = target })
+		return nil
+	}})
+	_, guest, nc := fenceDaemonWithManager(t, "vol-fenceheld", mgr)
+
+	subject, err := ebsprovider.FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	type answer struct {
+		msg *nats.Msg
+		err error
+	}
+	answers := make(chan answer, 1)
+	payload := fenceNoticePayload(t, ebsprovider.FenceNotice{Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-fenceheld", NodeID: "node-a"})
+	go func() {
+		msg, err := nc.Request(subject, payload, 10*time.Second)
+		answers <- answer{msg, err}
+	}()
+
+	select {
+	case <-entered:
+	case <-time.After(5 * time.Second):
+		t.Fatal("the fence never reached the guest's stop, so this test proved nothing about the ack order")
+	}
+	select {
+	case got := <-answers:
+		t.Fatalf("ADR-0007:S4: the fence was acknowledged (%v) before the guest's stop finished. "+
+			"Fix: ack only after FenceVolume returns in handleFenceNotice (daemon_handlers_fence.go)", got.err)
+	case <-time.After(300 * time.Millisecond):
+	}
+
+	releaseOnce.Do(func() { close(release) })
+	var got answer
+	select {
+	case got = <-answers:
+	case <-time.After(5 * time.Second):
+		t.Fatal("the fence was never acknowledged once the stop finished")
+	}
+	require.NoError(t, got.err)
+	var ack ebsprovider.FenceNoticeAck
+	require.NoError(t, json.Unmarshal(got.msg.Data, &ack))
+	assert.Equal(t, guest.ID, ack.InstanceID)
+}
+
+// TestS4_FenceNoticeIsServedByTheProductionSubscriptionTable keeps the
+// subscription in subscribeAll: a handler nothing subscribes is a fence no
+// daemon hears.
+func TestS4_FenceNoticeIsServedByTheProductionSubscriptionTable(t *testing.T) {
+	t.Setenv("XDG_RUNTIME_DIR", t.TempDir())
+	ns, nc, _ := testutil.StartTestJetStream(t)
+
+	d := newSubscribeTestDaemon(t, nc, "", "")
+	d.node = "node-a"
+	d.vmMgr = vm.NewManagerWithDeps(vm.Deps{NodeID: "node-a"})
+	guest := fencedGuest("vol-fencedsubscribed")
+	d.vmMgr.Insert(guest)
+	require.NoError(t, d.subscribeAll())
+	require.NoError(t, nc.Flush())
+
+	caller, err := nats.Connect(ns.ClientURL())
+	require.NoError(t, err)
+	t.Cleanup(caller.Close)
+	subject, err := ebsprovider.FenceNoticeSubject("node-a")
+	require.NoError(t, err)
+	reply, err := caller.Request(subject, fenceNoticePayload(t, ebsprovider.FenceNotice{
+		Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-fencedsubscribed", NodeID: "node-a",
+	}), 2*time.Second)
+	require.NoErrorf(t, err,
+		"ADR-0007:S4: the running daemon does not serve fence notices. Fix: append fenceNoticeSubs in subscribeAll (daemon.go)")
+	var ack ebsprovider.FenceNoticeAck
+	require.NoError(t, json.Unmarshal(reply.Data, &ack))
+	assert.Nil(t, ack.Error)
+	assert.Equal(t, guest.ID, ack.InstanceID)
+}
+
+// TestNewDaemon_RefusesANodeTheFenceNoticeCannotAddress keeps a daemon from
+// starting where no fence notice can reach it, which would leave a guest
+// running on a revoked export with nothing but a log line to say so.
+func TestNewDaemon_RefusesANodeTheFenceNoticeCannotAddress(t *testing.T) {
+	for _, node := range []string{"", "node.a", "node-*", "node>"} {
+		t.Run(node, func(t *testing.T) {
+			d, err := NewDaemon(&config.ClusterConfig{Node: node, Nodes: map[string]config.Config{node: {}}})
+			require.ErrorIs(t, err, ebsprovider.ErrInvalidArgument)
+			assert.Contains(t, err.Error(), "cluster config node")
+			assert.Nil(t, d)
+		})
+	}
+}
+
+// TestFenceNotice_SpanAttributesTheAccountAndTheRefusal reads the daemon's
+// span for a refused notice: a server span for the fenced verb, carrying the
+// sender's account and the refusal's code rather than a successful round trip.
+func TestFenceNotice_SpanAttributesTheAccountAndTheRefusal(t *testing.T) {
+	recorder := testutil.RecordSpans(t)
+	_, _, nc := fenceDaemon(t, "vol-unused")
+
+	ctx := ebsprovider.WithAccountID(t.Context(), "123456789012")
+	_, err := ebsprovider.NotifyFenced(ctx, nc, ebsprovider.FenceNotice{
+		Versioned: ebsprovider.Versioned{SchemaVersion: ebsprovider.SchemaVersion + 1}, VolumeID: "vol-unused", NodeID: "node-a",
+	}, 2*time.Second)
+	require.ErrorIs(t, err, ebsprovider.ErrUnsupportedVersion)
+
+	_, server := testutil.SpansByKind(t, recorder)
+	assert.Equal(t, "ebs.volume.fenced", server.Name())
+	assert.Equal(t, codes.Error, server.Status().Code)
+	assert.Equal(t, string(ebsprovider.ErrorCodeUnsupportedVersion), testutil.SpanAttribute(t, server, "ebs.provider.error_code").AsString())
+	assert.Equal(t, "123456789012", testutil.SpanAttribute(t, server, "aws.account_id").AsString())
+}
+
+// TestFenceNotice_AckEchoesTheNoticeVersion keeps the answer readable by the
+// sender: an ack in a version other than the notice's is one it may refuse.
+func TestFenceNotice_AckEchoesTheNoticeVersion(t *testing.T) {
+	for _, version := range ebsprovider.SupportedSchemaVersions() {
+		_, _, nc := fenceDaemon(t, "vol-versioned")
+		ack := requestFenceNotice(t, nc, fenceNoticePayload(t, ebsprovider.FenceNotice{
+			Versioned: ebsprovider.Versioned{SchemaVersion: version}, VolumeID: "vol-versioned", NodeID: "node-a",
+		}))
+		assert.Nil(t, ack.Error)
+		assert.Equalf(t, version, ack.SchemaVersion, "a notice at version %d was answered at another", version)
+	}
+}
+
+// TestS4_FenceNoticeFromAnyProviderStopsTheGuest holds the daemon to the
+// contract rather than to viperblockd: a provider that is not viperblock
+// revoking the export a local guest runs on gets that guest stopped.
+func TestS4_FenceNoticeFromAnyProviderStopsTheGuest(t *testing.T) {
+	d, guest, nc := fenceDaemon(t, "vol-memoryfenced")
+	provider := memoryProviderPublishing(t, nc, "vol-memoryfenced")
+
+	stopped, err := provider.RevokePublication(t.Context(), "vol-memoryfenced", "node-b")
+	require.NoErrorf(t, err,
+		"ADR-0007:S4: the daemon did not take a fence notice from a non-viperblock provider. "+
+			"Fix: subscribe the contract fence-notice subject in fenceNoticeSubs (daemon_handlers_fence.go)")
+	assert.Equal(t, guest.ID, stopped)
+	assert.Equal(t, vm.StateError, d.vmMgr.Status(guest),
+		"ADR-0007:S4: the guest runs on against an export its provider revoked")
+}
+
+// TestFenceNotice_VolumeNoLocalGuestUsesChangesNothing is the ordinary case: a
+// volume fenced after its guest has gone leaves every local guest as it was.
+func TestFenceNotice_VolumeNoLocalGuestUsesChangesNothing(t *testing.T) {
+	d, guest, nc := fenceDaemon(t, "vol-inuse")
+	provider := memoryProviderPublishing(t, nc, "vol-noguest")
+
+	stopped, err := provider.RevokePublication(t.Context(), "vol-noguest", "node-b")
+	require.NoError(t, err, "a fence with nothing to stop is not an error")
+	assert.Empty(t, stopped)
+	assert.Equal(t, vm.StateRunning, d.vmMgr.Status(guest), "a guest on another volume must not be stopped")
+}
+
 // TestS4_RepeatedFenceNoticeIsHarmless covers the cost of the retry: the notice
 // can arrive more than once, and a second copy must leave the fenced guest as the
 // first did rather than re-driving it or failing.
 func TestS4_RepeatedFenceNoticeIsHarmless(t *testing.T) {
 	d, guest, nc := fenceDaemon(t, "vol-fencedtwice")
 
-	payload, err := json.Marshal(viperblocklegacyv1.VolumeFencedEvent{Volume: "vol-fencedtwice", Winner: "node-b"})
-	require.NoError(t, err)
-
+	payload := fenceNoticePayload(t, ebsprovider.FenceNotice{
+		Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-fencedtwice", NodeID: "node-a", Winner: "node-b",
+	})
 	for range 2 {
-		reply, err := nc.Request(viperblocklegacyv1.VolumeFencedSubject(d.node), payload, 2*time.Second)
-		require.NoError(t, err)
-		var ack viperblocklegacyv1.VolumeFencedAck
-		require.NoError(t, json.Unmarshal(reply.Data, &ack))
-		assert.Empty(t, ack.Error, "a repeated notice is not an error")
+		ack := requestFenceNotice(t, nc, payload)
+		assert.Nil(t, ack.Error, "a repeated notice is not an error")
 	}
 	assert.Equal(t, vm.StateError, d.vmMgr.Status(guest))
 }
 
-// TestVolumeFenced_BadEventIsAnsweredWithAnError stops a notice that can never be
-// acted on from being repeated forever.
-func TestVolumeFenced_BadEventIsAnsweredWithAnError(t *testing.T) {
-	d, _, nc := fenceDaemon(t, "vol-unused")
-
-	reply, err := nc.Request(viperblocklegacyv1.VolumeFencedSubject(d.node), []byte(`{"volume":""}`), 2*time.Second)
-	require.NoError(t, err)
-	var ack viperblocklegacyv1.VolumeFencedAck
-	require.NoError(t, json.Unmarshal(reply.Data, &ack))
-	assert.NotEmpty(t, ack.Error)
+// TestFenceNotice_UnactionableNoticeIsAnsweredWithAnError stops a notice that
+// can never be acted on from being repeated forever, and leaves the guest alone.
+func TestFenceNotice_UnactionableNoticeIsAnsweredWithAnError(t *testing.T) {
+	cases := map[string]struct {
+		raw  []byte
+		code ebsprovider.ErrorCode
+	}{
+		"undecodable":     {[]byte(`{`), ebsprovider.ErrorCodeInvalidArgument},
+		"no volume":       {[]byte(`{"schema_version":1,"volume_id":""}`), ebsprovider.ErrorCodeInvalidArgument},
+		"unknown version": {[]byte(`{"schema_version":0,"volume_id":"vol-unused"}`), ebsprovider.ErrorCodeUnsupportedVersion},
+		"future version":  {fmt.Appendf(nil, `{"schema_version":%d,"volume_id":"vol-unused"}`, ebsprovider.SchemaVersion+1), ebsprovider.ErrorCodeUnsupportedVersion},
+	}
+	for name, tc := range cases {
+		t.Run(name, func(t *testing.T) {
+			d, guest, nc := fenceDaemon(t, "vol-unused")
+			ack := requestFenceNotice(t, nc, tc.raw)
+			require.NotNil(t, ack.Error)
+			assert.Equal(t, tc.code, ack.Error.Code)
+			if tc.code == ebsprovider.ErrorCodeUnsupportedVersion {
+				assert.Contains(t, ack.Error.Message, "unsupported schema version ")
+				assert.Contains(t, ack.Error.Message, ebsprovider.SupportedVersionsText())
+			}
+			assert.Equal(t, vm.StateRunning, d.vmMgr.Status(guest))
+		})
+	}
 }

spinifex/runtime/daemon/daemon_handlers_instance_test.go

@@ -38,9 +38,13 @@ func daemonWithFakeStateStore(t *testing.T, store *vmmock.StateStore) *Daemon {
 	t.Helper()
 	d := createTestDaemon(t, sharedNATSURL)
 	d.stateStore = store
+	objects := objectstore.NewMemoryObjectStore()
+	for _, stopped := range store.Stopped {
+		seedBootVolume(t, objects, d.config.Predastore.Bucket, stopped.AccountID, stopped.ID)
+	}
 	d.instanceService = ec2instance.NewInstanceServiceImpl(
 		d.config, d.resourceMgr.instanceTypes, d.natsConn,
-		objectstore.NewMemoryObjectStore(), d.vmMgr, d.resourceMgr, store,
+		objects, d.vmMgr, d.resourceMgr, store,
 	)
 	return d
 }
@@ -71,7 +75,7 @@ func decodeError(t *testing.T, data []byte) map[string]any {
 
 // stoppedVMFixture builds a minimally-valid stopped VM for handler tests.
 func stoppedVMFixture(id, accountID string) *vm.VM {
-	return &vm.VM{
+	return withBootVolume(&vm.VM{
 		ID:           id,
 		Status:       vm.StateStopped,
 		InstanceType: "t3.micro",
@@ -84,7 +88,7 @@ func stoppedVMFixture(id, accountID string) *vm.VM {
 			InstanceId:   aws.String(id),
 			InstanceType: aws.String("t3.micro"),
 		},
-	}
+	})
 }
 
 // --- handleEC2StartStoppedInstance ---

spinifex/runtime/daemon/daemon_handlers_test.go

@@ -1438,6 +1438,85 @@ func TestHandleEC2GetConsoleOutput(t *testing.T) {
 	assert.Contains(t, string(decoded), "Boot complete.")
 }
 
+func TestHandleEC2GetConsoleOutput_ReadsRetiredFileWhenLiveNameIsGone(t *testing.T) {
+	daemon := createFullTestDaemon(t, sharedNATSURL)
+
+	instanceID := "i-console-retired-001"
+	logPath := t.TempDir() + "/console-" + instanceID + ".log"
+	var body bytes.Buffer
+	for i := 0; body.Len() < 200<<10; i++ {
+		fmt.Fprintf(&body, "l%014d\n", i)
+	}
+	require.NoError(t, os.WriteFile(vm.RetiredConsolePath(logPath), body.Bytes(), 0o644))
+
+	daemon.vmMgr.Insert(&vm.VM{
+		ID:        instanceID,
+		Status:    vm.StateShuttingDown,
+		AccountID: testAccountID,
+		Config:    vm.Config{ConsoleLogPath: logPath},
+	})
+
+	topic := fmt.Sprintf("ec2.%s.GetConsoleOutput", instanceID)
+	sub, err := daemon.natsConn.Subscribe(topic, daemon.handleEC2GetConsoleOutput)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+	reqData, _ := json.Marshal(&ec2.GetConsoleOutputInput{InstanceId: aws.String(instanceID)})
+	reply, err := natsRequest(daemon.natsConn, topic, reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var output ec2.GetConsoleOutputOutput
+	require.NoError(t, json.Unmarshal(reply.Data, &output))
+	decoded, err := base64.StdEncoding.DecodeString(*output.Output)
+	require.NoError(t, err)
+	want := body.Bytes()[body.Len()-64*1024:]
+	decodedSize := len(decoded)
+	assert.Equal(t, 64*1024, decodedSize)
+	assert.True(t, bytes.Equal(want, decoded), "the newest 64 KiB of the retired file")
+}
+
+func TestHandleEC2GetConsoleOutput_AfterTrimReturnsNewest64KiB(t *testing.T) {
+	daemon := createFullTestDaemon(t, sharedNATSURL)
+
+	instanceID := "i-console-trim-001"
+	dir := t.TempDir()
+	logPath := dir + "/console-" + instanceID + ".log"
+	var body bytes.Buffer
+	for i := 0; body.Len() < 5<<20; i++ {
+		fmt.Fprintf(&body, "l%014d\n", i)
+	}
+	require.NoError(t, os.WriteFile(logPath, body.Bytes(), 0o644))
+
+	daemon.vmMgr.Insert(&vm.VM{
+		ID:        instanceID,
+		Status:    vm.StateRunning,
+		AccountID: testAccountID,
+		Config:    vm.Config{ConsoleLogPath: logPath},
+	})
+	reaper := vm.NewConsoleLogReaper(dir)
+	_, err := reaper.Sweep(context.Background())
+	require.NoError(t, err)
+	info, err := os.Stat(logPath)
+	require.NoError(t, err)
+	require.Equal(t, int64(1<<20), info.Size(), "the console must have been trimmed before it is read")
+
+	topic := fmt.Sprintf("ec2.%s.GetConsoleOutput", instanceID)
+	sub, err := daemon.natsConn.Subscribe(topic, daemon.handleEC2GetConsoleOutput)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+	reqData, _ := json.Marshal(&ec2.GetConsoleOutputInput{InstanceId: aws.String(instanceID)})
+	reply, err := natsRequest(daemon.natsConn, topic, reqData, 5*time.Second)
+	require.NoError(t, err)
+
+	var output ec2.GetConsoleOutputOutput
+	require.NoError(t, json.Unmarshal(reply.Data, &output))
+	decoded, err := base64.StdEncoding.DecodeString(*output.Output)
+	require.NoError(t, err)
+	want := body.Bytes()[body.Len()-64*1024:]
+	decodedSize := len(decoded)
+	assert.Equal(t, 64*1024, decodedSize)
+	assert.True(t, bytes.Equal(want, decoded), "the response is the newest 64 KiB of the original console")
+}
+
 func TestHandleEC2GetConsoleOutput_EmptyLog(t *testing.T) {
 	natsURL := sharedNATSURL
 	daemon := createFullTestDaemon(t, natsURL)
@@ -2763,7 +2842,7 @@ func TestHandleEC2DescribeInstanceTypes_NoFilter(t *testing.T) {
 func TestHandleEC2StartStoppedInstance_InstanceTypeNotAvailable(t *testing.T) {
 	daemon := createFullTestDaemonWithJetStream(t, sharedJSNATSURL)
 
-	stoppedVM := &vm.VM{
+	stoppedVM := withBootVolume(&vm.VM{
 		ID:           "i-start-badtype-001",
 		Status:       vm.StateStopped,
 		InstanceType: "z99.nonexistent",
@@ -2772,7 +2851,13 @@ func TestHandleEC2StartStoppedInstance_InstanceTypeNotAvailable(t *testing.T) {
 			InstanceId:   aws.String("i-start-badtype-001"),
 			InstanceType: aws.String("z99.nonexistent"),
 		},
-	}
+	})
+	objects := objectstore.NewMemoryObjectStore()
+	seedBootVolume(t, objects, daemon.config.Predastore.Bucket, testAccountID, stoppedVM.ID)
+	daemon.instanceService = ec2instance.NewInstanceServiceImpl(
+		daemon.config, daemon.resourceMgr.instanceTypes, daemon.natsConn,
+		objects, daemon.vmMgr, daemon.resourceMgr, daemon.jsManager,
+	)
 	err := daemon.jsManager.WriteStoppedInstance(stoppedVM.ID, stoppedVM)
 	require.NoError(t, err)
 	t.Cleanup(func() { _ = daemon.jsManager.DeleteStoppedInstance(stoppedVM.ID) })

spinifex/runtime/daemon/daemon_test.go

@@ -141,7 +141,7 @@ func createTestDaemon(t *testing.T, natsURL string) *Daemon {
 	// publish/unpublish over NATS using the test's connection.
 	daemon.vmMgr.SetDeps(vm.Deps{
 		NodeID:               daemon.node,
-		VolumeMounter:        newNATSMounter(daemon.natsConn, daemon.node, daemon.volumeService),
+		VolumeMounter:        newNATSMounter(daemon.natsConn, daemon.node),
 		VolumeStateUpdater:   daemon.volumeService,
 		DetachDelay:          daemon.detachDelay,
 		DeviceDeletedTimeout: daemon.deviceDeletedTimeout,
@@ -3322,7 +3322,7 @@ func TestVolumeMounterAdapter_UnmountOne_UnmountError(t *testing.T) {
 	fake.setUnpublish(func(string) ebsprovider.UnpublishVolumeResponse {
 		return unpublishRefusal(ebsprovider.ErrorCodeInternal, "seal volume: device busy")
 	})
-	adapter := newNATSMounter(daemon.natsConn, daemon.node, nil)
+	adapter := newNATSMounter(daemon.natsConn, daemon.node)
 
 	err := adapter.UnmountOne(t.Context(), "", viperblocklegacyv1.EBSRequest{Name: "vol-rollback-err"})
 	require.Error(t, err, "an unpublish refusal must propagate so DetachVolume can keep the volume attached")
@@ -3337,55 +3337,19 @@ func TestVolumeMounterAdapter_UnmountOne_RequestFailure(t *testing.T) {
 	require.NoError(t, err)
 	nc.Close()
 
-	adapter := newNATSMounter(nc, "node-1", nil)
+	adapter := newNATSMounter(nc, "node-1")
 	require.ErrorIs(t, adapter.UnmountOne(t.Context(), "", viperblocklegacyv1.EBSRequest{Name: "vol-timeout"}), nats.ErrConnectionClosed,
 		"a failed unpublish request must propagate as an error")
 }
 
-// recordingVolState records UpdateVolumeState calls so the bulk-unmount test can
-// assert which volumes transitioned to available.
-// accounts is recorded separately because the account is a key segment of the
-// document, not a label: an unmount that writes under anything but the
-// instance's own account leaves the volume stuck in-use, and the adapter only
-// logs the error.
-type recordingVolState struct {
-	mu       sync.Mutex
-	calls    []string
-	accounts []string
-}
-
-func (r *recordingVolState) UpdateVolumeState(accountID, volumeID, state, instanceID, attachmentDevice string) error {
-	r.mu.Lock()
-	defer r.mu.Unlock()
-	r.calls = append(r.calls, volumeID+":"+state)
-	r.accounts = append(r.accounts, accountID)
-	return nil
-}
-
-func (r *recordingVolState) snapshot() []string {
-	r.mu.Lock()
-	defer r.mu.Unlock()
-	return append([]string(nil), r.calls...)
-}
-
-func (r *recordingVolState) accountsSnapshot() []string {
-	r.mu.Lock()
-	defer r.mu.Unlock()
-	return append([]string(nil), r.accounts...)
-}
-
-var _ vm.VolumeStateUpdater = (*recordingVolState)(nil)
-
-// TestVolumeMounterAdapter_Unmount_SealFailureSkipsAvailable verifies the
-// teardown-path durability gate: a per-volume seal failure during the bulk
-// Unmount (stop/terminate) must NOT transition that volume to available — a
-// reattach on a WAL-less node would find no checkpoint (bad superblock) — while
-// other volumes still seal and the loop completes. The failure is returned, not
-// swallowed: the caller decides whether losing that seal is survivable.
-func TestVolumeMounterAdapter_Unmount_SealFailureSkipsAvailable(t *testing.T) {
+// TestVolumeMounterAdapter_Unmount_SealFailureIsReported
+// verifies that a per-volume seal failure during the bulk Unmount (stop/crash)
+// is returned, naming the volume, while the other volumes still seal and the
+// loop completes. Unmount only seals: it never writes volume state, so every
+// volume stays attached to the stopped instance.
+func TestVolumeMounterAdapter_Unmount_SealFailureIsReported(t *testing.T) {
 	daemon := createTestDaemon(t, sharedNATSURL)
-	volState := &recordingVolState{}
-	adapter := newNATSMounter(daemon.natsConn, daemon.node, volState)
+	adapter := newNATSMounter(daemon.natsConn, daemon.node)
 	fake := serveFakeNodeProvider(t, daemon.natsConn, daemon.node)
 	fake.setUnpublish(func(volumeID string) ebsprovider.UnpublishVolumeResponse {
 		if volumeID == "vol-fail" {
@@ -3403,22 +3367,15 @@ func TestVolumeMounterAdapter_Unmount_SealFailureSkipsAvailable(t *testing.T) {
 	err := adapter.Unmount(t.Context(), inst)
 	require.Error(t, err, "bulk Unmount must report a per-volume seal failure")
 	assert.Contains(t, err.Error(), "vol-fail", "the error must name the volume that did not seal")
-
-	calls := volState.snapshot()
-	assert.NotContains(t, calls, "vol-fail:available",
-		"a failed seal must not transition the volume to available")
-	assert.Contains(t, calls, "vol-ok:available",
-		"a successful seal must still transition the volume to available")
 }
 
-// TestVolumeMounterAdapter_Unmount_NotFoundFlipsToAvailable verifies that an
-// unpublish answered not_found, which means the provider holds no export of
-// the volume, still flips a non-boot data volume to available, while the
-// boot/EFI gate stays intact.
-func TestVolumeMounterAdapter_Unmount_NotFoundFlipsToAvailable(t *testing.T) {
+// TestVolumeMounterAdapter_Unmount_NotFoundIsASealedOutcome
+// verifies that an unpublish answered not_found, which means the provider holds
+// no export of the volume, is not an error and, like every Unmount outcome,
+// writes no volume state.
+func TestVolumeMounterAdapter_Unmount_NotFoundIsASealedOutcome(t *testing.T) {
 	daemon := createTestDaemon(t, sharedNATSURL)
-	volState := &recordingVolState{}
-	adapter := newNATSMounter(daemon.natsConn, daemon.node, volState)
+	adapter := newNATSMounter(daemon.natsConn, daemon.node)
 	fake := serveFakeNodeProvider(t, daemon.natsConn, daemon.node)
 	fake.setUnpublish(func(volumeID string) ebsprovider.UnpublishVolumeResponse {
 		return unpublishRefusal(ebsprovider.ErrorCodeNotFound, fmt.Sprintf("Volume %s not found", volumeID))
@@ -3437,14 +3394,6 @@ func TestVolumeMounterAdapter_Unmount_NotFoundFlipsToAvailable(t *testing.T) {
 	require.NoError(t, adapter.Unmount(t.Context(), inst))
 	require.NoError(t, adapter.UnmountOne(t.Context(), "000000000042", viperblocklegacyv1.EBSRequest{Name: "vol-data-retry"}),
 		"a hot detach of a volume the provider no longer holds is already done")
-
-	calls := volState.snapshot()
-	assert.Contains(t, calls, "vol-data-retry:available",
-		"a not_found unpublish must still flip a non-boot data volume to available")
-	assert.NotContains(t, calls, "vol-boot-retry:available",
-		"a boot volume must stay attached even when the unpublish reports not_found")
-	assert.Equal(t, []string{"000000000042"}, volState.accountsSnapshot(),
-		"the state write must be keyed on the instance's own account, not its ID")
 }
 
 // TestVolumeMounterAdapter_Mount_PartialFailureRollback verifies that when
@@ -3461,7 +3410,7 @@ func TestVolumeMounterAdapter_Mount_PartialFailureRollback(t *testing.T) {
 		}
 		return publishedNBD("nbd://mounted-" + volumeID)
 	})
-	adapter := newNATSMounter(nc, node, nil)
+	adapter := newNATSMounter(nc, node)
 
 	err := adapter.Mount(t.Context(), mounterTestVM("vol-1", "vol-2", "vol-3"))
 	require.Error(t, err, "Mount should propagate the vol-2 failure")
@@ -3474,7 +3423,7 @@ func TestVolumeMounterAdapter_Mount_PartialFailureRollback(t *testing.T) {
 // the failing volume is torn down along with the one already published.
 func TestVolumeMounterAdapter_Mount_UndecodableReplyRollsBackBoth(t *testing.T) {
 	daemon := createTestDaemon(t, sharedNATSURL)
-	adapter := newNATSMounter(daemon.natsConn, daemon.node, nil)
+	adapter := newNATSMounter(daemon.natsConn, daemon.node)
 
 	publishSubject, err := ebsprovider.PublishSubject(daemon.node)
 	require.NoError(t, err)
@@ -3523,7 +3472,7 @@ func TestVolumeMounterAdapter_Mount_UndecodableReplyRollsBackBoth(t *testing.T)
 // attached without surfacing the leak to the caller.
 func TestVolumeMounterAdapter_Mount_RollbackFailurePropagates(t *testing.T) {
 	daemon := createTestDaemon(t, sharedNATSURL)
-	adapter := newNATSMounter(daemon.natsConn, daemon.node, nil)
+	adapter := newNATSMounter(daemon.natsConn, daemon.node)
 	fake := serveFakeNodeProvider(t, daemon.natsConn, daemon.node)
 	fake.setPublish(func(volumeID string) ebsprovider.PublishVolumeResponse {
 		if volumeID == "vol-2" {

spinifex/runtime/daemon/instance_recovery_coverage_test.go

@@ -15,9 +15,11 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
 	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/clustersize"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/cache"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
 	"github.com/nats-io/nats.go"
@@ -30,6 +32,7 @@ import (
 type recoveryStore struct {
 	m        *JetStreamManager
 	liveness *instancecache.Liveness
+	objects  *objectstore.MemoryObjectStore
 }
 
 func newRecoveryStore(t *testing.T) *recoveryStore {
@@ -55,7 +58,7 @@ func newRecoveryStore(t *testing.T) *recoveryStore {
 		_ = m.js.DeleteKeyValue(context.Background(), name)
 		_ = m.js.DeleteKeyValue(context.Background(), name+"-hb")
 	})
-	return &recoveryStore{m: m, liveness: instancecache.NewLiveness(m.js, beats)}
+	return &recoveryStore{m: m, liveness: instancecache.NewLiveness(m.js, beats), objects: objectstore.NewMemoryObjectStore()}
 }
 
 // beat writes node's heartbeat stamped at.
@@ -70,6 +73,8 @@ func (s *recoveryStore) put(t *testing.T, id, node, instanceType string) {
 	record := runningOn(id, node)
 	record.Spec.InstanceType = instanceType
 	record.Status.Instance = &ec2.Instance{InstanceId: aws.String(id)}
+	record.Status.EBSRequests = []vm.EBSRequest{{Name: bootVolumeID(id), Boot: true, DeviceName: ebsmetadata.RootDeviceName}}
+	seedBootVolume(t, s.objects, "test-bucket", record.Metadata.AccountID, id)
 	require.NoError(t, s.m.records.Set(context.Background(), instanceRecordKey(id), record))
 }
 
@@ -92,6 +97,7 @@ func recoveryDaemon(t *testing.T, s *recoveryStore) (*Daemon, *instanceRecovery)
 
 	d := createTestDaemon(t, sharedNATSURL)
 	d.jsManager = s.m
+	d.instanceService = ec2instance.NewInstanceServiceImpl(d.config, d.resourceMgr.instanceTypes, d.natsConn, s.objects, d.vmMgr, d.resourceMgr, nil)
 	d.predastoreHealth.at = time.Now()
 	d.predastoreHealth.result = predastoreHealthOK
 
@@ -251,7 +257,7 @@ func TestRecoveryAttemptSucceeds(t *testing.T) {
 	// starting QEMU, which is the one part of a recovery this cannot do.
 	d.vmMgr.SetDeps(vm.Deps{
 		NodeID:             d.node,
-		VolumeMounter:      newNATSMounter(d.natsConn, d.node, d.volumeService),
+		VolumeMounter:      newNATSMounter(d.natsConn, d.node),
 		VolumeStateUpdater: d.volumeService,
 		Hooks: vm.ManagerHooks{BeforeInstanceRelaunch: func(v *vm.VM) error {
 			d.vmMgr.Inspect(v, func(v *vm.VM) { v.Status = vm.StateShuttingDown })

spinifex/runtime/daemon/jetstream.go

@@ -11,6 +11,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	clusterv1 "github.com/mulgadc/spinifex/contracts/cluster/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
@@ -88,6 +89,7 @@ type JetStreamManager struct {
 	termRecords   *kvstore.Store[vm.InstanceRecord] // i.<id>, terminated bucket
 	clusterKV     jetstream.KeyValue                // spinifex-cluster-state
 	clusterConfig *kvstore.Bucket                   // spinifex-cluster-config
+	uefiStores    *uefistore.Store                  // spinifex-uefi-stores
 	obs           KVSyncObserver
 	running       runningSetState
 }
@@ -232,6 +234,18 @@ func (m *JetStreamManager) InitClusterConfigBucket() error {
 	return err
 }
 
+// InitUEFIStoreBucket opens the UEFI store-record bucket, creating it if it
+// doesn't exist, so a bucket that cannot be created fails startup.
+func (m *JetStreamManager) InitUEFIStoreBucket() error {
+	m.uefiStores = uefistore.NewStore(m.js)
+	return m.uefiStores.Open(context.Background())
+}
+
+// UEFIStores returns the store-record bucket opened by InitUEFIStoreBucket.
+func (m *JetStreamManager) UEFIStores() *uefistore.Store {
+	return m.uefiStores
+}
+
 // InitTerminatedInstanceBucket initializes the terminated-instances KV bucket with a 1-hour TTL.
 // JetStream automatically purges keys after 1 hour, matching AWS behavior for terminated instances.
 func (m *JetStreamManager) InitTerminatedInstanceBucket() error {

spinifex/runtime/daemon/stopped_root_fixture_test.go

@@ -0,0 +1,42 @@
+package daemon
+
+import (
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/stretchr/testify/require"
+)
+
+// bootVolumeID is the root volume a fixture instance is given.
+func bootVolumeID(instanceID string) string { return "vol-root-" + instanceID }
+
+// withBootVolume gives an instance fixture the root volume every real instance
+// has: a boot request at the root device and the matching API mapping.
+func withBootVolume(v *vm.VM) *vm.VM {
+	id := bootVolumeID(v.ID)
+	v.EBSRequests.Requests = append(v.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{Name: id, Boot: true, DeviceName: ebsmetadata.RootDeviceName})
+	if v.Instance != nil {
+		v.Instance.BlockDeviceMappings = append(v.Instance.BlockDeviceMappings, &ec2.InstanceBlockDeviceMapping{
+			DeviceName: aws.String(ebsmetadata.RootDeviceName),
+			Ebs:        &ec2.EbsInstanceBlockDevice{VolumeId: aws.String(id)},
+		})
+	}
+	return v
+}
+
+// seedBootVolume registers the root volume document of an instance fixture, so
+// a start finds the volume its record names.
+func seedBootVolume(t *testing.T, store objectstore.ObjectStore, bucket, accountID, instanceID string) {
+	t.Helper()
+	require.NoError(t, ebsmetadata.NewStore(store, bucket).PutVolume(t.Context(), ebsmetadata.Volume{
+		VolumeID:         bootVolumeID(instanceID),
+		TenantID:         accountID,
+		State:            "in-use",
+		AttachedInstance: instanceID,
+	}))
+}

spinifex/runtime/daemon/uefi_store_bucket_test.go

@@ -0,0 +1,36 @@
+package daemon_test
+
+import (
+	"testing"
+
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/runtime/daemon"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// The daemon creates the store-record bucket at startup and hands out the
+// store it opened, so later readers share one handle.
+func TestInitUEFIStoreBucket_CreatesBucketAndServesTheStore(t *testing.T) {
+	_, nc, js := testutil.StartTestJetStream(t)
+	m, err := daemon.NewJetStreamManager(nc)
+	require.NoError(t, err)
+
+	require.NoError(t, m.InitUEFIStoreBucket())
+
+	_, err = js.KeyValue(t.Context(), uefistore.BucketName)
+	require.NoError(t, err, "init must create the bucket rather than leave it to first use")
+
+	rec := &uefistore.UEFIStoreRecord{InstanceID: "i-0123456789abcdef0", State: uefistore.StateLive}
+	rev, err := m.UEFIStores().Create(t.Context(), rec)
+	require.NoError(t, err)
+	got, gotRev, err := m.UEFIStores().Get(t.Context(), rec.InstanceID)
+	require.NoError(t, err)
+	assert.Equal(t, rev, gotRev)
+	assert.Equal(t, rec, got)
+
+	_, err = m.UEFIStores().Create(t.Context(), rec)
+	require.ErrorIs(t, err, kvstore.ErrExists)
+}

spinifex/runtime/daemon/vm_adapters.go

@@ -95,10 +95,9 @@ func (a *stateStoreAdapter) DeleteTerminatedInstance(id string) error {
 
 // volumeMounterAdapter satisfies vm.VolumeMounter over the EBS contract's
 // PublishVolume and UnpublishVolume, sent at the cluster's one contract
-// version to this node's provider. Unmount also marks volumes available.
+// version to this node's provider.
 type volumeMounterAdapter struct {
 	node     string
-	volState vm.VolumeStateUpdater
 	provider mounterProvider
 	// abandonTimeout bounds each no-seal unpublish Abandon sends.
 	abandonTimeout time.Duration
@@ -114,10 +113,9 @@ var _ mounterProvider = (*ebsprovider.NATSProvider)(nil)
 
 var _ vm.VolumeMounter = (*volumeMounterAdapter)(nil)
 
-func newVolumeMounterAdapter(provider mounterProvider, node string, volState vm.VolumeStateUpdater) *volumeMounterAdapter {
+func newVolumeMounterAdapter(provider mounterProvider, node string) *volumeMounterAdapter {
 	return &volumeMounterAdapter{
 		node:           node,
-		volState:       volState,
 		provider:       provider,
 		abandonTimeout: abandonTimeout,
 	}
@@ -296,27 +294,15 @@ func (a *volumeMounterAdapter) Unmount(ctx context.Context, instance *vm.VM) err
 
 	var sealErrs []error
 	for _, ebsRequest := range instance.EBSRequests.Requests {
-		// Every volume is attempted and the seal failures aggregated. An unsealed
-		// volume must NOT go available: a reattach on a node without the local
-		// WAL would find no checkpoint (bad superblock).
+		// Every volume is attempted and the seal failures aggregated. A stop or
+		// crash leaves every volume in-use and attached, so no state is written.
 		if err := a.unpublish(ctx, instance.AccountID, ebsRequest); err != nil {
-			slog.Error("Volume seal failed, leaving volume non-available",
+			slog.Error("Volume seal failed",
 				"instance", instance.ID, "volume", ebsRequest.Name, "err", err)
 			sealErrs = append(sealErrs, fmt.Errorf("seal %s: %w", ebsRequest.Name, err))
 			continue
 		}
 		slog.Info("Unmounted volume", "instance", instance.ID, "volume", ebsRequest.Name)
-
-		// Boot/root volumes must stay attached across stop/crash unmount: EFI is
-		// the wrong proxy for "boot" (BIOS-boot roots are Boot && !EFI), so gate on
-		// Boot too. Only DetachVolume and terminate release a boot volume; flipping
-		// it available here while the instance restarts splits the state record.
-		if !ebsRequest.EFI && !ebsRequest.Boot && a.volState != nil {
-			if err := a.volState.UpdateVolumeState(instance.AccountID, ebsRequest.Name, "available", "", ""); err != nil {
-				slog.Error("Failed to update volume state to available after unmount",
-					"volumeId", ebsRequest.Name, "err", err)
-			}
-		}
 	}
 
 	return errors.Join(sealErrs...)
@@ -677,7 +663,7 @@ func (d *Daemon) buildVMManagerDeps() vm.Deps {
 	return vm.Deps{
 		NodeID:             d.node,
 		StateStore:         d.stateStore,
-		VolumeMounter:      newVolumeMounterAdapter(d.ebsProvider, d.node, d.volumeService),
+		VolumeMounter:      newVolumeMounterAdapter(d.ebsProvider, d.node),
 		NetworkPlumber:     d.networkPlumber,
 		InstanceTypes:      newInstanceTypeResolverAdapter(d.resourceMgr),
 		Resources:          newResourceControllerAdapter(d.resourceMgr),

spinifex/runtime/daemon/vm_adapters_contract_test.go

@@ -31,8 +31,8 @@ func newMounterTestNode() string {
 }
 
 // newNATSMounter returns a mounter driving node's provider over nc.
-func newNATSMounter(nc *nats.Conn, node string, volState vm.VolumeStateUpdater) *volumeMounterAdapter {
-	return newVolumeMounterAdapter(ebsprovider.NewNATSProvider(nc, ebsProviderRequestTimeout), node, volState)
+func newNATSMounter(nc *nats.Conn, node string) *volumeMounterAdapter {
+	return newVolumeMounterAdapter(ebsprovider.NewNATSProvider(nc, ebsProviderRequestTimeout), node)
 }
 
 func connectMounterTest(t *testing.T) *nats.Conn {
@@ -247,8 +247,7 @@ func TestVolumeMounterAdapter_EveryRequestAtVersion3OverTheContract(t *testing.T
 	for _, subject := range []string{"ebs." + node + ".*", "ebs.mount", "ebs.unmount", "ebs.abandon"} {
 		legacySpies = append(legacySpies, spyOn(t, nc, subject))
 	}
-	volState := &recordingVolState{}
-	adapter := newNATSMounter(nc, node, volState)
+	adapter := newNATSMounter(nc, node)
 
 	instance := mounterTestVM("vol-a", "vol-b")
 	require.NoError(t, adapter.Mount(t.Context(), instance))
@@ -267,7 +266,6 @@ func TestVolumeMounterAdapter_EveryRequestAtVersion3OverTheContract(t *testing.T
 	for _, id := range []string{"vol-a", "vol-b"} {
 		assert.Equal(t, ebsprovider.VolumeStateAvailable, volumeState(t, provider, id), "unmount must unpublish %s", id)
 	}
-	assert.ElementsMatch(t, []string{"vol-a:available", "vol-b:available"}, volState.snapshot())
 
 	moved := mounterTestVM("vol-moved")
 	require.NoError(t, adapter.Mount(t.Context(), moved))
@@ -293,7 +291,7 @@ func TestVolumeMounterAdapter_RetryLaterRefusalBacksOffAndRecovers(t *testing.T)
 			node := newMounterTestNode()
 			fake := serveFakeNodeProvider(t, nc, node)
 			fake.setPublish(publishRefusal(code))
-			adapter := newNATSMounter(nc, node, nil)
+			adapter := newNATSMounter(nc, node)
 
 			instance := mounterTestVM("vol-relaunch")
 			err := adapter.Mount(t.Context(), instance)
@@ -321,7 +319,7 @@ func TestVolumeMounterAdapter_PermanentRefusalIsNotRetryable(t *testing.T) {
 			node := newMounterTestNode()
 			fake := serveFakeNodeProvider(t, nc, node)
 			fake.setPublish(publishRefusal(code))
-			adapter := newNATSMounter(nc, node, nil)
+			adapter := newNATSMounter(nc, node)
 
 			err := adapter.Mount(t.Context(), mounterTestVM("vol-refused"))
 			require.Error(t, err)
@@ -340,9 +338,9 @@ func TestVolumeMounterAdapter_HeldElsewhereReachesTheMountCaller(t *testing.T) {
 	serveMemoryProvider(t, nc, provider, owner)
 	serveMemoryProvider(t, nc, provider, other)
 
-	require.NoError(t, newNATSMounter(nc, owner, nil).Mount(t.Context(), mounterTestVM("vol-held")))
+	require.NoError(t, newNATSMounter(nc, owner).Mount(t.Context(), mounterTestVM("vol-held")))
 
-	err := newNATSMounter(nc, other, nil).Mount(t.Context(), mounterTestVM("vol-held"))
+	err := newNATSMounter(nc, other).Mount(t.Context(), mounterTestVM("vol-held"))
 	require.ErrorIs(t, err, ebsprovider.ErrVolumeHeldElsewhere)
 	var providerErr *ebsprovider.ProviderError
 	require.ErrorAs(t, err, &providerErr)
@@ -369,12 +367,10 @@ func TestVolumeMounterAdapter_UnpublishOutlastingRequestTimeoutCompletes(t *test
 		return publishedNBD("nbd://" + volumeID)
 	})
 
-	volState := &recordingVolState{}
-	adapter := newNATSMounter(nc, node, volState)
+	adapter := newNATSMounter(nc, node)
 	adapter.provider = ebsprovider.NewNATSProvider(nc, requestTimeout)
 
 	require.NoError(t, adapter.Unmount(t.Context(), mounterTestVM("vol-slow-seal")))
-	assert.Equal(t, []string{"vol-slow-seal:available"}, volState.snapshot())
 	require.NoError(t, adapter.UnmountOne(t.Context(), testAccountID, viperblocklegacyv1.EBSRequest{Name: "vol-slow-seal"}))
 
 	err := adapter.MountOne(t.Context(), testAccountID, &viperblocklegacyv1.EBSRequest{Name: "vol-slow-seal"})
@@ -388,7 +384,7 @@ func TestVolumeMounterAdapter_MountRollsBackOnFailure(t *testing.T) {
 	node := newMounterTestNode()
 	provider := newMemoryProvider(t, "vol-present")
 	serveMemoryProvider(t, nc, provider, node)
-	adapter := newNATSMounter(nc, node, nil)
+	adapter := newNATSMounter(nc, node)
 
 	err := adapter.Mount(t.Context(), mounterTestVM("vol-present", "vol-missing"))
 	require.ErrorIs(t, err, vm.ErrMountRetryable)
@@ -417,7 +413,7 @@ func TestVolumeMounterAdapter_BlockExportReplacesNBDURI(t *testing.T) {
 	adapter := newVolumeMounterAdapter(&stubMounterProvider{export: ebsprovider.VolumeExport{
 		Kind:  ebsprovider.ExportKindBlock,
 		Block: &ebsprovider.BlockExport{DevicePath: "/dev/sdx", SizeBytes: 1 << 30},
-	}}, "node-stub", nil)
+	}}, "node-stub")
 
 	req := &viperblocklegacyv1.EBSRequest{Name: "vol-block", NBDURI: "nbd://stale"}
 	require.NoError(t, adapter.MountOne(t.Context(), testAccountID, req))
@@ -473,7 +469,7 @@ func TestVolumeMounterAdapter_AmbiguousPublishIsTornDown(t *testing.T) {
 			node := newMounterTestNode()
 			fake := serveFakeNodeProvider(t, nc, node)
 			fake.setPublish(func(string) ebsprovider.PublishVolumeResponse { return tt.reply() })
-			adapter := newNATSMounter(nc, node, nil)
+			adapter := newNATSMounter(nc, node)
 			if tt.timeout > 0 {
 				adapter.provider = ebsprovider.NewNATSProvider(nc, tt.timeout)
 			}
@@ -507,7 +503,7 @@ func TestVolumeMounterAdapter_MountRollbackOutlivesCancelledCaller(t *testing.T)
 		}
 		return publishedNBD("nbd://" + volumeID)
 	})
-	adapter := newNATSMounter(nc, node, nil)
+	adapter := newNATSMounter(nc, node)
 
 	ctx, cancel := context.WithCancel(t.Context())
 	defer cancel()
@@ -537,7 +533,7 @@ func TestVolumeMounterAdapter_MountOneCancelledAfterSendIsAmbiguous(t *testing.T
 		<-release
 		return publishedNBD("nbd://" + volumeID)
 	})
-	adapter := newNATSMounter(nc, node, nil)
+	adapter := newNATSMounter(nc, node)
 
 	ctx, cancel := context.WithCancel(t.Context())
 	defer cancel()
@@ -562,7 +558,7 @@ func TestVolumeMounterAdapter_PublishAfterCallerGoneSendsNothing(t *testing.T) {
 	nc := connectMounterTest(t)
 	node := newMounterTestNode()
 	fake := serveFakeNodeProvider(t, nc, node)
-	adapter := newNATSMounter(nc, node, nil)
+	adapter := newNATSMounter(nc, node)
 
 	cancelled, cancel := context.WithCancel(t.Context())
 	cancel()
@@ -583,7 +579,7 @@ func TestVolumeMounterAdapter_ContractRequestsCarryTheAccount(t *testing.T) {
 	node := newMounterTestNode()
 	serveFakeNodeProvider(t, nc, node)
 	calls := spyOn(t, nc, "ebs.provider.v1."+node+".*")
-	adapter := newNATSMounter(nc, node, nil)
+	adapter := newNATSMounter(nc, node)
 
 	instance := mounterTestVM("vol-a")
 	require.NoError(t, adapter.Mount(t.Context(), instance))
@@ -611,7 +607,7 @@ func TestVolumeMounterAdapter_RequestSpanCarriesTheAccountUnderTheCaller(t *test
 	nc := connectMounterTest(t)
 	node := newMounterTestNode()
 	serveFakeNodeProvider(t, nc, node)
-	adapter := newNATSMounter(nc, node, nil)
+	adapter := newNATSMounter(nc, node)
 
 	ctx, caller := otel.Tracer("test").Start(t.Context(), "caller")
 	require.NoError(t, adapter.MountOne(ctx, testAccountID, &viperblocklegacyv1.EBSRequest{Name: "vol-acct"}))
@@ -638,8 +634,7 @@ func TestVolumeMounterAdapter_UnpublishOutlivesCallerContext(t *testing.T) {
 		sealed.Add(1)
 		return ebsprovider.UnpublishVolumeResponse{}
 	})
-	volState := &recordingVolState{}
-	adapter := newNATSMounter(nc, node, volState)
+	adapter := newNATSMounter(nc, node)
 
 	short, cancelShort := context.WithTimeout(t.Context(), sealTime/4)
 	defer cancelShort()
@@ -649,7 +644,6 @@ func TestVolumeMounterAdapter_UnpublishOutlivesCallerContext(t *testing.T) {
 	cancel()
 	require.NoError(t, adapter.Unmount(cancelled, mounterTestVM("vol-stop")))
 	assert.Equal(t, int32(2), sealed.Load())
-	assert.Equal(t, []string{"vol-stop:available"}, volState.snapshot())
 }
 
 // Abandon drops each export unsealed, at the version that carries NoSeal, and
@@ -671,7 +665,7 @@ func TestVolumeMounterAdapter_AbandonUnpublishesWithoutSealing(t *testing.T) {
 		return ebsprovider.UnpublishVolumeResponse{}
 	})
 	logs := captureSlogForTest(t)
-	adapter := newNATSMounter(nc, node, nil)
+	adapter := newNATSMounter(nc, node)
 
 	err := adapter.Abandon(t.Context(), mounterTestVM("vol-torn", "vol-absent", "vol-unknown", "vol-refused"), "superseded")
 	require.ErrorIs(t, err, ebsprovider.ErrUnavailable)
@@ -695,7 +689,7 @@ func TestVolumeMounterAdapter_AbandonUnpublishesWithoutSealing(t *testing.T) {
 // An abandon that finds no provider has not let go, so the caller is told.
 func TestVolumeMounterAdapter_AbandonNoResponderFails(t *testing.T) {
 	nc := connectMounterTest(t)
-	adapter := newNATSMounter(nc, newMounterTestNode(), nil)
+	adapter := newNATSMounter(nc, newMounterTestNode())
 
 	err := adapter.Abandon(t.Context(), mounterTestVM("vol-1", "vol-2"), "superseded")
 	require.ErrorIs(t, err, nats.ErrNoResponders)
@@ -710,7 +704,7 @@ func TestVolumeMounterAdapter_AbandonGivesUpWithinItsBound(t *testing.T) {
 	nc := connectMounterTest(t)
 	node := newMounterTestNode()
 	spyOn(t, nc, "ebs.provider.v1."+node+".*")
-	adapter := newNATSMounter(nc, node, nil)
+	adapter := newNATSMounter(nc, node)
 	adapter.abandonTimeout = bound
 
 	done := make(chan error, 1)
@@ -730,7 +724,7 @@ func TestVolumeMounterAdapter_AbandonSpanRecordsTheReason(t *testing.T) {
 	nc := connectMounterTest(t)
 	node := newMounterTestNode()
 	serveFakeNodeProvider(t, nc, node)
-	adapter := newNATSMounter(nc, node, nil)
+	adapter := newNATSMounter(nc, node)
 
 	require.NoError(t, adapter.Abandon(t.Context(), mounterTestVM("vol-moved"), "superseded"))
 

spinifex/runtime/daemon/vm_adapters_coverage_test.go

@@ -185,7 +185,7 @@ func TestBuildVMManagerDeps_BackingStoreReady(t *testing.T) {
 // A connection that is down refuses the publish before sending it, so restore
 // backs off and retries, and there is no export to tear down.
 func TestVolumeMounterAdapter_Mount_DisconnectedIsRetryable(t *testing.T) {
-	adapter := newNATSMounter(closedAdapterTestConn(t), "node-closed", nil)
+	adapter := newNATSMounter(closedAdapterTestConn(t), "node-closed")
 	instance := &vm.VM{ID: "i-mount-closed"}
 	instance.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-closed"}}
 
@@ -196,23 +196,11 @@ func TestVolumeMounterAdapter_Mount_DisconnectedIsRetryable(t *testing.T) {
 	require.ErrorIs(t, adapter.MountOne(t.Context(), testAccountID, &viperblocklegacyv1.EBSRequest{Name: "vol-hot"}), vm.ErrMountRetryable)
 }
 
-// recordingVolumeState records UpdateVolumeState calls and returns err.
-type recordingVolumeState struct {
-	err     error
-	updated []string
-}
-
-func (r *recordingVolumeState) UpdateVolumeState(_, volumeID, state, _, _ string) error {
-	r.updated = append(r.updated, volumeID+"="+state)
-	return r.err
-}
-
 func TestVolumeMounterAdapter_Unmount_Errors(t *testing.T) {
 	nc := connectAdapterTestNATS(t, sharedNATSURL)
 
-	t.Run("no responder fails the seal and leaves the volume", func(t *testing.T) {
-		volState := &recordingVolumeState{}
-		adapter := newNATSMounter(nc, uniqueAdapterID("node-unmount"), volState)
+	t.Run("no responder fails the seal for every volume", func(t *testing.T) {
+		adapter := newNATSMounter(nc, uniqueAdapterID("node-unmount"))
 		instance := &vm.VM{ID: "i-unmount-none"}
 		instance.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-a"}, {Name: "vol-b"}}
 
@@ -220,19 +208,16 @@ func TestVolumeMounterAdapter_Unmount_Errors(t *testing.T) {
 		require.ErrorIs(t, err, nats.ErrNoResponders)
 		assert.Contains(t, err.Error(), "vol-a")
 		assert.Contains(t, err.Error(), "vol-b", "every volume is attempted")
-		assert.Empty(t, volState.updated, "an unsealed volume must not go available")
 	})
 
-	t.Run("state update failure does not fail the unmount", func(t *testing.T) {
+	t.Run("a sealed unmount succeeds", func(t *testing.T) {
 		node := uniqueAdapterID("node-unmount")
 		serveFakeNodeProvider(t, nc, node)
-		volState := &recordingVolumeState{err: errInjected}
-		adapter := newNATSMounter(nc, node, volState)
+		adapter := newNATSMounter(nc, node)
 		instance := &vm.VM{ID: "i-unmount-state"}
 		instance.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-a"}}
 
 		require.NoError(t, adapter.Unmount(t.Context(), instance))
-		assert.Equal(t, []string{"vol-a=available"}, volState.updated)
 	})
 }
 

spinifex/runtime/daemon/vm_adapters_invariants_test.go

@@ -10,21 +10,19 @@ import (
 )
 
 // TestUnmountNeverReleasesBootVolume locks the stop/crash attachment-persistence
-// contract: Unmount seals the block map but must NOT mark a boot/root volume
-// "available". Unmount is driven by stop (runtime/compute/vm/shutdown.go) and crash recovery
-// (runtime/compute/vm/crash_recovery.go), where the instance keeps its attachment and restarts —
-// only DetachVolume and terminate release a boot volume. Releasing it here while
+// contract: Unmount seals the block map but never marks any attached volume
+// "available", boot or data. Unmount is driven by stop (runtime/compute/vm/shutdown.go)
+// and crash recovery (runtime/compute/vm/crash_recovery.go), where the instance keeps
+// its attachments and restarts; as on AWS, EBS root and data volumes persist across
+// stop. Only DetachVolume and terminate release a volume. Releasing one here while
 // the instance still owns it splits the volume-state record (describe-instances
-// "attached" vs describe-volumes "available"). EFI is not a
-// sufficient proxy for "boot": BIOS-boot roots are Boot && !EFI, so the gate must
-// also exclude Boot. Only non-boot data volumes may be released by Unmount.
+// "attached" vs describe-volumes "available").
 func TestUnmountNeverReleasesBootVolume(t *testing.T) {
 	daemon := createTestDaemon(t, sharedNATSURL)
-	volState := &recordingVolState{}
-	adapter := newNATSMounter(daemon.natsConn, daemon.node, volState)
+	adapter := newNATSMounter(daemon.natsConn, daemon.node)
 
 	// Stand in for the provider: every unpublish succeeds (sealed).
-	serveFakeNodeProvider(t, daemon.natsConn, daemon.node)
+	fake := serveFakeNodeProvider(t, daemon.natsConn, daemon.node)
 
 	inst := &vm.VM{
 		ID: "i-unmount-boot",
@@ -38,11 +36,7 @@ func TestUnmountNeverReleasesBootVolume(t *testing.T) {
 	}
 	require.NoError(t, adapter.Unmount(t.Context(), inst))
 
-	calls := volState.snapshot()
-	assert.NotContains(t, calls, "vol-bios-root:available",
-		"a BIOS-boot root volume (Boot && !EFI) must stay attached across stop/crash unmount, not be released to available")
-	assert.NotContains(t, calls, "vol-efi-root:available",
-		"an EFI-boot root volume must stay attached across stop/crash unmount")
-	assert.Contains(t, calls, "vol-data:available",
-		"a non-boot data volume is released to available by Unmount")
+	// Boot, EFI and data volumes alike are sealed and left attached: stop and
+	// crash unmount never release any attached volume.
+	assert.ElementsMatch(t, []string{"vol-bios-root", "vol-efi-root", "vol-data"}, fake.unpublished())
 }

spinifex/runtime/daemon/vm_adapters_stopped_volumes_test.go

@@ -0,0 +1,97 @@
+package daemon
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/contracts/ec2/v1"
+	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/qmp"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// A stopped or crashed instance still owns its data volumes, so Unmount seals
+// them and releases the node-local mount without touching their state.
+func TestUnmount_LeavesDataVolumesAttached(t *testing.T) {
+	daemon := createTestDaemon(t, sharedNATSURL)
+	adapter := newNATSMounter(daemon.natsConn, daemon.node)
+	fake := serveFakeNodeProvider(t, daemon.natsConn, daemon.node)
+
+	inst := &vm.VM{
+		ID:        "i-stop-keep",
+		AccountID: "000000000042",
+		EBSRequests: vm.EBSRequests{
+			Requests: []viperblocklegacyv1.EBSRequest{
+				{Name: "vol-root", Boot: true},
+				{Name: "vol-efi", Boot: true, EFI: true},
+				{Name: "vol-data-a", DeviceName: "/dev/sdf"},
+				{Name: "vol-data-b", DeviceName: "/dev/sdg"},
+			},
+		},
+	}
+	require.NoError(t, adapter.Unmount(t.Context(), inst))
+
+	assert.ElementsMatch(t, []string{"vol-root", "vol-efi", "vol-data-a", "vol-data-b"}, fake.unpublished(),
+		"every volume is still sealed on stop")
+}
+
+// The point of keeping the attachment: after the stopped instance's unmount,
+// another instance cannot take the volume, which is what AWS does.
+func TestStoppedInstanceVolume_CannotBeAttachedElsewhere(t *testing.T) {
+	daemon, store := createFullTestDaemonWithStore(t, sharedNATSURL)
+
+	const (
+		stoppedID = "i-stopped-holder"
+		otherID   = "i-wants-the-volume"
+		volumeID  = "vol-held-by-stopped"
+	)
+	seedVolumeConfig(t, daemon, store, ebsmetadata.Volume{
+		VolumeID:         volumeID,
+		CapacityGiB:      10,
+		State:            "in-use",
+		TenantID:         testAccountID,
+		AttachedInstance: stoppedID,
+		DeviceName:       "/dev/sdf",
+	})
+
+	serveFakeNodeProvider(t, daemon.natsConn, daemon.node)
+
+	adapter := newNATSMounter(daemon.natsConn, daemon.node)
+	stopped := &vm.VM{
+		ID:        stoppedID,
+		AccountID: testAccountID,
+		EBSRequests: vm.EBSRequests{
+			Requests: []viperblocklegacyv1.EBSRequest{{Name: volumeID, DeviceName: "/dev/sdf"}},
+		},
+	}
+	require.NoError(t, adapter.Unmount(t.Context(), stopped))
+
+	other := &vm.VM{
+		ID:           otherID,
+		Status:       vm.StateRunning,
+		AccountID:    testAccountID,
+		InstanceType: getTestInstanceType(t),
+		Instance:     &ec2.Instance{},
+		QMPClient:    &qmp.QMPClient{},
+	}
+	daemon.vmMgr.Insert(other)
+	sub, err := daemon.natsConn.Subscribe(fmt.Sprintf("ec2.cmd.%s", otherID), daemon.handleEC2Events)
+	require.NoError(t, err)
+	defer sub.Unsubscribe()
+
+	cmdData, err := json.Marshal(ec2v1.EC2InstanceCommand{
+		ID:               otherID,
+		Attributes:       ec2v1.EC2CommandAttributes{AttachVolume: true},
+		AttachVolumeData: &ec2v1.AttachVolumeData{VolumeID: volumeID},
+	})
+	require.NoError(t, err)
+	resp, err := natsRequest(daemon.natsConn, fmt.Sprintf("ec2.cmd.%s", otherID), cmdData, 5*time.Second)
+	require.NoError(t, err)
+	assert.Contains(t, string(resp.Data), "VolumeInUse")
+}

spinifex/runtime/daemon/vm_adapters_test.go

@@ -563,7 +563,7 @@ func TestInstanceCleanerAdapter_ReleaseAttachedENIs_ListInstanceENIsErrorTolerat
 
 func TestVolumeMounterAdapter_Mount_NoResponderIsRetryable(t *testing.T) {
 	daemon := createTestDaemon(t, sharedNATSURL)
-	adapter := newNATSMounter(daemon.natsConn, daemon.node, nil)
+	adapter := newNATSMounter(daemon.natsConn, daemon.node)
 
 	// No subscriber on the mount subject: viperblockd is still starting, so
 	// the request returns nats.ErrNoResponders, which must be retryable.

tests/e2e/single/lifecycle_test.go

@@ -16,9 +16,8 @@ var sshHealth harness.SSHHealth
 
 // runStopStart stops fix.Instance and waits for it to reach the
 // "stopped" state, then asserts that rebooting a stopped instance is
-// rejected with IncorrectInstanceState. Leaves the instance stopped so
-// Phase 7a / 7b can act against the stopped state. Maps to run-e2e.sh
-// ~1027–1053.
+// rejected with IncorrectInstanceState, then starts it again. Maps to
+// run-e2e.sh ~1027–1053.
 func runStopStart(t *testing.T, fix *Fixture) {
 	harness.Phase(t, "Single — Instance State Transitions (Stop)")
 
@@ -54,22 +53,21 @@ func runStopStart(t *testing.T, fix *Fixture) {
 	harness.WaitForInstanceState(t, fix.AWS, instanceID, "running")
 }
 
-// runAttachToStoppedError creates a fresh 10 GiB volume and attempts
-// to attach it to the (currently stopped) primary instance. The attach
-// must fail with IncorrectInstanceState. The test volume is deleted
-// in-line on success and via t.Cleanup on early failure. Maps to
-// run-e2e.sh ~1054–1068.
-func runAttachToStoppedError(t *testing.T, fix *Fixture) {
-	harness.Phase(t, "Single — Attach Volume to Stopped Instance (Error Path)")
+// runAttachToStopped attaches a fresh 10 GiB volume to the (stopped) primary
+// instance, as AWS allows: the attach edits the stopped record and claims the
+// volume in-use, a delete is refused while it is attached, and a detach
+// releases it to available. The volume is deleted in-line on success and via
+// t.Cleanup on early failure.
+func runAttachToStopped(t *testing.T, fix *Fixture) {
+	harness.Phase(t, "Single — Attach/Detach Volume on a Stopped Instance")
 
 	inst, _ := needInstance(t, fix)
 	instanceID := aws.StringValue(inst.InstanceId)
 	az := needAZ(t, fix)
 
-	// Stop the primary instance so the IncorrectInstanceState assertion has
-	// something to bite on, then restore it at end so sibling Test* see the
-	// canonical running row.
-	harness.Step(t, "stop-instances %s (precondition for 7a)", instanceID)
+	// Stop the primary instance, then restore it at end so sibling Test* see
+	// the canonical running row.
+	harness.Step(t, "stop-instances %s (precondition)", instanceID)
 	_, err := fix.AWS.EC2.StopInstances(&ec2.StopInstancesInput{
 		InstanceIds: []*string{aws.String(instanceID)},
 	})
@@ -92,13 +90,16 @@ func runAttachToStoppedError(t *testing.T, fix *Fixture) {
 	require.NotEmpty(t, stoppedVolID, "create-volume returned empty VolumeId")
 	harness.Detail(t, "test_volume", stoppedVolID)
 
-	// Best-effort cleanup if the negative assertion or anything else fails
-	// before we reach the explicit delete-volume below.
+	// Best-effort cleanup if anything fails before the explicit delete below.
 	cleaned := false
 	t.Cleanup(func() {
 		if cleaned {
 			return
 		}
+		_, _ = fix.AWS.EC2.DetachVolume(&ec2.DetachVolumeInput{
+			VolumeId:   aws.String(stoppedVolID),
+			InstanceId: aws.String(instanceID),
+		})
 		_, _ = fix.AWS.EC2.DeleteVolume(&ec2.DeleteVolumeInput{
 			VolumeId: aws.String(stoppedVolID),
 		})
@@ -106,16 +107,34 @@ func runAttachToStoppedError(t *testing.T, fix *Fixture) {
 
 	harness.WaitForVolumeState(t, fix.AWS, stoppedVolID, "available")
 
-	harness.Step(t, "attach-volume %s -> %s (should fail)", stoppedVolID, instanceID)
-	harness.ExpectError(t, "IncorrectInstanceState", func() error {
-		_, err := fix.AWS.EC2.AttachVolume(&ec2.AttachVolumeInput{
-			VolumeId:   aws.String(stoppedVolID),
-			InstanceId: aws.String(instanceID),
-			Device:     aws.String("/dev/sdg"),
+	harness.Step(t, "attach-volume %s -> %s (stopped)", stoppedVolID, instanceID)
+	_, err = fix.AWS.EC2.AttachVolume(&ec2.AttachVolumeInput{
+		VolumeId:   aws.String(stoppedVolID),
+		InstanceId: aws.String(instanceID),
+		Device:     aws.String("/dev/sdg"),
+	})
+	require.NoError(t, err, "attach-volume to a stopped instance")
+	vol := harness.WaitForVolumeState(t, fix.AWS, stoppedVolID, "in-use")
+	require.Len(t, vol.Attachments, 1, "attachments on %s", stoppedVolID)
+	require.Equal(t, instanceID, aws.StringValue(vol.Attachments[0].InstanceId), "attached instance")
+	require.Equal(t, "/dev/sdg", aws.StringValue(vol.Attachments[0].Device), "attached device")
+
+	harness.Step(t, "delete-volume %s while attached (should fail)", stoppedVolID)
+	harness.ExpectError(t, "VolumeInUse", func() error {
+		_, err := fix.AWS.EC2.DeleteVolume(&ec2.DeleteVolumeInput{
+			VolumeId: aws.String(stoppedVolID),
 		})
 		return err
 	})
 
+	harness.Step(t, "detach-volume %s from %s (stopped)", stoppedVolID, instanceID)
+	_, err = fix.AWS.EC2.DetachVolume(&ec2.DetachVolumeInput{
+		VolumeId:   aws.String(stoppedVolID),
+		InstanceId: aws.String(instanceID),
+	})
+	require.NoError(t, err, "detach-volume from a stopped instance")
+	harness.WaitForVolumeState(t, fix.AWS, stoppedVolID, "available")
+
 	harness.Step(t, "delete-volume %s", stoppedVolID)
 	_, err = fix.AWS.EC2.DeleteVolume(&ec2.DeleteVolumeInput{
 		VolumeId: aws.String(stoppedVolID),

tests/e2e/single/tests_test.go

@@ -135,8 +135,8 @@ func TestStopStart(t *testing.T) {
 	runStopStart(t, requireSingleNodeFixture(t))
 }
 
-func TestAttachToStoppedError(t *testing.T) {
-	runAttachToStoppedError(t, requireSingleNodeFixture(t))
+func TestAttachToStopped(t *testing.T) {
+	runAttachToStopped(t, requireSingleNodeFixture(t))
 }
 
 // TestSpotInstanceLifecycle drives the two spot terminal transitions that need