test: wait on daemon signals in TestDaemonReconnectBumpsRetryCount (#1675)

The test waited for the NATS client to report disconnected, but nats.go marks the client reconnecting before it runs the disconnect callback, so the daemon could still report cluster mode and the count was read before any reconnect. Wait for standalone mode, then for the retry count, then assert cluster mode.

.github/scripts/suite-sets.sh

@@ -40,7 +40,11 @@
 # received. It is last because it SIGSTOPs a cluster-wide
 # service: anything sharing the environment would see the same outage and report
 # a failure that says nothing about itself.
-E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota route53 storagefault"
+# uefistore is single-only because it runs on the node it tests: it restarts
+# that node's daemon under a running guest and reads the variable store with
+# the node's own Predastore credentials. Guests outlive a daemon restart, so
+# suites sharing the node see a brief control-plane gap, not an outage.
+E2E_SUITES_SINGLE="single iam cert eks ecs storagegrowth partialblock rds quota route53 uefistore storagefault"
 
 # Suites runnable against a multi-node environment.
 #
@@ -195,6 +199,10 @@ e2e_suite_timeout() {
     # took 24-29m, and an Oracle host ran out at 30m inside the last test's
     # cleanup. The default left it no headroom at all.
     single) echo "40m" ;;
+    # uefistore's prune test has to outwait the 10-minute prune age and a GC
+    # sweep after its last write-back, on top of three stop/starts and a boot.
+    # A pass is estimated at 40 minutes, unmeasured; the rest is the watch's cap.
+    uefistore) echo "60m" ;;
     *) echo "30m" ;;
   esac
 }

.github/workflows/actions.lock

@@ -65,7 +65,7 @@ workflows:
         - 'actions/setup-go@v7.0.0'
         - 'actions/upload-artifact@v7.0.2'
         - 'docker/setup-buildx-action@v4.4.1'
-        - 'softprops/action-gh-release@v3.0.3'
+        - 'softprops/action-gh-release@v3.0.4'
     '.github/workflows/segscan-oracle.yml':
         - 'actions/checkout@v7.0.1'
         - 'actions/create-github-app-token@v3.3.0'
@@ -79,7 +79,7 @@ workflows:
         - 'actions/checkout@v7.0.1'
         - 'actions/setup-node@v7.1.0'
         - 'actions/upload-artifact@v7.0.2'
-        - 'pnpm/action-setup@v6.1.0'
+        - 'pnpm/action-setup@v6.1.1'
 dependencies:
     'actions/cache@v6.1.0':
         ref: 'v6.1.0'
@@ -131,13 +131,13 @@ dependencies:
         commit: 'sha1-dfe3c3f87815947d99a8997f908cb6525fc44e9e'
         owner_id: 761456
         repo_id: 258271226
-    'pnpm/action-setup@v6.1.0':
-        ref: 'v6.1.0'
-        commit: 'sha1-ea17c68df8912ef543352723c149a84f56e3d413'
+    'pnpm/action-setup@v6.1.1':
+        ref: 'v6.1.1'
+        commit: 'sha1-73e46992e05b19abd31f2f3c20e0e7370d762115'
         owner_id: 21320719
         repo_id: 262289747
-    'softprops/action-gh-release@v3.0.3':
-        ref: 'v3.0.3'
-        commit: 'sha1-efb35369e0ad2afab669f228072c1b0d510eae64'
+    'softprops/action-gh-release@v3.0.4':
+        ref: 'v3.0.4'
+        commit: 'sha1-8a0cd651e5b2365bd640854a3fa5dfc70e4875f3'
         owner_id: 2242
         repo_id: 204253808

.github/workflows/e2e-nightly.yml

@@ -578,6 +578,8 @@ jobs:
              echo \"::group::suite ${SUITE}\"; \
              date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-${SUITE}.start'; \
              GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+               SPINIFEX_NODES='${{ steps.vm.outputs.wan_ip }}' SPINIFEX_SSH_USER=tf-user \
+               SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \
                ARTIFACT_DIR='${ARTIFACT_DIR}' \
                ./${SUITE}.test -test.v -test.timeout=${TMO} \
                2>&1 | tee '${ARTIFACT_DIR}/test-${SUITE}.log'; \

.github/workflows/e2e.yml

@@ -478,7 +478,9 @@ jobs:
                printf '\n%s%s━━ suite %s ━━%s\n' \"\$B\" \"\$C\" \"\$SUITE\" \"\$R\"; \
                date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \
                TMO=30m; for kv in ${SUITE_TMOS}; do case \$kv in \$SUITE:*) TMO=\${kv#*:} ;; esac; done; \
-               GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
+               NODE_SSH=; if [ \"\$SUITE\" = eks ]; then \
+                 NODE_SSH=\"SPINIFEX_NODES=${{ steps.vm.outputs.wan_ip }} SPINIFEX_SSH_USER=tf-user SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2\"; fi; \
+               env \$NODE_SSH GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \
                  ARTIFACT_DIR='${ARTIFACT_DIR}' \
                  ./\$SUITE.test -test.v -test.timeout=\$TMO ${TEST_RUN_FLAG} \
                  2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \

.github/workflows/release.yml

@@ -150,7 +150,7 @@ jobs:
           path: artifacts
 
       - name: Create Draft GitHub Release
-        uses: softprops/action-gh-release@v3.0.3
+        uses: softprops/action-gh-release@v3.0.4
         with:
           draft: true
           files: |

.github/workflows/ui.yml

@@ -51,7 +51,7 @@ jobs:
           sparse-checkout: spinifex/runtime/roles/spinifexui/frontend
 
       - name: Setup pnpm
-        uses: pnpm/action-setup@v6.1.0
+        uses: pnpm/action-setup@v6.1.1
         with:
           package_json_file: spinifex/runtime/roles/spinifexui/frontend/package.json
 
@@ -127,7 +127,7 @@ jobs:
           sparse-checkout: spinifex/runtime/roles/spinifexui/frontend
 
       - name: Setup pnpm
-        uses: pnpm/action-setup@v6.1.0
+        uses: pnpm/action-setup@v6.1.1
         with:
           package_json_file: spinifex/runtime/roles/spinifexui/frontend/package.json
 

architecture/layering/allowlist_test.go

@@ -17,7 +17,6 @@ var allowlist = []allowance{
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/instance", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/routetable", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/snapshot", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
-	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/volume", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 	{From: "operator/accountlifecycle", To: "domains/ec2/awsapi/vpc", Debt: "accountteardown (dependency)", Reason: "EC2 reapers call the awsapi action adapters; needs domain-owned cleanup capabilities"},
 
 	{From: "agents/ecs/agent", To: "domains/ecs", Debt: "agents/ecs/agent", Reason: "assignment/state wire types from the ECS implementation; needs a guest/controller contract"},

build/microvm/init.sh

@@ -32,6 +32,8 @@ load_mod qemu_fw_cfg || load_mod fw_cfg_sysfs || true
 load_mod failover || true
 load_mod net_failover || true
 load_mod virtio_net || true
+# lb-agent probes each data NIC's gateway on an AF_PACKET socket.
+load_mod af_packet || true
 
 NETCFG=/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
 i=0

build/microvm/lambda-runtime/init.sh

@@ -16,8 +16,8 @@ load_mod() {
     done
     return 1
 }
-# lambda-agent reads its identity from fw_cfg and unloads qemu_fw_cfg before
-# anything else runs. virtio_net needs net_failover and failover, and the
+# lambda-agent reads its identity from fw_cfg, or the configuration disk, and
+# seals it before anything else runs. virtio_net needs net_failover and failover, and the
 # insmod fallback resolves no dependencies, so load the chain bottom-up.
 load_mod qemu_fw_cfg || echo "[init] qemu_fw_cfg did not load" >/dev/console
 load_mod failover || true
@@ -26,8 +26,30 @@ load_mod virtio_net || echo "[init] virtio_net did not load" >/dev/console
 
 ip link set lo up
 
+# Without fw_cfg data the executor attaches the same entries as a tar archive
+# on the second disk; lambda-agent reads them here and erases the disk before
+# the function starts. qemu_fw_cfg loads without a device, so test the entry.
+CFG_ROOT=/spinifex-cfg
+FWCFG_ROOT=/sys/firmware/qemu_fw_cfg
+if [ ! -f "$FWCFG_ROOT/by_name/opt/spinifex/netcfg/raw" ]; then
+    load_mod virtio_blk || echo "[init] virtio_blk did not load" >/dev/console
+    i=0
+    while [ "$i" -lt 25 ] && [ ! -b /dev/vdb ]; do
+        sleep 0.04
+        i=$((i + 1))
+    done
+    if [ -b /dev/vdb ]; then
+        mkdir -m 0700 "$CFG_ROOT"
+        if ! mount -t tmpfs -o mode=0700,size=1m tmpfs "$CFG_ROOT" ||
+            ! tar -xf /dev/vdb -C "$CFG_ROOT"; then
+            echo "[init] configuration disk not readable" >/dev/console
+        fi
+        FWCFG_ROOT=$CFG_ROOT
+    fi
+fi
+
 # The guest's addresses come from the same netcfg entry as other system VMs.
-NETCFG=/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
+NETCFG=$FWCFG_ROOT/by_name/opt/spinifex/netcfg/raw
 i=0
 while [ "$i" -lt 25 ] && [ ! -f "$NETCFG" ]; do
     sleep 0.04

cmd/lambda-agent/connect.go

@@ -5,6 +5,7 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
+	"os"
 
 	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
@@ -25,9 +26,22 @@ type startup struct {
 }
 
 func defaultStartup() startup {
+	if _, err := os.Stat(hostlink.ConfigDiskRoot); err == nil {
+		return configDiskStartup(hostlink.ConfigDiskRoot, hostlink.ConfigDiskDevice)
+	}
 	return startup{fwCfgDir: hostlink.FwCfgDir, fwCfgRoot: hostlink.FwCfgRoot, unload: hostlink.UnloadFwCfg}
 }
 
+// configDiskStartup reads the entries init extracted from the configuration
+// disk, and seals by erasing the disk and removing the extracted tree.
+func configDiskStartup(root, device string) startup {
+	return startup{
+		fwCfgDir:  root + "/by_name/" + lambdav1.FwCfgPrefix,
+		fwCfgRoot: root,
+		unload:    func() error { return hostlink.SealConfigDisk(device, root) },
+	}
+}
+
 // connect loads the identity, seals fw_cfg whether or not the load succeeded,
 // then dials the host. Under TLS 1.3 a host refusal is seen only on first read.
 func (s startup) connect(ctx context.Context) (*tls.Conn, lambdav1.Scope, int, error) {

cmd/lambda-agent/connect_configdisk_test.go

@@ -0,0 +1,43 @@
+//test:in-package — package main cannot be imported by an external test package.
+
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// An identity that does not load still seals the configuration disk: the
+// guest key on it is erased before the agent refuses to start.
+func TestConfigDiskStartup_RefusedLoadErasesDisk(t *testing.T) {
+	dir := t.TempDir()
+	root := filepath.Join(dir, "cfg")
+	require.NoError(t, os.MkdirAll(filepath.Join(root, "by_name"), 0o700))
+	device := filepath.Join(dir, "vdb")
+	require.NoError(t, os.WriteFile(device, bytes.Repeat([]byte("guest-key"), 512), 0o600))
+
+	_, _, code, err := configDiskStartup(root, device).connect(t.Context())
+	require.Error(t, err)
+	require.Equal(t, exitRefused, code)
+	got, rerr := os.ReadFile(device)
+	require.NoError(t, rerr)
+	require.Len(t, got, 9*512)
+	require.Equal(t, make([]byte, len(got)), got, "configuration disk erased")
+}
+
+// A disk that cannot be erased fails closed: the agent refuses, and the
+// extracted tree is kept so the seal check sees it.
+func TestConfigDiskStartup_UnerasableDiskRefuses(t *testing.T) {
+	dir := t.TempDir()
+	root := filepath.Join(dir, "cfg")
+	require.NoError(t, os.MkdirAll(filepath.Join(root, "by_name"), 0o700))
+
+	_, _, code, err := configDiskStartup(root, filepath.Join(dir, "missing")).connect(t.Context())
+	require.ErrorContains(t, err, "erase configuration disk")
+	require.Equal(t, exitRefused, code)
+	require.DirExists(t, root)
+}

cmd/lambda-agent/sdk_config_testlink.go

@@ -0,0 +1,65 @@
+//go:build lambdatestlink
+
+package main
+
+import (
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"net"
+	"net/url"
+	"os"
+	"path/filepath"
+)
+
+const (
+	sdkEndpointEntry = "test-sdk-endpoint"
+	sdkCAEntry       = "test-sdk-ca"
+	sdkBundlePath    = "/run/lambda-sdk/ca-bundle.pem"
+	systemBundlePath = "/etc/ssl/certs/ca-certificates.crt"
+)
+
+type sdkConfig struct {
+	endpoint string
+	ca       []byte
+}
+
+// readSDKConfig reads optional public settings before the boot config is sealed.
+func readSDKConfig(dir string) (*sdkConfig, error) {
+	endpoint, endpointErr := os.ReadFile(filepath.Join(dir, sdkEndpointEntry, "raw"))
+	ca, caErr := os.ReadFile(filepath.Join(dir, sdkCAEntry, "raw"))
+	if errors.Is(endpointErr, os.ErrNotExist) && errors.Is(caErr, os.ErrNotExist) {
+		return nil, nil
+	}
+	if endpointErr != nil || caErr != nil {
+		return nil, errors.New("incomplete SDK gateway settings")
+	}
+	u, err := url.Parse(string(endpoint))
+	if err != nil || u.Scheme != "https" || u.User != nil || u.Path != "" || u.RawQuery != "" || u.Fragment != "" ||
+		net.ParseIP(u.Hostname()) == nil || u.Port() == "" {
+		return nil, errors.New("invalid SDK gateway endpoint")
+	}
+	if len(ca) > 64<<10 || !x509.NewCertPool().AppendCertsFromPEM(ca) {
+		return nil, errors.New("invalid SDK gateway CA")
+	}
+	return &sdkConfig{endpoint: string(endpoint), ca: ca}, nil
+}
+
+// writeBundle combines the system roots with the cluster's public root.
+func (c *sdkConfig) writeBundle(path, systemPath string) error {
+	system, err := os.ReadFile(systemPath)
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("read system CA bundle: %w", err)
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		return fmt.Errorf("create SDK CA directory: %w", err)
+	}
+	if err := os.Chmod(filepath.Dir(path), 0o755); err != nil {
+		return fmt.Errorf("make SDK CA directory readable: %w", err)
+	}
+	bundle := append(append(system, '\n'), c.ca...)
+	if err := os.WriteFile(path, bundle, 0o644); err != nil {
+		return fmt.Errorf("write SDK CA bundle: %w", err)
+	}
+	return os.Chmod(path, 0o644)
+}

cmd/lambda-agent/sdk_config_testlink_test.go

@@ -0,0 +1,106 @@
+//go:build lambdatestlink
+
+//test:in-package exercises the unexported boot configuration reader
+package main
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSDKConfig_OptionalAndInvalid(t *testing.T) {
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
+	defer server.Close()
+	ca := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw})
+	for _, tc := range []struct {
+		name     string
+		endpoint []byte
+		ca       []byte
+		wantErr  bool
+		want     bool
+	}{
+		{name: "absent"},
+		{name: "valid", endpoint: []byte(server.URL), ca: ca, want: true},
+		{name: "missing ca", endpoint: []byte(server.URL), wantErr: true},
+		{name: "missing endpoint", ca: ca, wantErr: true},
+		{name: "invalid endpoint", endpoint: []byte("http://127.0.0.1:1234"), ca: ca, wantErr: true},
+		{name: "invalid ca", endpoint: []byte(server.URL), ca: []byte("not a certificate"), wantErr: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			dir := t.TempDir()
+			for name, value := range map[string][]byte{sdkEndpointEntry: tc.endpoint, sdkCAEntry: tc.ca} {
+				if value != nil {
+					require.NoError(t, os.MkdirAll(filepath.Join(dir, name), 0o700))
+					require.NoError(t, os.WriteFile(filepath.Join(dir, name, "raw"), value, 0o600))
+				}
+			}
+			cfg, err := readSDKConfig(dir)
+			if tc.wantErr {
+				require.Error(t, err)
+				require.Nil(t, cfg)
+				return
+			}
+			require.NoError(t, err)
+			if !tc.want {
+				require.Nil(t, cfg)
+				return
+			}
+			bundle := filepath.Join(t.TempDir(), "sdk", "ca.pem")
+			require.NoError(t, cfg.writeBundle(bundle, filepath.Join(t.TempDir(), "absent")))
+			data, err := os.ReadFile(bundle)
+			require.NoError(t, err)
+			roots := x509.NewCertPool()
+			require.True(t, roots.AppendCertsFromPEM(data))
+			client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: roots}}}
+			resp, err := client.Get(cfg.endpoint)
+			require.NoError(t, err)
+			defer resp.Body.Close()
+			assert.Equal(t, http.StatusOK, resp.StatusCode)
+		})
+	}
+}
+
+func TestSDKConfig_BundleKeepsSystemRoots(t *testing.T) {
+	cluster := httptest.NewTLSServer(nil)
+	defer cluster.Close()
+	clusterCA := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cluster.Certificate().Raw})
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+	template := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test system root"},
+		NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour),
+		IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign}
+	der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
+	require.NoError(t, err)
+	systemCA := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+	dir := t.TempDir()
+	systemPath, bundlePath := filepath.Join(dir, "system.pem"), filepath.Join(dir, "sdk", "ca.pem")
+	require.NoError(t, os.WriteFile(systemPath, systemCA, 0o600))
+	require.NoError(t, (&sdkConfig{ca: clusterCA}).writeBundle(bundlePath, systemPath))
+	bundle, err := os.ReadFile(bundlePath)
+	require.NoError(t, err)
+	roots := x509.NewCertPool()
+	require.True(t, roots.AppendCertsFromPEM(bundle))
+	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: roots}}}
+	resp, err := client.Get(cluster.URL)
+	require.NoError(t, err)
+	require.NoError(t, resp.Body.Close())
+	systemCert, err := x509.ParseCertificate(der)
+	require.NoError(t, err)
+	_, err = systemCert.Verify(x509.VerifyOptions{Roots: roots})
+	require.NoError(t, err)
+}

cmd/lambda-agent/start_testlink.go

@@ -12,11 +12,14 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
+	"sort"
+	"strings"
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/agent"
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/hostlink"
 	"github.com/mulgadc/spinifex/spinifex/agents/lambda/runtimeapi"
+	"github.com/mulgadc/spinifex/spinifex/foundation/aws/lambdaenv"
 )
 
 const (
@@ -31,13 +34,17 @@ type testlink struct {
 	pkg            taskPackage
 	runtimeAddr    string
 	startBootstrap func(ctx context.Context, env []string) (*exec.Cmd, error)
+	caBundlePath   string
+	systemCAPath   string
 }
 
 func start(ctx context.Context) int {
 	t := testlink{
-		startup:     defaultStartup(),
-		pkg:         devTaskPackage(),
-		runtimeAddr: runtimeAddr,
+		startup:      defaultStartup(),
+		pkg:          devTaskPackage(),
+		runtimeAddr:  runtimeAddr,
+		caBundlePath: sdkBundlePath,
+		systemCAPath: systemBundlePath,
 		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
 			return agent.StartBootstrap(ctx, taskRoot+"/bootstrap", env, os.Stdout, os.Stderr)
 		},
@@ -53,6 +60,7 @@ func (t testlink) run(parent context.Context) (int, error) {
 	// The package values are fw_cfg entries, so read them before connect seals it.
 	spec, specErr := readPackageSpec(t.fwCfgDir)
 	grant, grantErr := readGrant(t.fwCfgDir)
+	sdk, sdkErr := readSDKConfig(t.fwCfgDir)
 	conn, scope, code, err := t.connect(parent)
 	if err != nil {
 		return code, errors.Join(err, specErr, grantErr)
@@ -85,6 +93,18 @@ func (t testlink) run(parent context.Context) (int, error) {
 	}
 	cred := *first.Credential
 	first.Credential = nil
+	if err := lambdaenv.Validate(cred.Environment); err != nil {
+		return exitRefused, errors.New("first frame carries an invalid function environment")
+	}
+	if sdkErr != nil {
+		slog.Warn("lambda-agent: SDK gateway settings unavailable", "err", sdkErr)
+	}
+	if sdk != nil {
+		if err := sdk.writeBundle(t.caBundlePath, t.systemCAPath); err != nil {
+			slog.Warn("lambda-agent: SDK gateway settings unavailable", "err", err)
+			sdk = nil
+		}
+	}
 	if err := t.pkg.install(spec); err != nil {
 		return exitRefused, fmt.Errorf("install package: %w", err)
 	}
@@ -100,7 +120,8 @@ func (t testlink) run(parent context.Context) (int, error) {
 
 	ctx, cancel := context.WithCancelCause(parent)
 	defer cancel(nil)
-	cmd, err := t.startBootstrap(ctx, bootstrapEnv(ln.Addr().String(), spec.handler, cred))
+	region, _, _ := strings.Cut(string(scope), "/")
+	cmd, err := t.startBootstrap(ctx, bootstrapEnv(ln.Addr().String(), spec.handler, cred, region, sdk, t.caBundlePath))
 	if err != nil {
 		return exitLink, fmt.Errorf("start bootstrap: %w", err)
 	}
@@ -124,18 +145,35 @@ func (t testlink) run(parent context.Context) (int, error) {
 	return exitClosed, nil
 }
 
-// bootstrapEnv is the bootstrap's whole environment: the runtime variables and
-// the execution role's three credential variables.
-func bootstrapEnv(runtimeAPI, handler string, cred hostlink.Credential) []string {
-	return []string{
-		"AWS_LAMBDA_RUNTIME_API=" + runtimeAPI,
-		"_HANDLER=" + handler,
-		"LAMBDA_TASK_ROOT=" + taskRoot,
-		"PATH=/bin:/sbin:/usr/bin",
-		"AWS_ACCESS_KEY_ID=" + cred.AccessKeyID,
-		"AWS_SECRET_ACCESS_KEY=" + cred.SecretAccessKey,
-		"AWS_SESSION_TOKEN=" + cred.SessionToken,
+// bootstrapEnv supplies the runtime, execution-role and SDK settings.
+func bootstrapEnv(runtimeAPI, handler string, cred hostlink.Credential, region string, sdk *sdkConfig, caPath string) []string {
+	vars := map[string]string{
+		"AWS_LAMBDA_RUNTIME_API": runtimeAPI,
+		"_HANDLER":               handler,
+		"LAMBDA_TASK_ROOT":       taskRoot,
+		"PATH":                   "/bin:/sbin:/usr/bin",
+		"AWS_ACCESS_KEY_ID":      cred.AccessKeyID,
+		"AWS_SECRET_ACCESS_KEY":  cred.SecretAccessKey,
+		"AWS_SESSION_TOKEN":      cred.SessionToken,
+		"AWS_REGION":             region,
+		"AWS_DEFAULT_REGION":     region,
 	}
+	if sdk != nil {
+		vars["AWS_ENDPOINT_URL"], vars["AWS_CA_BUNDLE"] = sdk.endpoint, caPath
+	}
+	for k, v := range cred.Environment {
+		vars[k] = v
+	}
+	keys := make([]string, 0, len(vars))
+	for k := range vars {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	env := make([]string, 0, len(keys))
+	for _, k := range keys {
+		env = append(env, k+"="+vars[k])
+	}
+	return env
 }
 
 // shutdownGrace is how long past the invocation deadline the agent waits for

cmd/lambda-agent/start_testlink_test.go

@@ -7,8 +7,10 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
+	"encoding/pem"
 	"io"
 	"net/http"
+	"net/http/httptest"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -70,6 +72,7 @@ type bootstrapStarts struct {
 	sealed  bool
 	// handlerOut receives the _HANDLER the bootstrap process itself saw.
 	handlerOut string
+	caBundle   string
 	// env is the environment the agent gave the bootstrap.
 	env []string
 }
@@ -94,6 +97,7 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 	b.started = make(chan struct{}, 1)
 	b.task = t.TempDir()
 	b.handlerOut = filepath.Join(t.TempDir(), "handler")
+	b.caBundle = filepath.Join(t.TempDir(), "sdk", "ca-bundle.pem")
 	return testlink{
 		startup: g.startup(),
 		pkg: taskPackage{
@@ -102,7 +106,9 @@ func (b *bootstrapStarts) testlink(t *testing.T, entries map[string][]byte) test
 			mountDir:   func(string) error { return nil },
 			sealDir:    func(string) error { b.sealed = true; return nil },
 		},
-		runtimeAddr: "127.0.0.1:0",
+		runtimeAddr:  "127.0.0.1:0",
+		caBundlePath: b.caBundle,
+		systemCAPath: filepath.Join(t.TempDir(), "system-ca.pem"),
 		startBootstrap: func(ctx context.Context, env []string) (*exec.Cmd, error) {
 			b.n++
 			b.env = env
@@ -176,10 +182,87 @@ func TestTestlinkServesOneInvocationThenStops(t *testing.T) {
 		"AWS_ACCESS_KEY_ID=" + testCredential.AccessKeyID,
 		"AWS_SECRET_ACCESS_KEY=" + testCredential.SecretAccessKey,
 		"AWS_SESSION_TOKEN=" + testCredential.SessionToken,
+		"AWS_REGION=ap-southeast-2",
+		"AWS_DEFAULT_REGION=ap-southeast-2",
 	}, b.env, "the bootstrap gets exactly the runtime and credential variables")
 	assert.NotEmpty(t, envValue(b.env, "AWS_LAMBDA_RUNTIME_API"))
 }
 
+func TestTestlinkProvidesSDKSettingsToBootstrap(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }))
+	defer server.Close()
+	h := newHost(t)
+	h.accept(ctx)
+	entries := h.entries()
+	entries[sdkEndpointEntry] = []byte(server.URL)
+	entries[sdkCAEntry] = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw})
+	var b bootstrapStarts
+	done := runTestlinkAsync(ctx, b.testlink(t, entries))
+	conn := h.guest(t)
+	defer conn.Close()
+	hs := hostlink.NewTestStream(conn)
+	req := hostlink.Request{Credential: &testCredential, GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: []byte(`{}`)}
+	require.NoError(t, hs.HostSend(req))
+	_, err := hs.HostResult(ctx)
+	require.NoError(t, err)
+	res := wait(t, done, "agent to stop")
+	require.NoError(t, res.err)
+	assert.Equal(t, server.URL, envValue(b.env, "AWS_ENDPOINT_URL"))
+	assert.Equal(t, b.caBundle, envValue(b.env, "AWS_CA_BUNDLE"))
+	assert.Equal(t, "ap-southeast-2", envValue(b.env, "AWS_REGION"))
+	assert.Equal(t, "ap-southeast-2", envValue(b.env, "AWS_DEFAULT_REGION"))
+	ca, err := os.ReadFile(b.caBundle)
+	require.NoError(t, err)
+	assert.Contains(t, string(ca), string(entries[sdkCAEntry]))
+	info, err := os.Stat(b.caBundle)
+	require.NoError(t, err)
+	assert.Equal(t, os.FileMode(0o644), info.Mode().Perm())
+}
+
+func TestTestlinkProvidesFunctionEnvironment(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	var b bootstrapStarts
+	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
+	conn := h.guest(t)
+	defer conn.Close()
+	hs := hostlink.NewTestStream(conn)
+	cred := testCredential
+	cred.Environment = map[string]string{"GREETING": "hello $WORLD", "PATH": "/my/bin"}
+	req := hostlink.Request{Credential: &cred, GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: []byte(`{}`)}
+	require.NoError(t, hs.HostSend(req))
+	_, err := hs.HostResult(ctx)
+	require.NoError(t, err)
+	require.NoError(t, wait(t, done, "agent to stop").err)
+	assert.Equal(t, "hello $WORLD", envValue(b.env, "GREETING"))
+	assert.Equal(t, "/my/bin", envValue(b.env, "PATH"))
+	assert.Equal(t, testCredential.SecretAccessKey, envValue(b.env, "AWS_SECRET_ACCESS_KEY"))
+}
+
+func TestTestlinkRejectsReservedFunctionEnvironment(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	h := newHost(t)
+	h.accept(ctx)
+	var b bootstrapStarts
+	done := runTestlinkAsync(ctx, b.testlink(t, h.entries()))
+	conn := h.guest(t)
+	defer conn.Close()
+	cred := testCredential
+	cred.Environment = map[string]string{"AWS_SECRET_ACCESS_KEY": "forged"}
+	req := hostlink.Request{Credential: &cred, GrantID: "g-1", AttemptID: "a-1", RequestID: "r-1", DeadlineMs: time.Now().Add(5 * time.Second).UnixMilli(), Payload: []byte(`{}`)}
+	require.NoError(t, hostlink.NewTestStream(conn).HostSend(req))
+	result := wait(t, done, "agent to refuse the environment")
+	assert.Equal(t, exitRefused, result.code)
+	assert.Error(t, result.err)
+	assert.NotContains(t, result.err.Error(), "forged")
+	assert.Zero(t, b.n)
+}
+
 var testCredential = hostlink.Credential{AccessKeyID: "ASIAGUESTKEY", SecretAccessKey: "guest-secret", SessionToken: "guest-token"}
 
 func envValue(env []string, name string) string {

contracts/cluster/v1/heartbeat.go

@@ -1,6 +1,9 @@
 package clusterv1
 
-import "time"
+import (
+	"encoding/json"
+	"time"
+)
 
 // ClusterStateBucket is the KV bucket holding each daemon's heartbeat alongside
 // other cluster-wide state.
@@ -11,6 +14,27 @@ func HeartbeatKey(node string) string {
 	return "heartbeat." + node
 }
 
+// LivenessWindow is how recently a node must have heartbeated for a reader
+// deciding on a takeover to count it live. It is six heartbeats, compared
+// across node clocks, where NTP skew is small against it.
+const LivenessWindow = 60 * time.Second
+
+// LiveAt reports whether a heartbeat written at written still vouches for its
+// node at now: it does while it is under LivenessWindow old.
+func LiveAt(written, now time.Time) bool {
+	return now.Sub(written) < LivenessWindow
+}
+
+// HeartbeatWritten dates a stored heartbeat value. One that cannot be decoded
+// or dated vouches for nothing, which is an answer rather than a fault.
+func HeartbeatWritten(data []byte) (time.Time, bool) {
+	var beat HeartbeatTimestamp
+	if json.Unmarshal(data, &beat) != nil {
+		return time.Time{}, false
+	}
+	return beat.Time()
+}
+
 // HeartbeatTimestamp is the read side of a heartbeat for node liveness. It
 // decodes only the timestamp, so a reader does not depend on the capacity
 // figures the daemon writes beside it.

contracts/cluster/v1/heartbeat_test.go

@@ -33,3 +33,23 @@ func TestHeartbeatTimestamp_UndatedBeatVouchesForNothing(t *testing.T) {
 		assert.False(t, ok, "timestamp %q", ts)
 	}
 }
+
+// A beat is live strictly under the window, so one exactly the window old is
+// the first a takeover may treat as stopped.
+func TestLiveAt_IsStrictlyUnderTheWindow(t *testing.T) {
+	now := time.Date(2026, 11, 5, 12, 0, 0, 0, time.UTC)
+	assert.True(t, clusterv1.LiveAt(now.Add(-clusterv1.LivenessWindow+time.Nanosecond), now))
+	assert.False(t, clusterv1.LiveAt(now.Add(-clusterv1.LivenessWindow), now))
+	assert.True(t, clusterv1.LiveAt(now.Add(time.Second), now), "a beat from a clock running ahead is live")
+}
+
+func TestHeartbeatWritten_DatesAStoredBeatAndRefusesAnUndecodableOne(t *testing.T) {
+	written, ok := clusterv1.HeartbeatWritten([]byte(`{"node":"node-a","timestamp":"2026-10-30T01:02:03Z"}`))
+	require.True(t, ok)
+	assert.True(t, written.Equal(time.Date(2026, 10, 30, 1, 2, 3, 0, time.UTC)))
+
+	for _, raw := range []string{"not json", `{"node":"node-a"}`} {
+		_, ok := clusterv1.HeartbeatWritten([]byte(raw))
+		assert.False(t, ok, raw)
+	}
+}

contracts/ec2/v1/commands.go

@@ -12,6 +12,11 @@ const (
 	// this contract family. It is the NATS permission/subscription pattern for
 	// the per-instance command route, rather than the broader descendant match.
 	InstanceCommandSubjectWildcard = InstanceCommandSubjectPrefix + "*"
+
+	// HandledHeader is set to "1" on every reply an owner sends to a
+	// modify-instance-metadata-options or set-disable-api-termination command.
+	// An owner that predates those commands cannot set it.
+	HandledHeader = "Spinifex-Ec2-Cmd-Handled"
 )
 
 // InstanceCommandSubject returns the targeted command subject for instanceID.
@@ -23,7 +28,9 @@ func InstanceCommandSubject(instanceID string) string {
 }
 
 // EC2InstanceCommand is the NATS wire format for EC2 instance commands
-// (stop, terminate, start, attach/detach-volume, attach/detach-eni).
+// (stop, terminate, start, reboot, attach/detach-volume, attach/detach-eni,
+// IAM profile, spot lineage, tags, monitoring, metadata options and
+// termination protection).
 type EC2InstanceCommand struct {
 	ID                        string                     `json:"id"`
 	Attributes                EC2CommandAttributes       `json:"attributes"`
@@ -35,6 +42,9 @@ type EC2InstanceCommand struct {
 	SpotLineageData           *SpotLineageData           `json:"spot_lineage_data,omitempty"`
 	InstanceTagsData          *InstanceTagsData          `json:"instance_tags_data,omitempty"`
 	InstanceMonitoringData    *InstanceMonitoringData    `json:"instance_monitoring_data,omitempty"`
+
+	InstanceMetadataOptionsData *InstanceMetadataOptionsData `json:"instance_metadata_options_data,omitempty"`
+	DisableApiTerminationData   *DisableApiTerminationData   `json:"disable_api_termination_data,omitempty"`
 }
 
 // EC2CommandAttributes indicates which action the daemon should perform.
@@ -52,6 +62,9 @@ type EC2CommandAttributes struct {
 	SetInstanceTags             bool `json:"set_instance_tags,omitempty"`
 	RemoveInstanceTags          bool `json:"remove_instance_tags,omitempty"`
 	SetInstanceMonitoring       bool `json:"set_instance_monitoring,omitempty"`
+
+	ModifyInstanceMetadataOptions bool `json:"modify_instance_metadata_options,omitempty"`
+	SetDisableApiTermination      bool `json:"set_disable_api_termination,omitempty"`
 }
 
 // AttachVolumeData carries parameters for an attach-volume command.
@@ -103,6 +116,23 @@ type InstanceMonitoringData struct {
 	Enabled bool `json:"enabled"`
 }
 
+// InstanceMetadataOptionsData carries the requested metadata options for a
+// modify-instance-metadata-options command. Unset fields are left unchanged.
+type InstanceMetadataOptionsData struct {
+	HttpTokens              string `json:"http_tokens,omitempty"`
+	HttpEndpoint            string `json:"http_endpoint,omitempty"`
+	HttpProtocolIpv6        string `json:"http_protocol_ipv6,omitempty"`
+	InstanceMetadataTags    string `json:"instance_metadata_tags,omitempty"`
+	HttpPutResponseHopLimit *int64 `json:"http_put_response_hop_limit,omitempty"`
+}
+
+// DisableApiTerminationData carries the desired termination-protection value
+// for a set-disable-api-termination command. Carried as data so "false" is an
+// explicit value, not the absence of one.
+type DisableApiTerminationData struct {
+	Value bool `json:"value"`
+}
+
 // SpotLineageData carries the SIR id stamped onto a spot-launched instance.
 // Lifecycle is always "spot" for this command, so only the SIR id travels.
 type SpotLineageData struct {

contracts/ec2/v1/commands_test.go

@@ -24,6 +24,9 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 			SetInstanceTags:             true,
 			RemoveInstanceTags:          true,
 			SetInstanceMonitoring:       true,
+
+			ModifyInstanceMetadataOptions: true,
+			SetDisableApiTermination:      true,
 		},
 		AttachVolumeData:          &AttachVolumeData{VolumeID: "vol-1", Device: "/dev/sdf"},
 		DetachVolumeData:          &DetachVolumeData{VolumeID: "vol-2", Device: "/dev/sdg", Force: true},
@@ -33,6 +36,15 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 		SpotLineageData:           &SpotLineageData{SpotInstanceRequestId: "sir-1"},
 		InstanceTagsData:          &InstanceTagsData{Tags: map[string]string{"Name": "example"}, TagKeys: []string{"old"}},
 		InstanceMonitoringData:    &InstanceMonitoringData{Enabled: true},
+
+		InstanceMetadataOptionsData: &InstanceMetadataOptionsData{
+			HttpTokens:              "required",
+			HttpEndpoint:            "enabled",
+			HttpProtocolIpv6:        "disabled",
+			InstanceMetadataTags:    "enabled",
+			HttpPutResponseHopLimit: int64Ptr(3),
+		},
+		DisableApiTerminationData: &DisableApiTerminationData{Value: false},
 	}
 
 	data, err := json.Marshal(command)
@@ -52,7 +64,9 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 			"set_spot_lineage": true,
 			"set_instance_tags": true,
 			"remove_instance_tags": true,
-			"set_instance_monitoring": true
+			"set_instance_monitoring": true,
+			"modify_instance_metadata_options": true,
+			"set_disable_api_termination": true
 		},
 		"attach_volume_data": {"volume_id": "vol-1", "device": "/dev/sdf"},
 		"detach_volume_data": {"volume_id": "vol-2", "device": "/dev/sdg", "force": true},
@@ -61,10 +75,64 @@ func TestEC2InstanceCommandJSONContract(t *testing.T) {
 		"iam_profile_association_data": {"instance_profile_arn": "arn:aws:iam::123456789012:instance-profile/example"},
 		"spot_lineage_data": {"spot_instance_request_id": "sir-1"},
 		"instance_tags_data": {"tags": {"Name": "example"}, "tag_keys": ["old"]},
-		"instance_monitoring_data": {"enabled": true}
+		"instance_monitoring_data": {"enabled": true},
+		"instance_metadata_options_data": {
+			"http_tokens": "required",
+			"http_endpoint": "enabled",
+			"http_protocol_ipv6": "disabled",
+			"instance_metadata_tags": "enabled",
+			"http_put_response_hop_limit": 3
+		},
+		"disable_api_termination_data": {"value": false}
 	}`, string(data))
 }
 
+func int64Ptr(v int64) *int64 { return &v }
+
+// An owner that predates the metadata and termination-protection commands
+// sends and receives payloads without the new fields.
+func TestEC2InstanceCommandNewFieldsAreOptionalOnTheWire(t *testing.T) {
+	var n1 EC2InstanceCommand
+	require.NoError(t, json.Unmarshal([]byte(`{
+		"id": "i-0123456789abcdef0",
+		"attributes": {"stop_instance": true},
+		"instance_monitoring_data": {"enabled": true}
+	}`), &n1))
+	require.Equal(t, "i-0123456789abcdef0", n1.ID)
+	require.True(t, n1.Attributes.StopInstance)
+	require.False(t, n1.Attributes.ModifyInstanceMetadataOptions)
+	require.False(t, n1.Attributes.SetDisableApiTermination)
+	require.Nil(t, n1.InstanceMetadataOptionsData)
+	require.Nil(t, n1.DisableApiTerminationData)
+
+	unset, err := json.Marshal(EC2InstanceCommand{ID: "i-1", Attributes: EC2CommandAttributes{StopInstance: true}})
+	require.NoError(t, err)
+	require.NotContains(t, string(unset), "modify_instance_metadata_options")
+	require.NotContains(t, string(unset), "set_disable_api_termination")
+	require.NotContains(t, string(unset), "instance_metadata_options_data")
+	require.NotContains(t, string(unset), "disable_api_termination_data")
+}
+
+func TestInstanceMetadataOptionsDataOmitsUnsetFields(t *testing.T) {
+	data, err := json.Marshal(InstanceMetadataOptionsData{HttpPutResponseHopLimit: int64Ptr(1)})
+	require.NoError(t, err)
+	require.JSONEq(t, `{"http_put_response_hop_limit": 1}`, string(data))
+
+	data, err = json.Marshal(InstanceMetadataOptionsData{})
+	require.NoError(t, err)
+	require.JSONEq(t, `{}`, string(data))
+}
+
+func TestDisableApiTerminationDataKeepsExplicitFalse(t *testing.T) {
+	data, err := json.Marshal(DisableApiTerminationData{Value: false})
+	require.NoError(t, err)
+	require.JSONEq(t, `{"value": false}`, string(data))
+}
+
+func TestHandledHeaderName(t *testing.T) {
+	require.Equal(t, "Spinifex-Ec2-Cmd-Handled", HandledHeader)
+}
+
 func TestInstanceCommandSubjects(t *testing.T) {
 	require.Equal(t, "ec2.cmd.i-0123456789abcdef0", InstanceCommandSubject("i-0123456789abcdef0"))
 	require.Equal(t, "ec2.cmd.*", InstanceCommandSubjectWildcard)

contracts/lambda/v1/README.md

@@ -21,13 +21,16 @@ Tenant code may be able to reach the listener, but it cannot open the agent's co
 **Authentication:** mutual TLS 1.3, bound to the environment ID and realization generation, using material the user runtime cannot read (ADR-0009:S17).
 For each generation the executor creates ephemeral self-signed host and guest certificates whose single URI SAN is `spinifex-lambda://<Region>/<account>/<function ID>/<environment ID>/<generation>/<role>`, with role `host` or `guest`.
 Each end validates the peer certificate against the one pinned peer certificate and separately requires that URI SAN with the opposite role; hostname verification is not relied on.
-ALPN carries the L-GUEST major version, `spinifex-lambda-guest/1`, and an unsupported version fails the handshake. Builds with the `lambdatestlink` tag offer only `spinifex-lambda-testlink/2` on both host and guest, whose first frame carries the execution-role credential, so a test-link end and a release end fail the handshake; release builds are unchanged.
-In the same builds, the gateway reaches the host daemon on the local abstract socket `spinifex-lambda-testlink-control/3`, and each request carries the runtime-image digest the gateway was built with; the daemon refuses a missing or different digest before starting the attempt, and a gateway and daemon on different socket names never connect.
+ALPN carries the L-GUEST major version, `spinifex-lambda-guest/1`, and an unsupported version fails the handshake. Builds with the `lambdatestlink` tag offer only `spinifex-lambda-testlink/3` on both host and guest, whose first frame carries the execution-role credential and function environment, so a test-link end and a release end fail the handshake; release builds are unchanged.
+In the same builds, the gateway reaches the host daemon on the local abstract socket `spinifex-lambda-testlink-control/4`, and each request carries the runtime-image digest the gateway was built with; the daemon refuses a missing or different digest before starting the attempt, and a gateway and daemon on different socket names never connect.
 On that socket a request's raw payload is at most 6 MiB, checked by the gateway before encoding and by the daemon after decoding.
 Its encoded frame is at most the base64 length of 6 MiB plus 64 KiB of metadata headroom and 64 bytes of field slack, and a receiver checks the frame length before allocating; metadata has no separate bound within the frame.
-The credential travels as its own frame of at most 8 KiB, and the reply has the same payload and frame bounds as the request.
+The credential and environment travel as their own frame of at most 40 KiB, and the reply has the same payload and frame bounds as the request.
 The daemon holds at most two connections at once, from accept to close, and leaves the rest unaccepted in the listen backlog.
 Under TLS 1.3 the guest's handshake can complete before the host refuses its certificate, so the guest treats the connection as accepted only after the host's first message.
+In the same builds, the function owner moves a created function with stored code from `Pending` to `Active` once this node's daemon serves the pinned image and the cluster configuration has exactly one node; a code copy over the test-link bound stays `Pending`, and `StateReason` names the unmet condition.
+Release builds never leave `Pending`.
+The Mulga deploy role installs a test-link build with `deploy_lambda_testlink=true`; this profile does not meet Q-199 fail-closed revocation, and its guests take management addresses whose record can expire (mulga-0hsub).
 
 **Bootstrap material:** the executor passes these fw_cfg entries under `opt/spinifex/lambda/`, and the agent reads them as root and unloads `qemu_fw_cfg` before user code starts.
 
@@ -41,6 +44,8 @@ Under TLS 1.3 the guest's handshake can complete before the host refuses its cer
 
 The guest's own interface settings come from `opt/spinifex/netcfg`, the entry other system VMs use.
 The agent unloads `qemu_fw_cfg` even when an entry is missing, and refuses to start if the fw_cfg tree is still readable afterwards.
+On a VMM without fw_cfg (the test-link Firecracker executor), the executor attaches the same entries as a tar archive on the second, writable virtio-blk disk, laid out as `by_name/<entry>/raw`.
+The runtime image's init extracts it to a root-only tmpfs at `/spinifex-cfg`, and the agent reads the entries there, then overwrites the whole disk with zeros and removes the tree; it refuses to start if the tree is still present.
 Until the message format exists, the agent authenticates and holds the connection but accepts no host message and starts no user code.
 
 **Authority:** one environment's attempt only.

contracts/viperblockd/legacy/v1/README.md

@@ -1,95 +0,0 @@
-# Viperblockd legacy contracts, version 1
-
-This package owns the deployed, transitional configuration-update and
-state-reload request/reply protocols between Spinifex's legacy control plane
-and `viperblockd`. It makes those existing compatibility boundaries explicit
-while the generic `providers/ebs` contract is adopted; it is not the EBS
-provider abstraction and new provider integrations must not depend on it.
-
-## Configuration update routes
-
-`ConfigUpdateSubject` is `ebs.config`. Every `viperblockd` daemon subscribes
-to it in the `spinifex-workers` queue group. It is the fallback path for an
-encrypted volume with no live mount: the selected daemon opens the volume
-exclusively and reseals its state.
-
-`VolumeConfigUpdateSubject(volume)` is `ebs.config.<volume>`. The daemon with
-the live mounted volume subscribes directly so the sole holder of the volume's
-state sequence number applies the update.
-
-Both routes carry `EBSConfigUpdateRequest` and respond with
-`EBSConfigUpdateResponse`. The JSON field spelling, route shapes, and
-request/reply semantics are deployed compatibility behaviour. Incompatible
-changes require a new contract version or an explicit migration plan.
-
-## State reload route
-
-`SyncSubject` is `ebs.sync`. After a successful EC2 `ModifyVolume`, the daemon
-uses request/reply to ask a `spinifex-workers` queue-group member to reload the
-volume's state from its backend. The selected Viperblockd worker responds with
-`EBSSyncResponse`; it returns an error when it does not own a live mount of the
-volume. The daemon records that error as a warning after the API response has
-already been returned. This is deployed legacy behaviour, not a guarantee that
-every modification has reached its owning mount.
-
-The route carries `EBSSyncRequest` and `EBSSyncResponse`. Their JSON shape and
-the queue-group route are compatibility behaviour.
-
-## Volume lifecycle routes
-
-`MountSubject(node)` and `UnmountSubject(node)` retain the deployed mount and
-unmount routes. With a node name they address that node's Viperblockd process
-(`ebs.<node>.mount` or `ebs.<node>.unmount`); without one they retain the
-single-node `spinifex-workers` queue-group routes. Both carry `EBSRequest`.
-`EBSMountResponse` supplies the exported NBD URI, while
-`EBSUnMountResponse` reports the seal result. The response broadcasts remain
-`ebs.mount.response` and `ebs.unmount.response` alongside ordinary
-request/reply replies.
-
-`DeleteSubject` retains the `ebs.delete` queue-group route. It carries
-`EBSDeleteRequest` and responds with `EBSDeleteResponse` after Viperblockd has
-removed its local state and export. These routes and JSON shapes are deployed
-legacy compatibility behaviour, not a generic EBS-provider interface.
-
-## Abandon route
-
-`VolumeAbandonSubject(node)` is `ebs.<node>.abandon`. The daemon uses
-request/reply to ask the Viperblock adapter on that node to tear an export
-down without sealing, because the instance using it belongs to another node
-now. The route carries `VolumeAbandonRequest` and responds with
-`VolumeAbandonResponse`. `Abandoned` is false with no error when the volume
-was not exported there, which is the ordinary case: the request is
-idempotent by design, since the caller is reconciling rather than
-transacting.
-
-## Dirty-marker bucket
-
-`DirtyBucket` is the JetStream KV bucket `VIPERBLOCK_VOLUME_DIRTY`, keyed by
-volume name under `VolumeKeyPattern`, storing `DirtyRecord` JSON. It records
-which node holds writes for a volume that the backend may not have.
-
-Owner: `providers/ebs/viperblock` owns the dirty-volume state and its
-transitions. `services/viperblockd` was only its transitional custodian.
-
-Producers: only the Viperblock adapter writes, clears or changes a dirty
-record.
-
-Consumers: Viperblock recovery and mount logic on any node. `spx admin` may
-obtain a read-only operational view.
-
-Storage contract: the bucket name, `VolumeKeyPattern` and the `DirtyRecord`
-JSON shape are this package's legacy v1 contract. There is no TTL; a record
-persists until an explicit adapter-owned recovery or unseal transition
-removes it.
-
-Compatibility: this is a deployed legacy v1 contract, and coordinated
-writers and readers support N/N-1. No renames, type changes or semantic
-reinterpretation of existing fields; a breaking change needs a successor
-contract and a migration.
-
-CLI authority: the CLI is never a writer and must not perform arbitrary
-bucket operations.
-
-This package does not authorize requests, validate a volume ID, store volume
-state, or define the generic storage-provider API. Those concerns remain with
-the consuming daemon, the EC2 domain, and `providers/ebs` respectively.

contracts/viperblockd/legacy/v1/abandon.go

@@ -1,36 +0,0 @@
-package viperblocklegacyv1
-
-// VolumeAbandonSubject is where the local daemon asks for an export to be torn
-// down because the instance using it belongs to another node now.
-//
-// It exists because an unmount is the wrong request and killing the guest is not
-// enough. An unmount seals, and this node's copy is the stale one, so sealing
-// would publish an older state over the winner's. Meanwhile the export holds the
-// volume lease, and a lease this node keeps renewing is one the winner can never
-// acquire — the guest would be stopped here and unable to start anywhere.
-//
-// Node-addressed, like mount and unmount: the export is local, and no other
-// node's viperblockd has anything it could tear down.
-func VolumeAbandonSubject(node string) string {
-	if node == "" {
-		return "ebs.abandon"
-	}
-	return "ebs." + node + ".abandon"
-}
-
-// VolumeAbandonRequest asks for one volume's export to go away without sealing.
-// Reason is logged, so a node that dropped an export can be told apart from one
-// that was fenced.
-type VolumeAbandonRequest struct {
-	Volume string `json:"volume"`
-	Reason string `json:"reason"`
-}
-
-// VolumeAbandonResponse reports what happened. Abandoned is false with no error
-// when the volume was not exported here, which is the ordinary case and not a
-// failure: the request is idempotent by design, since the caller is reconciling
-// rather than transacting.
-type VolumeAbandonResponse struct {
-	Abandoned bool   `json:"abandoned"`
-	Error     string `json:"error,omitempty"`
-}

contracts/viperblockd/legacy/v1/abandon_test.go

@@ -1,63 +0,0 @@
-package viperblocklegacyv1_test
-
-import (
-	"encoding/json"
-	"testing"
-
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// TestVolumeAbandonSubject_BothForms pins the abandon route: node-addressed,
-// with an empty-node fallback for a single-node daemon.
-func TestVolumeAbandonSubject_BothForms(t *testing.T) {
-	assert.Equal(t, "ebs.node-a.abandon", viperblocklegacyv1.VolumeAbandonSubject("node-a"))
-	assert.Equal(t, "ebs.abandon", viperblocklegacyv1.VolumeAbandonSubject(""))
-}
-
-// TestVolumeAbandonRequest_JSON pins the request runtime/daemon/vm_adapters.go sends
-// on the abandon route.
-func TestVolumeAbandonRequest_JSON(t *testing.T) {
-	golden := `{"volume":"vol-1","reason":"instance owned by another node"}`
-	req := viperblocklegacyv1.VolumeAbandonRequest{Volume: "vol-1", Reason: "instance owned by another node"}
-
-	marshaled, err := json.Marshal(req)
-	require.NoError(t, err)
-	assert.JSONEq(t, golden, string(marshaled))
-
-	var decoded viperblocklegacyv1.VolumeAbandonRequest
-	require.NoError(t, json.Unmarshal([]byte(golden), &decoded))
-	assert.Equal(t, req, decoded)
-}
-
-// TestVolumeAbandonResponse_JSON pins both response shapes: the ordinary
-// reconciling case with no error, and the failure case where Error is set.
-// Error is omitempty, so the two must not be mistaken for the same shape.
-func TestVolumeAbandonResponse_JSON(t *testing.T) {
-	t.Run("abandoned, no error", func(t *testing.T) {
-		golden := `{"abandoned":true}`
-		resp := viperblocklegacyv1.VolumeAbandonResponse{Abandoned: true}
-
-		marshaled, err := json.Marshal(resp)
-		require.NoError(t, err)
-		assert.JSONEq(t, golden, string(marshaled))
-
-		var decoded viperblocklegacyv1.VolumeAbandonResponse
-		require.NoError(t, json.Unmarshal([]byte(golden), &decoded))
-		assert.Equal(t, resp, decoded)
-	})
-
-	t.Run("not abandoned, with error", func(t *testing.T) {
-		golden := `{"abandoned":false,"error":"nbdkit did not exit"}`
-		resp := viperblocklegacyv1.VolumeAbandonResponse{Abandoned: false, Error: "nbdkit did not exit"}
-
-		marshaled, err := json.Marshal(resp)
-		require.NoError(t, err)
-		assert.JSONEq(t, golden, string(marshaled))
-
-		var decoded viperblocklegacyv1.VolumeAbandonResponse
-		require.NoError(t, json.Unmarshal([]byte(golden), &decoded))
-		assert.Equal(t, resp, decoded)
-	})
-}

contracts/viperblockd/legacy/v1/config_update.go

@@ -1,37 +0,0 @@
-// Package viperblocklegacyv1 contains the versioned, transitional wire
-// contracts between Spinifex's legacy control-plane routes and viperblockd.
-package viperblocklegacyv1
-
-import "encoding/json"
-
-const (
-	// ConfigUpdateSubject is the legacy queue-group route for an encrypted
-	// volume configuration update when no mounted volume owns the request.
-	ConfigUpdateSubject = "ebs.config"
-)
-
-// VolumeConfigUpdateSubject returns the legacy route for an update handled by
-// the node with a live mount of volume. Volume remains unvalidated here: route
-// authorization and volume-name validation are responsibilities of the
-// consumer, as they were before this contract was extracted.
-func VolumeConfigUpdateSubject(volume string) string {
-	return ConfigUpdateSubject + "." + volume
-}
-
-// EBSConfigUpdateRequest carries a control-plane VolumeConfig update for an
-// encrypted volume. config.json is a sealed VBState; only the master-key holder
-// (viperblockd) can reseal it, so the EC2 edge ships the new config here instead
-// of rewriting the object directly. VolumeConfig is a marshaled
-// viperblock.VolumeConfig (RawMessage keeps this contract dependency-free).
-type EBSConfigUpdateRequest struct {
-	Volume       string          `json:"Volume"`
-	VolumeConfig json.RawMessage `json:"VolumeConfig"`
-}
-
-// EBSConfigUpdateResponse is the request/reply response for either legacy
-// configuration-update route.
-type EBSConfigUpdateResponse struct {
-	Volume  string `json:"Volume"`
-	Success bool   `json:"Success"`
-	Error   string `json:"Error"`
-}

contracts/viperblockd/legacy/v1/config_update_test.go

@@ -1,35 +0,0 @@
-package viperblocklegacyv1
-
-import (
-	"encoding/json"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestConfigUpdateSubjects(t *testing.T) {
-	require.Equal(t, "ebs.config", ConfigUpdateSubject)
-	require.Equal(t, "ebs.config.vol-123", VolumeConfigUpdateSubject("vol-123"))
-	// The deployed route builder did not validate its final token. Validation
-	// remains with the handler and is not silently introduced by this move.
-	require.Equal(t, "ebs.config.", VolumeConfigUpdateSubject(""))
-}
-
-func TestConfigUpdateRequestJSONContract(t *testing.T) {
-	data, err := json.Marshal(EBSConfigUpdateRequest{
-		Volume:       "vol-123",
-		VolumeConfig: json.RawMessage(`{"size_gib":8}`),
-	})
-	require.NoError(t, err)
-	require.JSONEq(t, `{"Volume":"vol-123","VolumeConfig":{"size_gib":8}}`, string(data))
-}
-
-func TestConfigUpdateResponseJSONContract(t *testing.T) {
-	data, err := json.Marshal(EBSConfigUpdateResponse{
-		Volume:  "vol-123",
-		Success: true,
-		Error:   "",
-	})
-	require.NoError(t, err)
-	require.JSONEq(t, `{"Volume":"vol-123","Success":true,"Error":""}`, string(data))
-}

contracts/viperblockd/legacy/v1/sync.go

@@ -1,22 +0,0 @@
-package viperblocklegacyv1
-
-const (
-	// SyncSubject is the legacy queue-group route that asks a viperblockd
-	// worker to reload a mounted volume's state from its backend.
-	SyncSubject = "ebs.sync"
-)
-
-// EBSSyncRequest identifies the volume whose mounted Viperblock state should
-// be reloaded by the selected legacy worker.
-type EBSSyncRequest struct {
-	Volume string `json:"Volume"`
-}
-
-// EBSSyncResponse reports the selected worker's reload result. A worker that
-// does not own the mounted volume returns an error rather than forwarding the
-// request.
-type EBSSyncResponse struct {
-	Volume string `json:"Volume"`
-	Synced bool   `json:"Synced"`
-	Error  string `json:"Error"`
-}

contracts/viperblockd/legacy/v1/sync_test.go

@@ -1,28 +0,0 @@
-package viperblocklegacyv1
-
-import (
-	"encoding/json"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestSyncSubject(t *testing.T) {
-	require.Equal(t, "ebs.sync", SyncSubject)
-}
-
-func TestSyncRequestJSONContract(t *testing.T) {
-	data, err := json.Marshal(EBSSyncRequest{Volume: "vol-123"})
-	require.NoError(t, err)
-	require.JSONEq(t, `{"Volume":"vol-123"}`, string(data))
-}
-
-func TestSyncResponseJSONContract(t *testing.T) {
-	data, err := json.Marshal(EBSSyncResponse{
-		Volume: "vol-123",
-		Synced: true,
-		Error:  "",
-	})
-	require.NoError(t, err)
-	require.JSONEq(t, `{"Volume":"vol-123","Synced":true,"Error":""}`, string(data))
-}

contracts/viperblockd/legacy/v1/volume.go

@@ -1,87 +0,0 @@
-package viperblocklegacyv1
-
-import "fmt"
-
-const (
-	// DeleteSubject is the legacy queue-group route that deletes a volume's
-	// Viperblockd-local state and any active export.
-	DeleteSubject = "ebs.delete"
-
-	// MountResponseSubject is the legacy broadcast emitted after an ebs.mount
-	// request is handled. Request/reply callers receive the same payload.
-	MountResponseSubject = "ebs.mount.response"
-
-	// UnmountResponseSubject is the legacy broadcast emitted after an
-	// ebs.unmount request is handled. Request/reply callers receive the same
-	// payload.
-	UnmountResponseSubject = "ebs.unmount.response"
-)
-
-// MountSubject returns the deployed mount route. An empty node retains the
-// single-node queue-group route; a named node selects the Viperblockd worker
-// colocated with the instance runtime.
-func MountSubject(node string) string {
-	if node == "" {
-		return "ebs.mount"
-	}
-	return fmt.Sprintf("ebs.%s.mount", node)
-}
-
-// UnmountSubject returns the deployed unmount route. Its empty-node behaviour
-// matches MountSubject for single-node compatibility.
-func UnmountSubject(node string) string {
-	if node == "" {
-		return "ebs.unmount"
-	}
-	return fmt.Sprintf("ebs.%s.unmount", node)
-}
-
-// EBSRequest is the deployed request payload shared by the legacy mount and
-// unmount routes. It is intentionally separate from the generic EBS provider
-// contract: this is Viperblockd's transitional NATS protocol.
-type EBSRequest struct {
-	Name                string `json:"Name"`
-	VolType             string `json:"VolType"`
-	Boot                bool   `json:"Boot"`
-	EFI                 bool   `json:"EFI"`
-	DeleteOnTermination bool   `json:"DeleteOnTermination"`
-	NBDURI              string `json:"NBDURI"`
-	DeviceName          string `json:"DeviceName"`
-	HotplugPort         int    `json:"HotplugPort,omitempty"`
-	// BlockDevice and BlockSizeBytes describe a host block device export, the
-	// alternative to NBDURI; the guest sees BlockSizeBytes of it. Exactly one
-	// of NBDURI and BlockDevice may be set.
-	BlockDevice    string `json:"BlockDevice,omitempty"`
-	BlockSizeBytes uint64 `json:"BlockSizeBytes,omitempty"`
-}
-
-// EBSMountResponse is the response to an EBSRequest on MountSubject.
-type EBSMountResponse struct {
-	URI       string `json:"URI"`
-	Mounted   bool   `json:"Mounted"`
-	Error     string `json:"Error"`
-	Retryable bool   `json:"Retryable"`
-}
-
-// EBSUnMountResponse is the response to an EBSRequest on UnmountSubject.
-// NotFound reports an idempotent retry after an already-complete seal; Reaped
-// reports a leaked export that was cleaned up without a registry entry.
-type EBSUnMountResponse struct {
-	Volume   string `json:"Volume"`
-	Mounted  bool   `json:"Mounted"`
-	Error    string `json:"Error"`
-	NotFound bool   `json:"NotFound"`
-	Reaped   bool   `json:"Reaped,omitempty"`
-}
-
-// EBSDeleteRequest identifies the volume to delete on DeleteSubject.
-type EBSDeleteRequest struct {
-	Volume string `json:"Volume"`
-}
-
-// EBSDeleteResponse is the response to EBSDeleteRequest on DeleteSubject.
-type EBSDeleteResponse struct {
-	Volume  string `json:"Volume"`
-	Success bool   `json:"Success"`
-	Error   string `json:"Error"`
-}

contracts/viperblockd/legacy/v1/volume_test.go

@@ -1,78 +0,0 @@
-package viperblocklegacyv1
-
-import (
-	"encoding/json"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestVolumeSubjects(t *testing.T) {
-	require.Equal(t, "ebs.delete", DeleteSubject)
-	require.Equal(t, "ebs.mount", MountSubject(""))
-	require.Equal(t, "ebs.unmount", UnmountSubject(""))
-	require.Equal(t, "ebs.node-a.mount", MountSubject("node-a"))
-	require.Equal(t, "ebs.node-a.unmount", UnmountSubject("node-a"))
-	require.Equal(t, "ebs.mount.response", MountResponseSubject)
-	require.Equal(t, "ebs.unmount.response", UnmountResponseSubject)
-}
-
-func TestEBSRequestJSONContract(t *testing.T) {
-	data, err := json.Marshal(EBSRequest{
-		Name:                "vol-123",
-		VolType:             "gp3",
-		Boot:                true,
-		EFI:                 true,
-		DeleteOnTermination: true,
-		NBDURI:              "nbd:unix:/run/spinifex/vol-123.sock",
-		DeviceName:          "/dev/sdf",
-		HotplugPort:         3,
-	})
-	require.NoError(t, err)
-	require.JSONEq(t, `{
-		"Name":"vol-123", "VolType":"gp3", "Boot":true, "EFI":true,
-		"DeleteOnTermination":true,
-		"NBDURI":"nbd:unix:/run/spinifex/vol-123.sock",
-		"DeviceName":"/dev/sdf", "HotplugPort":3
-	}`, string(data))
-
-	block, err := json.Marshal(EBSRequest{
-		Name:           "vol-456",
-		BlockDevice:    "/dev/sdx",
-		BlockSizeBytes: 10 << 30,
-	})
-	require.NoError(t, err)
-	require.JSONEq(t, `{
-		"Name":"vol-456", "VolType":"", "Boot":false, "EFI":false,
-		"DeleteOnTermination":false, "NBDURI":"", "DeviceName":"",
-		"BlockDevice":"/dev/sdx", "BlockSizeBytes":10737418240
-	}`, string(block))
-}
-
-func TestVolumeResponseJSONContracts(t *testing.T) {
-	mount, err := json.Marshal(EBSMountResponse{
-		URI: "nbd:unix:/run/spinifex/vol-123.sock", Mounted: true, Retryable: true,
-	})
-	require.NoError(t, err)
-	require.JSONEq(t, `{
-		"URI":"nbd:unix:/run/spinifex/vol-123.sock", "Mounted":true,
-		"Error":"", "Retryable":true
-	}`, string(mount))
-
-	unmount, err := json.Marshal(EBSUnMountResponse{
-		Volume: "vol-123", NotFound: true, Reaped: true,
-	})
-	require.NoError(t, err)
-	require.JSONEq(t, `{
-		"Volume":"vol-123", "Mounted":false, "Error":"",
-		"NotFound":true, "Reaped":true
-	}`, string(unmount))
-
-	deleteRequest, err := json.Marshal(EBSDeleteRequest{Volume: "vol-123"})
-	require.NoError(t, err)
-	require.JSONEq(t, `{"Volume":"vol-123"}`, string(deleteRequest))
-
-	deleteResponse, err := json.Marshal(EBSDeleteResponse{Volume: "vol-123", Success: true})
-	require.NoError(t, err)
-	require.JSONEq(t, `{"Volume":"vol-123", "Success":true, "Error":""}`, string(deleteResponse))
-}

docs/COMMANDS.md

@@ -174,8 +174,8 @@ Operational commands for inspecting cluster state. These fan out NATS requests t
 | `spx admin images import` | `--name`, `--file`, `--distro`, `--version`, `--arch`, `--platform`, `--boot-mode` (bios/uefi/uefi-preferred), `--tag`, `--force`, `--skip-verify`, `--checksum`, `--ami-name` (override the registered `DescribeImages` name, which otherwise defaults to `ami-{distro}-{version}-{arch}`; used for locally-built appliances such as `--ami-name spinifex-eks-node`), `--tmp-dir` (default: `<spinifex-dir>/images`) | Catalog imports (`--name`) download the image, fetch the catalog `Checksum` URL, verify the SHA-256/SHA-512 digest, and inherit `BootMode` from the catalog entry. `--boot-mode` overrides the catalog value when set. Mismatch fails closed; the cached file is left on disk and `--force` re-downloads. `--file` imports require an explicit `--boot-mode` because there is no catalog metadata to inherit from, and are verified only when `--checksum <sums-file>` is passed. `--checksum` takes the sums file the image was published with (`SHA256SUMS`, `SHA512SUMS`, `<image>.sha256`/`.sha512`, coreutils, BSD or bare-digest format), looks up the entry for the image's basename, and infers sha256 or sha512 from the digest length. A mismatch exits 1 before anything is written to storage and leaves the file in place. `--checksum` proves only that the image matches the sums file supplied with it, not that either is authentic; verify the sums file's vendor signature separately. `--checksum` is refused without `--file` or with `--skip-verify`. A `--file` import without `--checksum` prints `Importing <file> without checksum validation` to stderr and proceeds; `--name --file` does not fall back to the catalog digest. Every import records the source artifact's digest on the AMI with who verified it (`operator`, `catalog` or `unverified`, the unverified cases hashed as sha256), prints it on success, and shows it in `spx admin images describe`. The digest is of the file as supplied, not of the stored volume. `--skip-verify` bypasses verification for catalog imports and prints a warning to stderr; use only for debugging or when upstream mirrors are confirmed-broken. Import never modifies the image: the bytes written to storage are the bytes that were verified. Compressed and qcow2 images are extracted into a hidden `.import-*` scratch directory under `--tmp-dir`, which is removed when the import succeeds or fails, so the catalog directory keeps only the downloaded source (for `--name`) and the AMI manifest. Point `--tmp-dir` at a disk-backed filesystem with room for the full raw image, never at a tmpfs such as `/tmp`. A guest that needs to trust the deployment CA for in-guest SDK-over-TLS calls fetches it at runtime from `http://169.254.169.254/spinifex/ca.pem` (see the IMDS section), so a CA rotation reaches guests without re-importing anything. |
 | `spx admin images list` | — | Lists available OS images that can be imported or downloaded |
 | `spx admin images describe` | `--image-id` (required) | Read-only. Prints the AMI's name, owner, creation date, boot mode and architecture, then its source digest: algorithm and value, verification (`operator` with the sums file name, `catalog` with the sums URL, or `unverified`), and the source file name. AMIs imported before digest recording print `Source digest:  not recorded`. The digest is not exposed through `DescribeImages`. |
-| `spx admin images promote` | `--image-id` (required), `--yes` | Reads `ami-<id>/config.json`, validates the AMI is account-owned, then rewrites `ImageOwnerAlias` to `"system"` in-place. No block data is copied. The change takes effect immediately — the AMI becomes visible to all accounts via `DescribeImages`. Prompts for confirmation (skipped with `--yes`). Already-system AMIs are refused. |
-| `spx admin images remove` | `--image-id` (required), `--force`, `--yes` | Loads `ami-<id>/config.json`, walks transitive dependents — copied snapshots whose `VolumeID == imageID`, volumes whose `SnapshotID` references the internal `snap-ami-<id>` or any derived snap, and account AMIs created via `CopyImage` whose `SnapshotID` is a derived snap — then prompts (skipped with `--yes`) before deleting `ami-<id>/config.json` (the DescribeImages barrier) followed by the rest of `ami-<id>/` and `snap-ami-<id>/`. Account-owned AMIs are refused with a hint pointing at `aws ec2 deregister-image` + `aws ec2 delete-snapshot`. `--force` bypasses the dependency, ownership and config-corrupt checks for salvage of orphaned blocks. |
+| `spx admin images promote` | `--image-id` (required), `--yes` | Reads the AMI's metadata document, validates the AMI is account-owned, then rewrites its `ImageOwnerAlias` to `"system"` and re-keys its snapshot document under the global account. No block data is copied. The change takes effect immediately — the AMI becomes visible to all accounts via `DescribeImages`. Prompts for confirmation (skipped with `--yes`). Already-system AMIs are refused. |
+| `spx admin images remove` | `--image-id` (required), `--force`, `--yes` | Connects to NATS and predastore. Loads the AMI's metadata document, asks the EBS provider for the capacity of the image's volume (`absent` when the provider answers not found, `unknown` with the provider's error on any other failure, which never aborts the preview), and walks transitive dependents — copied snapshots whose `VolumeID == imageID`, volumes whose `SnapshotID` references the internal `snap-ami-<id>` or any derived snap, and account AMIs created via `CopyImage` whose `SnapshotID` is a derived snap. It then prompts (skipped with `--yes`) before deleting the image's UEFI variable store, then its metadata document (the DescribeImages barrier), then asking the provider to delete snapshot `snap-ami-<id>` and then volume `ami-<id>`. Both provider deletes succeed on an absent target, so a rerun finishes a half-done removal; the provider refuses to delete a volume that is still published. Account-owned AMIs are refused with a hint pointing at `aws ec2 deregister-image` + `aws ec2 delete-snapshot`. `--force` bypasses the dependency, ownership and corrupt-document checks for salvage of an orphaned volume and snapshot. |
 
 ### GPU Management
 

docs/DESIGN.md

@@ -111,9 +111,9 @@ Daemons (`spinifex/runtime/daemon/daemon.go`) subscribe to NATS topics and handl
 When a daemon handles `RunInstances`:
 
 1. **Resource Check**: Validates CPU/memory availability via `ResourceManager`
-2. **Volume Generation**: Creates boot and EFI volumes via Viperblock
-   A UEFI instance whose record carries a variable store pointer has no EFI volume: its `vars.fd` is fetched from the object under `uefi/` in the configured Predastore bucket that the pointer names, recorded in the `spinifex-uefi-stores` JetStream bucket (or reused when `seeded.json` on this node already names the same object) after a firmware architecture and VARS size check, and a record with both or neither is refused.
-   RunInstances still creates the EFI volume, nothing writes `vars.fd` back to the object store yet, and a terminated instance's store is removed by a cluster-wide reaper.
+2. **Volume Generation**: Creates the boot volume through the EBS provider and, for a UEFI guest, seeds its variable store as an object under `uefi/<instance-id>/` with a store record in `spinifex-uefi-stores` that points at it
+   At launch a UEFI instance's `vars.fd` is fetched from the object its store pointer names (or reused when `seeded.json` on this node already names the same object) after a firmware architecture and VARS size check, and a UEFI instance with no live store record naming a pointer, or one that still carries an EFI volume request, is refused.
+   The store is written back to the object store on write and at stop, and a terminated instance's store is removed by a cluster-wide reaper.
 3. **Volume Mount**: Publishes each volume through the EBS provider contract (`PublishVolume` on `ebs.provider.v1.{node}.mount`), receives its export
 4. **QEMU Launch**: Builds and executes QEMU command with NBD-backed disks
 5. **QMP Monitoring**: Establishes QEMU Machine Protocol connection for VM management

docs/PACKAGE_BOUNDARY_MIGRATION.md

@@ -695,7 +695,7 @@ series above; "needs decision" is open.
 | `gateway/{ecs,elbv2}` | — | Done. The action adapters moved to `domains/{ecs,elbv2}/awsapi` in `spinifex#1273` with no API, NATS subject, storage or behaviour change; the root `gateway` dispatch that still imports them is recorded `gateway` (service registration) debt. |
 | `gateway/spx` | — | Done. The gather helpers, storage-status config and image-promote request moved to `operator/admin` with no behaviour change and the package is removed. The `spinifex` Query dispatch and the `/admin/{method}` handlers that call them now live in `operator/admin` as one seam registration. |
 | `gateway/tagging` | — | Done. `domains/tagging` owns the read-only cross-resource `GetResources` aggregation and its API semantics (`0d830e901`, `aedd0b2c4`); AWSGW composes the owner listers, the EC2 tag index and ELBv2 (`f3545e3ea`); the dispatch moved to `domains/tagging/awsapi` and the root `gateway` no longer serves Tagging. The package is removed, and its two allowlist rows and frozen-legacy entry are gone. Owner tag mutation capabilities remain future direction, so the RDS and ECS tag-writing edges above stay open. |
-| `services/viperblockd` | — | Done. The adapter is `providers/ebs/viperblock` (with the read-only `dirty` reader), the shared vocabulary is `contracts/viperblockd/legacy/v1`, and the role is `runtime/roles/viperblockd`. `services/` is empty. |
+| `services/viperblockd` | — | Done. The adapter is `providers/ebs/viperblock` (with the `dirty` package, which owns the dirty-marker bucket and its read-only reader), it serves only the `providers/ebs` contract, and the role is `runtime/roles/viperblockd`. `services/` is empty. |
 | `utils` | — | Done and deleted. `ec2_subject.go` went to `contracts/ec2/v1`, the vpcd protocol to `contracts/network/v1` with the typed client in `domains/network/projection`, and the master-key loader to `domains/ec2/ebs/encryption`. |
 | `vpcd` | — | Done. The IMDS local-file reader and record-space opener moved to `domains/ec2/instance/state` and are injected through `guestmetadata.LocalStateReader`/`RecordLoader` (recorded above), so `vpcd` no longer imports `daemon` directly; its inter-process boundary now exists (`contracts/network/v1`, with `domains/network/subscribers` as the consumer and `domains/network/projection` as the producer client). The package moved unchanged to `runtime/roles/vpcd` (recorded below), and the legacy directory is removed. |
 | `daemon` | `runtime/daemon` (whole-package move, above) | Done. Q-28's ownership and deployment refactor remains deferred debt; the amended ADR-0001 row makes it not a relocation prerequisite. |

docs/admin/uefi-variable-store/README.md

@@ -0,0 +1,160 @@
+---
+title: "UEFI Variable Store"
+seoTitle: "UEFI Variable Store Durability and Recovery — Spinifex Docs"
+description: "Where a UEFI instance's variable store lives, how much a node loss can lose, how owed write-backs recover, and which firmware differences refuse a launch."
+category: "Admin"
+tags:
+  - uefi
+  - firmware
+  - instances
+  - storage
+resources:
+  - title: "Spinifex Repository"
+    url: "https://github.com/mulgadc/spinifex"
+  - title: "Host and Guest Lifecycle"
+    url: "/docs/host-lifecycle"
+---
+
+# UEFI Variable Store
+
+> Where a UEFI instance keeps its firmware variables, how durable they are, and what an operator sees when something goes wrong.
+
+## Table of Contents
+
+- [Overview](#overview)
+- [Where the Store Lives](#where-the-store-lives)
+- [Durability](#durability)
+- [Owed Write-Backs](#owed-write-backs)
+- [Starting on Another Node](#starting-on-another-node)
+- [Firmware Compatibility](#firmware-compatibility)
+- [Instances Refused at Launch](#instances-refused-at-launch)
+- [Terminate](#terminate)
+- [Images](#images)
+- [Inspecting Stores](#inspecting-stores)
+- [Metrics](#metrics)
+
+---
+
+## Overview
+
+An instance whose boot mode is `uefi` or `uefi-preferred` boots through UEFI firmware, which keeps its variables (`BootOrder`, `BootNext`, secure-boot state) in a VARS region. Spinifex gives each such instance its own variable store, so those variables survive a reboot, a stop and start, and a start on a different node. A direct-boot microvm boots without UEFI and has no store.
+
+RunInstances seeds the store from the launching image's store when the image carries one (see [Images](#images)), and from the launching node's VARS template otherwise. From then on the node running the instance writes the guest's changes back to the platform object store, and every later start boots from the last change that landed there.
+
+## Where the Store Lives
+
+A store has three parts.
+
+| Part | Where | What it holds |
+|---|---|---|
+| Store objects | The platform object store, in the bucket set as the node's Predastore bucket, under `uefi/<instance-id>/` | The VARS region, gzip-compressed, one object per write. Each write goes to a fresh key, so no write replaces an object a record points at. Each object's metadata names the instance, the writer node, the source (`template`, `write-back` or `image:<ami-id>`) and the firmware it was written under. |
+| Store record | KV bucket `spinifex-uefi-stores`, keyed by instance ID | The instance and account, the state (`live` or `deleting`), the claim generation that fences writes, and the pointer: the one object the instance boots from, with the firmware it was written under, the writer node and the write time. |
+| Local copy | `<data-dir>/uefi/<instance-id>/` on the node running the instance (`/var/lib/spinifex/uefi/<instance-id>/` by default) | `vars.fd`, the file QEMU opens as pflash unit 1, and `seeded.json`, which records the key the copy came from or last landed at, the claim generation, the firmware, and whether a write-back is owed. |
+
+The record outlives the instance record, so the store's deletion obligation is not lost when a terminated instance record expires. The bucket is never recreated empty if it goes missing, because an empty bucket would report every store as having no obligations.
+
+Only one node may write a store at a time. That node holds the store's claim, `uefi.<instance-id>` in the cluster's sync bucket, and a write moves the pointer only while the record still carries that claim's generation. A node that takes the claim stamps the record with its own generation before it boots, so a write from the previous holder is refused rather than landing over newer variables.
+
+## Durability
+
+The running node watches `vars.fd` for writes. Once the guest changes a variable, the change is written back 2 seconds after the last write, or 30 seconds after the first write in the window, whichever is sooner. A guest that writes continuously therefore still lands a write-back every 30 seconds. A node that cannot watch the file polls it every 10 seconds instead, and logs `Cannot watch the UEFI variable store; polling it for changes instead`.
+
+A write-back puts the region at a fresh key (bounded at 10 seconds) and then moves the store record's pointer to it (bounded at 10 seconds). A write-back that fails is retried 30 seconds later, and the changes stay owed until one lands.
+
+When an instance stops, the node writes the store back once QEMU has exited and before the instance's volumes unmount, then releases the claim.
+
+**What a node loss can lose:** the variable changes made since the last write-back that landed. While the object store is reachable that is at most the 30-second window plus the write in flight. While write-backs are failing, it is every change since the last one that landed, however long the failures last. The node's local copy still holds those changes, and they are recovered if the node comes back before another node takes the store over (see below).
+
+## Owed Write-Backs
+
+A write-back is owed whenever `vars.fd` may hold changes no object has. The node records that in `seeded.json` as `write_back_owed` when a window opens and clears it when the write lands. A stop whose write-back fails keeps the claim, marks the copy owed and logs `The stop could not write the UEFI variable store back; this node keeps its claim and retries`.
+
+Each node runs a `uefi-write-back` reaper on the garbage-collection sweep, every 2 minutes while KV is healthy. For each local copy whose instance is not running on the node, it:
+
+- writes an owed copy back and releases the claim once it lands (`trigger=reaper` on the write-back metric);
+- releases the claim over a clean copy once the stream leader confirms the record still points at the object the copy came from;
+- drops an owed copy whose claim another node now holds, or whose record a later claim has stamped, logging `vm/gc: dropping an owed UEFI variable store write-back; its changes are lost` and counting `mulga.vm.uefi_store.write_backs_lost`;
+- removes the copy of a store that is being deleted.
+
+The same reaper starts a writer for a running guest that has none, such as one whose reconnect after a daemon restart could not read the claim. A guest that outlives a daemon restart has its writer resumed with the window open, so any change the restart missed is written back.
+
+The reaper also deletes objects no pointer names under each prefix its write-backs touched, keeping the pointer's object and anything younger than 10 minutes.
+
+An operator does nothing to recover an owed write-back on a node that is up: the reaper retries until it lands. If it keeps failing, the node's log carries `Failed to write the UEFI variable store back` with the error, and the fix is whatever stops the node writing to the object store.
+
+## Starting on Another Node
+
+A start on a node that does not hold the claim takes it if it is free. If another node holds it, the start is refused while that node's heartbeat is fresh (under 60 seconds old) or cannot be read, because that node may still land newer variables. StartInstances returns `InsufficientInstanceCapacity` for that refusal, and the node logs `Refusing to start: another live node holds the UEFI variable store claim`. This is what an operator sees when a stopped instance's owed write-back has not landed yet: start it on the node that holds the claim, or let the reaper land the write-back and release the claim.
+
+Once the holder's heartbeat is 60 seconds old or absent, the starting node takes the claim over and logs `Took over the UEFI variable store claim from a node whose heartbeat has stopped; variable changes it never wrote back are lost`. If the old node comes back, its reaper finds the claim held elsewhere and drops its owed copy.
+
+A start on the node that holds the claim under its copy's generation boots that copy, because it may carry changes no object has.
+
+## Firmware Compatibility
+
+A store records the firmware it was written under: architecture, VARS size and the SHA-256 of the CODE blob. A node's firmware is the first installed pair from its firmware candidates (`/usr/share/OVMF/OVMF_CODE_4M.fd` and `OVMF_VARS_4M.fd` first on x86_64, `/usr/share/AAVMF/AAVMF_CODE.fd` and `AAVMF_VARS.fd` first on arm64).
+
+| Difference | Result |
+|---|---|
+| Architecture | The launch fails before the claim is taken, so the store is left untouched. |
+| VARS size | The launch fails before the claim is taken, so the store is left untouched. pflash needs a byte-exact region. |
+| CODE hash only | The instance boots. The node logs `UEFI variable store was written under a different CODE build; booting it` and counts `mulga.vm.uefi_store.firmware_mismatches` with `kind=code`. |
+
+A refused launch logs `UEFI variable store firmware does not match this node; refusing the launch` naming both firmwares, counts the mismatch with `kind=arch` or `kind=vars_size`, and records a `Server.InternalError` state reason naming both firmwares.
+
+Every node of an architecture should carry the same firmware package. A node with a different VARS size cannot boot any store written elsewhere, and a node with a different CODE build boots stores with a counted warning on every launch.
+
+## Instances Refused at Launch
+
+A UEFI instance is refused before anything about it changes when:
+
+- it still carries an EFI volume request (`<root>-efi`), which no node boots;
+- it has no store record, its record is `deleting`, or its record names no pointer;
+- its record cannot be read as valid.
+
+These starts fail with `Server.InternalError`, and the node logs `refusing to launch a UEFI instance without exactly one variable store` or `Refusing to launch a UEFI instance without a variable store` with the reason. A start whose store object is missing or does not decode to the node's VARS size also fails, and retrying does not help.
+
+## Terminate
+
+Terminate moves the store record to `deleting`, releases a claim the node holds, and removes the node's local copy. It does not delete objects. A cluster-wide `uefi-store` reaper deletes every object under the instance's prefix, then the claim, then the record, and only for an instance whose terminate is decided. It completes a terminate whose mark failed from the terminated instance record. A `deleting` record whose instance is not terminating is left for an operator and logged as `uefi-store: refusing to delete the store of an instance that is not terminating`.
+
+## Images
+
+An image made from a UEFI instance carries a copy of the instance's variable store, so instances launched from it start with the same variables.
+
+- **CreateImage** reads the instance's store record from the stream leader and copies the object its pointer names to a fresh key under `uefi/<ami-id>/`, before the root snapshot is taken. The AMI document records that key and its firmware. The image therefore carries the last write-back that landed, not changes still waiting in the write-back window. An instance with no store record, or a record with no pointer, gives an image with no store. A store being deleted refuses the request with `IncorrectInstanceState`.
+- **CopyImage** copies the source image's store to a fresh key under the new image's prefix, so deregistering either image leaves the other's store intact.
+- **RunInstances** from an image that carries a store seeds the new instance from it, with the object source `image:<ami-id>`. A store whose CODE hash differs from the node's seeds with a warning and counts `kind=code`. A store whose architecture or VARS size differs cannot boot on the node, so the instance is seeded from the node's VARS template instead, logged as `The image's UEFI variable store cannot boot under this node's firmware; seeding from the VARS template` and counted as `kind=image`.
+- **DeregisterImage** deletes the image's store objects before the AMI document. If that delete fails, the request fails with `Server.InternalError` and the image stays, so a retry finishes it. `spx admin images remove` deletes the store first in the same way.
+
+A CreateImage or CopyImage that fails after copying the store can leave an object under `uefi/<ami-id>/` with no AMI document, which `spx admin volumes orphans` reports.
+
+## Inspecting Stores
+
+`spx admin volumes orphans` reports UEFI variable stores nothing owns before it reports orphaned volumes:
+
+- `uefi/i-*/` prefixes with neither an instance record nor a store record;
+- `uefi/ami-*/` prefixes with no AMI document;
+- prefixes named for neither an instance nor an image;
+- store records that are not `deleting` and have no instance record.
+
+It only reads, and attaches to the buckets rather than creating them. It never deletes: an absent instance record is not proof of a terminate, so settling each one is an operator decision.
+
+To read one instance's store record:
+
+```bash
+nats kv get spinifex-uefi-stores i-0123456789abcdef0
+```
+
+## Metrics
+
+| Metric | Meaning |
+|---|---|
+| `mulga.vm.uefi_store.seeds` | UEFI launches by where the store came from: `source=local` (the node's copy) or `source=object`. |
+| `mulga.vm.uefi_store.write_backs` | Write-backs by `outcome` (`landed`, `failed`, `fenced`, `deleting`) and `trigger` (`change`, `stop`, `reaper`). |
+| `mulga.vm.uefi_store.write_back_skips` | Writers stopped without writing back, by the path that stopped them. |
+| `mulga.vm.uefi_store.write_backs_lost` | Owed write-backs dropped because another node took the claim over. |
+| `mulga.vm.uefi_store.pruned_objects` | Superseded store objects a prune deleted. |
+| `mulga.vm.uefi_store.firmware_mismatches` | Launches whose store was written under different firmware, by `kind`. `kind=code` boots the store, and `kind=image` seeds a new instance from the VARS template instead of the image's store. |
+| `mulga.vm.uefi_store.claim_refusals` | Starts refused because another node holds the claim and may be live, by `reason`. |
+| `mulga.vm.uefi_store.claim_takeovers` | Claims taken over from a node whose heartbeat had stopped. |

docs/testing/e2e-coverage.md

@@ -6,7 +6,7 @@ Suites live in `tests/e2e/<name>/` and build to `tests/e2e/_bin/<name>.test`. Ev
 
 | Set | Suites |
 | --- | --- |
-| `E2E_SUITES_SINGLE` | `single iam cert eks ecs storagegrowth partialblock rds quota route53 storagefault` |
+| `E2E_SUITES_SINGLE` | `single iam cert eks ecs storagegrowth partialblock rds quota route53 uefistore storagefault` |
 | `E2E_SUITES_MULTI` | `multinode lb cert quota route53 kvquorum lbrecovery instancerecovery storagefault` |
 | `E2E_SUITES_NIGHTLY_SINGLE` | `single cert iam` |
 | `E2E_SUITES_NIGHTLY_MULTI` | `multinode cert lb` |
@@ -108,6 +108,8 @@ Three entry points, in the order they run:
 
 **Its own suite and its own cell, for instancerecovery's reasons plus one.** It takes a node away, so anything sharing the environment reports that outage as its own failure; it needs its own VPC, public subnet and a backend placement no shared fixture describes; and a failure here has to read as "the load balancer did not survive its host" rather than as one more red subtest in a suite about load balancer features.
 
+**A dead data NIC is the other half, and there the state does change.** `LBSelfHealsADeadDataNIC` disables one data NIC of a running load balancer guest through a second QMP client and runs for an internet-facing NLB and an internal ALB. The backends and an unrelated customer instance in the same subnet are launched more than 21 minutes before the fault, so a repair that reached a customer instance would find one the platform has long stopped expecting to change. A state watch samples `DescribeLoadBalancers` every two seconds and requires the sequence `active`, `active_impaired`, `active` with nothing in between, while the internet-facing run also watches the name continuously and serves traffic through it within five minutes. Afterwards the guest, its interfaces and its address are unchanged but its process is not, the backends and the bystander keep their instance ID, launch time and process (ADR-0003:S7), and the daemon journals exactly one `ELBv2 data-path repair started` line carrying the fields an operator needs. The internal ALB has no client inside the VPC, so it asserts the state sequence and target health rather than traffic through its name.
+
 ### `iam` — everything IAM and STS
 
 Policy evaluation against live requests rather than unit-level: a deny applying to the next request without a new credential, resource-scoped grants naming one key pair, a user rename needing `iam:UpdateUser` on both the old and the new name, `aws:SourceIp` seeing the real client address, assumed-role sessions carrying role policies, and revocation stopping one key alone.
@@ -129,7 +131,7 @@ The client reports on its serial console because the runner has no route into th
 
 ### `eks`, `ecs`, `ecr` — container services
 
-`eks`: cluster create/delete, access entries, addon delivery, `GetToken`, kubeconfig artifacts, the EBS CSI volume path, an IRSA web-identity exchange under the stock AWS trust policy (`StringEquals` on the issuer host-path `:sub`/`:aud`, a mismatched `:sub` denied with `AccessDenied`, a `StringLike` `:sub` granted), and an IRSA pod whose spec names only a ServiceAccount with the role-arn annotation, where the pod-identity webhook injects the token volume, `AWS_*` environment, STS endpoint and CA bundle and the pod assumes the role, while a pod of an unannotated ServiceAccount is left untouched. The unmodified aws-load-balancer-controller chart 1.11.0 installs with the values an AWS user passes plus the gateway endpoint override, the Shield and WAF switches and the single-subnet feature gate, and reconciles an Ingress into an active ALB. Worker IMDS is covered too: a managed nodegroup worker reports IMDSv2 required with hop limit 2, a pod with no IRSA resolves to the node role through IMDS, and a nodegroup built from a launch template takes the template's metadata options. Control-plane isolation is covered as well: no control-plane host registers as a Node, a DaemonSet tolerating every taint runs on workers only, metrics-server is Ready, and logs work for every kube-system pod. `ecs`: clusters, container instances, the three network modes (`awsvpc`, `bridge`, `host`), task-role credentials fetched from the container credentials endpoint, a service behind an ELB, and ENI release on terminate. `ecr`: control-plane CRUD plus a real OCI data plane — docker login/build/push/pull round trips, multi-layer images, and raw blob HTTP.
+`eks`: cluster create/delete, access entries, addon delivery, `GetToken`, kubeconfig artifacts, the EBS CSI volume path, an IRSA web-identity exchange under the stock AWS trust policy (`StringEquals` on the issuer host-path `:sub`/`:aud`, a mismatched `:sub` denied with `AccessDenied`, a `StringLike` `:sub` granted), and an IRSA pod whose spec names only a ServiceAccount with the role-arn annotation, where the pod-identity webhook injects the token volume, `AWS_*` environment, STS endpoint and CA bundle and the pod assumes the role, while a pod of an unannotated ServiceAccount is left untouched. The unmodified aws-load-balancer-controller chart 1.11.0 installs with the values an AWS user passes plus the gateway endpoint override, the Shield and WAF switches and the single-subnet feature gate, and reconciles an Ingress into an active ALB. Worker IMDS is covered too: a managed nodegroup worker reports IMDSv2 required with hop limit 2, a pod with no IRSA resolves to the node role through IMDS, and a nodegroup built from a launch template takes the template's metadata options. The private-endpoint NIC of the cluster's endpoint NLB is disabled through QMP and every worker must be Ready again within ten minutes while the cluster stays `ACTIVE` and the NLB keeps its guest instance. Control-plane isolation is covered as well: no control-plane host registers as a Node, a DaemonSet tolerating every taint runs on workers only, metrics-server is Ready, and logs work for every kube-system pod. `ecs`: clusters, container instances, the three network modes (`awsvpc`, `bridge`, `host`), task-role credentials fetched from the container credentials endpoint, a service behind an ELB, and ENI release on terminate. `ecr`: control-plane CRUD plus a real OCI data plane — docker login/build/push/pull round trips, multi-layer images, and raw blob HTTP.
 
 ### `rds` — managed databases
 
@@ -142,6 +144,11 @@ The create/describe/connect path against a live cluster: a DB VM that actually b
 - **`storagegrowth`** — volume growth and image pull. Requires `external_mode=pool`; skips on a nat env, which has no EIP and therefore no guest SSH.
 - **`reboot`** — a slow guest waited for and keeping what it wrote, and a wedged guest forced off and relaunched once its budget expires. Proves a real reboot by requiring a *new* boot ID.
 - **`diskperf`** — guest disk performance gate, pinned fio profile on a dedicated blank volume. Also requires `external_mode=pool` for guest SSH.
+- **`uefistore`** — a real UEFI guest's variable writes reaching the variable store and surviving what it is built to survive. Single-node, and it runs on the node: it restarts that node's daemon and reads the store with the node's own Predastore credentials. Its three tests share one Ubuntu guest, which writes non-volatile variables (attributes 0x7) through efivarfs, and each landing is proved by fetching the object the store record points at and finding the variable's UTF-16 name and its run-unique value in the VARS region.
+  - `TestUEFIGuestVariableSurvivesStopStart`: the pointer moves to a fresh object holding the variable within 60 seconds (the documented bound is 30), the stop leaves the store holding it, and the guest reads it back after the start with a new boot ID.
+  - `TestUEFIWriterResumesAfterDaemonRestart`: a variable lands, `spinifex-daemon` restarts under the guest (whose boot ID must not change), a second variable lands, and both survive a stop and start.
+  - **`TestS7_UEFIStorePruneNeverDeletesThePointedObject` is the ADR-0003:S7 evidence for the prune.** It lands three versions of one variable and requires all three objects to remain, then polls the record and the prefix every 10 seconds until the prefix holds one object, failing if the pointed object or any object under 10 minutes old disappears. The prune must finish on an instance older than the 10-minute prune age, and the last version survives a final stop and start.
+  - Host reboot and node loss are not covered: both need host control or a second node that this suite does not have.
 
 ### Other
 

go.mod

@@ -4,27 +4,28 @@ go 1.27.3
 
 require (
 	github.com/aws/aws-sdk-go v1.55.8
-	github.com/aws/aws-sdk-go-v2 v1.47.5
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25
-	github.com/aws/aws-sdk-go-v2/config v1.33.12
-	github.com/aws/aws-sdk-go-v2/credentials v1.20.12
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5
-	github.com/aws/aws-sdk-go-v2/service/lambda v1.113.4
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6
-	github.com/aws/aws-sdk-go-v2/service/sqs v1.52.7
-	github.com/aws/smithy-go v1.28.7
+	github.com/aws/aws-sdk-go-v2 v1.47.7
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.26
+	github.com/aws/aws-sdk-go-v2/config v1.33.15
+	github.com/aws/aws-sdk-go-v2/credentials v1.20.15
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.7
+	github.com/aws/aws-sdk-go-v2/service/lambda v1.114.2
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.114.8
+	github.com/aws/aws-sdk-go-v2/service/sqs v1.52.10
+	github.com/aws/smithy-go v1.28.8
 	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/charmbracelet/bubbles v1.0.0
 	github.com/charmbracelet/bubbletea v1.3.10
 	github.com/charmbracelet/lipgloss v1.1.0
 	github.com/containerd/containerd/v2 v2.4.2
 	github.com/distribution/reference v0.6.0
+	github.com/fsnotify/fsnotify v1.10.1
 	github.com/go-chi/chi/v5 v5.5.0
 	github.com/golang-jwt/jwt/v5 v5.3.1
 	github.com/insomniacslk/dhcp v0.0.0-20260719225207-c76316d4aa82
-	github.com/jackc/pgx/v5 v5.11.0
+	github.com/jackc/pgx/v5 v5.12.0
 	github.com/klauspost/compress v1.20.3
-	github.com/klauspost/cpuid/v2 v2.4.1
+	github.com/klauspost/cpuid/v2 v2.4.2
 	github.com/miekg/dns v1.1.73
 	github.com/minio/highwayhash v1.0.4
 	github.com/mulgadc/bluebottle v1.24.0
@@ -33,14 +34,14 @@ require (
 	github.com/mulgadc/viperblock v1.24.0
 	github.com/nats-io/nats-server/v2 v2.15.2
 	github.com/nats-io/nats.go v1.55.0
-	github.com/nats-io/nkeys v0.4.16
+	github.com/nats-io/nkeys v0.4.17
 	github.com/opencontainers/runtime-spec v1.3.0
-	github.com/oracle/oci-go-sdk/v65 v65.128.0
-	github.com/ovn-kubernetes/libovsdb v0.9.0
+	github.com/oracle/oci-go-sdk/v65 v65.129.0
+	github.com/ovn-kubernetes/libovsdb v0.9.1
 	github.com/pelletier/go-toml/v2 v2.4.3
 	github.com/pterm/pterm v0.12.84
 	github.com/spf13/cobra v1.10.2
-	github.com/spf13/pflag v1.0.10
+	github.com/spf13/pflag v1.1.0
 	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.12.1
 	go.etcd.io/etcd/api/v3 v3.7.3
@@ -55,7 +56,7 @@ require (
 	go.uber.org/goleak v1.3.1-0.20260602155401-5f791e3ef83b
 	go.yaml.in/yaml/v3 v3.0.6
 	golang.org/x/crypto v0.58.0
-	golang.org/x/sync v0.24.0
+	golang.org/x/sync v0.25.0
 	golang.org/x/sys v0.49.0
 	golang.org/x/term v0.47.0
 	golang.org/x/time v0.16.0
@@ -72,17 +73,17 @@ require (
 	github.com/Microsoft/hcsshim v0.15.0-rc.4 // indirect
 	github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect
 	github.com/atotto/clipboard v0.1.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/signin v1.12.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.39.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.24 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.11 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.10 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.12.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.39.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.51.8 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
@@ -119,7 +120,6 @@ require (
 	github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
 	github.com/fatih/color v1.19.0 // indirect
 	github.com/felixge/httpsnoop v1.1.0 // indirect
-	github.com/fsnotify/fsnotify v1.10.1 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.15 // indirect
 	github.com/go-logr/logr v1.4.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
@@ -144,7 +144,7 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
-	github.com/jackc/puddle/v2 v2.2.2 // indirect
+	github.com/jackc/puddle/v2 v2.2.3 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/native v1.1.0 // indirect
 	github.com/jstemmer/go-junit-report/v2 v2.1.0 // indirect

go.sum

@@ -22,46 +22,46 @@ github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z
 github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
 github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ=
 github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk=
-github.com/aws/aws-sdk-go-v2 v1.47.5 h1:5yA+A+BiXbDq1ZLbYwPViwhEGDMdR0MEhL8Jh2LnLS4=
-github.com/aws/aws-sdk-go-v2 v1.47.5/go.mod h1:iNIkogrRvFwm+1OQIUOP2lPBtlXZboOIUWiCa+pHEMw=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 h1:MW0Iql1SOTgQPj8mCxrU/eH/K/AIDWi08ItOsXaWRLU=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25/go.mod h1:+ucVYmHYB3JqVueY0E1RHahdXAnKQbIHqybSbvOUbOM=
-github.com/aws/aws-sdk-go-v2/config v1.33.12 h1:IV1YARaXahwCqUIyP8IlWsP85Lil+5ozTozDQXPumRk=
-github.com/aws/aws-sdk-go-v2/config v1.33.12/go.mod h1:h51R06bzMVwNBRZOX5ylHhNDnkB/azZeq/LvxM6q2pQ=
-github.com/aws/aws-sdk-go-v2/credentials v1.20.12 h1:6P7Q7yH/GIwHg6MLKv3ySsBqaCoE1z+C0OTzYMenhEY=
-github.com/aws/aws-sdk-go-v2/credentials v1.20.12/go.mod h1:Kw/SaoW/wc7IqJErqXJIzKo8b7Aelxfbuc6XlbrRnsc=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 h1:iXjy2SFhGYNB1jWZBhQqq8HbhsbPF23w+/zbMvPXsTg=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5/go.mod h1:k0RUiBnyXU0AXRJ9tJmY2jQke6XUIX2PNVnWtvgaPtc=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 h1:gzIWc3DTPfPbWKrQnmL6BV6WD7NPD1htxzgdk/bQ/Z4=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8/go.mod h1:vvUA5cDAZWGo4R14FZPZfXfJCmgsafOWEq78Kg0D+ug=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 h1:PWv+vG8aCLS8qd+HPdYhK7Vees90aKHWFFQ1AGYD/8E=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8/go.mod h1:xxPZEpflYHJA12366ZGyq66z9dHPCoeCIu99Jck+MPw=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 h1:WOp5De9iWdEHDPh2Kb0tBP3vGBm/dVAuRGh4a4tqduo=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8/go.mod h1:z8W8YgfhbWl62XqcTtK+CxPrE+iHGRVUOtrrbIOp7ZY=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 h1:MCIR6iZaCKO4UE0hUvoWWLI6JVjppopV64JfAJaMjng=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23/go.mod h1:pphWFS+z0MmIPivj8c/6bATd1Z+Av7Ust75DPdEFsPA=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 h1:GACgECI422Xt5n4Pojr3O0aocPv8lfH4PJe2Aon5Wv8=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9/go.mod h1:lPfnRkHpn1qWZJLbVEw10rIqSuCee1pNQTyGkUjAYh8=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 h1:+PMpWlJYOn2tm7sm6OZ/ZLd4jStl17T7wjNapd1Rczs=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8/go.mod h1:cnG3xZLGQL4c/bt8pjdzlFtZsW7epBBuNS04a5xMjdo=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 h1:RS1Wt+K0r4X0ci9Vf4JYTUfHzsQihYA2LgGJeEneKpk=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8/go.mod h1:adMnTy5kyoUW86jhXnGSVc/mvMQI4GZd1qYWMeaS8M8=
-github.com/aws/aws-sdk-go-v2/service/lambda v1.113.4 h1:wvAdq1sNfwuedO1BX8GjY+oZWzoyeD4hhdjJI+4EPnI=
-github.com/aws/aws-sdk-go-v2/service/lambda v1.113.4/go.mod h1:HgkYN4d72Fawg3MAJQepmNnROFoN11XYu14hgFfChTY=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6 h1:to7+Vr3rLkpYRwVKJ5QRriI0eE306vrVN/6TnVHgxvY=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6/go.mod h1:Jc89ASa2k7HMAskj3bSnt8bf+k+p2IApzkoXnEjkgUw=
-github.com/aws/aws-sdk-go-v2/service/signin v1.12.0 h1:3vu/eUqh0rjh7Zy4N4YKcpZX4Aud+zinB3VsLhhlQFw=
-github.com/aws/aws-sdk-go-v2/service/signin v1.12.0/go.mod h1:wxAk0D2oVBSjuXEIzvcKxTZfM90zticdAbh/VD42L0Y=
-github.com/aws/aws-sdk-go-v2/service/sqs v1.52.7 h1:Eip+mZ0wFZY/10e5w89lWhF09Pa8tbbGwxePP3T2iMc=
-github.com/aws/aws-sdk-go-v2/service/sqs v1.52.7/go.mod h1:bXp38eBNyvSevFIao60mYcC3pQIAkPYWOddhFVv9z8E=
-github.com/aws/aws-sdk-go-v2/service/sso v1.39.0 h1:kIEpD5dUry7VET3Ybl7zj7PPHRB4lb52G9iTz+a1Tqk=
-github.com/aws/aws-sdk-go-v2/service/sso v1.39.0/go.mod h1:UGsEYgWz4Gi+46vDQ5zkFLqUgJziyOE2SXjZjgmfmHc=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0 h1:WfFev+HgTodggcNon5HieiLxxwWwb8Br8R2+nDkixvk=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0/go.mod h1:gJSdIY6YUzDvqQLcMtPVZgBRHrFwzHryvpE8Igzbxps=
-github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 h1:BVjKrtRUtSYKztBCCfE1oAB+d3ViKcW2cKZfserOqMQ=
-github.com/aws/aws-sdk-go-v2/service/sts v1.51.6/go.mod h1:eSukFLeLkPMm+1OxDR+F0joauZdyBkF47Ow7MqjZiJw=
-github.com/aws/smithy-go v1.28.7 h1:h/n1SHrwVtaDYLz8mR1/Kq4Xh4RaFbFW7bOYckg9mfE=
-github.com/aws/smithy-go v1.28.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
+github.com/aws/aws-sdk-go-v2 v1.47.7 h1:jBmtzRtGWVAzbfc4RBWo1dTte0ZZH8p9U42YJIw2+Bw=
+github.com/aws/aws-sdk-go-v2 v1.47.7/go.mod h1:I8lFSH4MGrZiw2m7miCrRYHsgpC1+HW8uiVGOiPYPcw=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.26 h1:SJNA4XaM8Z09adeTHf/vME93jzl44lJV70pSawIBXac=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.26/go.mod h1:l0ME5pd2QytH8UZzETcrt90x9Bm+3TyBVeWfVEcZ82c=
+github.com/aws/aws-sdk-go-v2/config v1.33.15 h1:eSO7lvznhY5nMN1K9xK+pPTwNNH0chVmCIzH3C5as+E=
+github.com/aws/aws-sdk-go-v2/config v1.33.15/go.mod h1:QLBxqEJFcgbrsw2dHaWMDW6DcIFWkHplXJMZGB/s2Ew=
+github.com/aws/aws-sdk-go-v2/credentials v1.20.15 h1:LUFG3364w8wKBhGxGQ2XUxCCs+yOUrSg0dWUtBJskp8=
+github.com/aws/aws-sdk-go-v2/credentials v1.20.15/go.mod h1:Q+tKQpy3sU1MjBqXzhZauN7ffcNRF9p7Rvw293sYwE4=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.7 h1:5GGOwjXU738VrsBT0EDFtP5JtLPmJQILp8RS6P+KqLM=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.7/go.mod h1:jY/VeijtHNuTiy5XI9HjpLiamtaaLw+KInZULfookJ8=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.10 h1:DKtPO+uoekvZX1tIFGVlWypYGSYkHPb7rliUI1wZtEw=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.10/go.mod h1:uAi0RDdZZz8IRpXERptWALeCgQG9Ye/HiMLxkxlt8IU=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.10 h1:r8p95YELmfmuV9VvAaFcLesVBZduCCCccxpzQKkNThw=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.10/go.mod h1:W3M52Mqai5TjgzyL/dEG+UwE+Xz9PykyjXelvbAzWIA=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.10 h1:iytL6kFchhbCWWEfLzMGq1+yRecF7+BOXnp70J9zNRM=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.10/go.mod h1:OaBcfeWMHPSyFqiHL/KN1zVIricTvYDNiO4XhR781eM=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.24 h1:jGzHjc2jDx08Rji1JTBtMxuXUDG7muTEw+ExYXqFzPg=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.24/go.mod h1:uJt75pK2cEyY80F5C+DK9YkCkJWsMkJViHxLYZRHl8Y=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.11 h1:+YhA5h9QMjVsOHfkfY/iVU7FfbT25LcGttfVzzed6v8=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.11/go.mod h1:Xn22cWokj55agZo3I0w1JFQUiHhA3qLmHi/F5t1COi0=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.10 h1:3nCD0vZOTTMypcaEEJW0rAWqEgUXVzfmNn3dDI1IQ7I=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.10/go.mod h1:KJbydA7vvlrQSceENaM/C//yI18Rt36x//hG0Bw3uW0=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.10 h1:bCz6PRKhO7ErM3F0XqLl8pbQq+r0Fra6R8PpfPnjgbo=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.10/go.mod h1:N8UvNaHipKSUnUFR07OpGdjU8B5aS/XBxgrcKURb2DQ=
+github.com/aws/aws-sdk-go-v2/service/lambda v1.114.2 h1:SbfR8t3crViVjf8bCh9ohXwQYDbPJV/6P6CE3d9iKXc=
+github.com/aws/aws-sdk-go-v2/service/lambda v1.114.2/go.mod h1:is4sHkblHDcfDUxxZVrDR9hSzf5I4id6qgX6hH0tUDA=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.114.8 h1:0g/xXBWQEyR6Mcxo1ukBYqzeRX0bQ/ZdXrtKkIo+yT0=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.114.8/go.mod h1:g6EsKHPrO5GpWYKwBfenLLtjuoPRE+WDIlLPQOIixLI=
+github.com/aws/aws-sdk-go-v2/service/signin v1.12.3 h1:8cQxxirJ0d0tvmS89hYUpnVArXIKlzwRyiaSfdaokvQ=
+github.com/aws/aws-sdk-go-v2/service/signin v1.12.3/go.mod h1:cou3YaR7daAzApgGTG9il1c3iCh/bs72w6p4WFvJnaY=
+github.com/aws/aws-sdk-go-v2/service/sqs v1.52.10 h1:ojdbdSOuwvVqcY+yfFO7Kz9Qow7brLZCALVBhorjUv8=
+github.com/aws/aws-sdk-go-v2/service/sqs v1.52.10/go.mod h1:zPf8oFenYL6wvZ8izCh+fzFsSwpNNIBQP965awjBaXQ=
+github.com/aws/aws-sdk-go-v2/service/sso v1.39.3 h1:r5DHs3x9vC1p3VQh+V9EFfHIN5QmWDdS0yKITZM2kBI=
+github.com/aws/aws-sdk-go-v2/service/sso v1.39.3/go.mod h1:u/xiahjixIqyBFiYNeQkmuimZycycBUJEomieSVsrXk=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.3 h1:ENjUto22h2MiHmv8WehZ2874SSCh5knHKsNxeyjzKpY=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.3/go.mod h1:jmVZvJzjoaStVpr4Nd11mMYCnyrq+sWT6UiDXVnHyco=
+github.com/aws/aws-sdk-go-v2/service/sts v1.51.8 h1:lervRo8V4a9tticNT+nPjqDq9olQPDmgqYczjvJbNio=
+github.com/aws/aws-sdk-go-v2/service/sts v1.51.8/go.mod h1:NBCcbmgAjvOsYwA6CPgMmipPUxsEep1JxKb3RWBr9rA=
+github.com/aws/smithy-go v1.28.8 h1:aEmdAq7gVCoGCnvf1NnELlu3I6D/nhqrC321FyFiQkQ=
+github.com/aws/smithy-go v1.28.8/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
 github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -234,10 +234,10 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
 github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
 github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
-github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg=
-github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
-github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
-github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
+github.com/jackc/pgx/v5 v5.12.0 h1:hWwax8YiGT0YiN50U1oIDn5kgR7KC2tG3rmavwpPkxY=
+github.com/jackc/pgx/v5 v5.12.0/go.mod h1:u2Fi6no03HCSBYE+ZVdLxbwdu3xyi2vU+K/YNJioSds=
+github.com/jackc/puddle/v2 v2.2.3 h1:f8HoREISb3l5eO9coPgK7U3hKLhWGokOFVcThl5jrao=
+github.com/jackc/puddle/v2 v2.2.3/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
@@ -249,8 +249,8 @@ github.com/jstemmer/go-junit-report/v2 v2.1.0 h1:X3+hPYlSczH9IMIpSC9CQSZA0L+BipY
 github.com/jstemmer/go-junit-report/v2 v2.1.0/go.mod h1:mgHVr7VUo5Tn8OLVr1cKnLuEy0M92wdRntM99h7RkgQ=
 github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4=
 github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
-github.com/klauspost/cpuid/v2 v2.4.1 h1:I9KxadAkvo+QjtcEE2f2tyfEvmIla/T0m+14PrZNl0U=
-github.com/klauspost/cpuid/v2 v2.4.1/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
+github.com/klauspost/cpuid/v2 v2.4.2 h1:xRw7IZEsAgVCp6trS/DgWbTAqpixhmywgVgBjYGg1bQ=
+github.com/klauspost/cpuid/v2 v2.4.2/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
 github.com/klauspost/reedsolomon v1.14.2 h1:SafJYwpBBQBI6amHUygcjxZjXeN2HpiENHQDwuPWCCQ=
 github.com/klauspost/reedsolomon v1.14.2/go.mod h1:yjqqjgMTQkBUHSG97/rm4zipffCNbCiZcB3kTqr++sQ=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@@ -317,8 +317,8 @@ github.com/nats-io/nats-server/v2 v2.15.2 h1:yLsgt2kslgymvxvjFq1WGWck5hkat2YYfbX
 github.com/nats-io/nats-server/v2 v2.15.2/go.mod h1:VXHCZgDglV5wO2B8ZE58yG2HzqxQxYUbQqibUh5Xcr8=
 github.com/nats-io/nats.go v1.55.0 h1:Ng8QKHdvdHIWoaiq0+d/aH1BWkxvcJxWJ0Y/9BPKx3M=
 github.com/nats-io/nats.go v1.55.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
-github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
-github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
+github.com/nats-io/nkeys v0.4.17 h1:JAJrlDhOyQqOt0a1eg4cbIZ1Kc7+vmkCigAltpwQpVA=
+github.com/nats-io/nkeys v0.4.17/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
@@ -327,10 +327,10 @@ github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJw
 github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
 github.com/opencontainers/runtime-spec v1.3.0 h1:YZupQUdctfhpZy3TM39nN9Ika5CBWT5diQ8ibYCRkxg=
 github.com/opencontainers/runtime-spec v1.3.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/oracle/oci-go-sdk/v65 v65.128.0 h1:QWh0D4VsQ+94XMFy4sapwHVuW5+VOF4dom+h8/cyJsw=
-github.com/oracle/oci-go-sdk/v65 v65.128.0/go.mod h1:87yiCsoPSaOZkpt5KIEEThoSirgT8WolahE3I5/qz78=
-github.com/ovn-kubernetes/libovsdb v0.9.0 h1:WNQM8oCmh793mKa7PIIsF5JYoCa91lgXxG1nEEDStFU=
-github.com/ovn-kubernetes/libovsdb v0.9.0/go.mod h1:mbCZzSSa9FQ0YEFgeO2rXW/OYoMh6vqNrPGpsS2XYAo=
+github.com/oracle/oci-go-sdk/v65 v65.129.0 h1:keTL99Tc9APm7Cm3QQLtiHZ3KU9ZRkiXJzzgjzjpKRk=
+github.com/oracle/oci-go-sdk/v65 v65.129.0/go.mod h1:87yiCsoPSaOZkpt5KIEEThoSirgT8WolahE3I5/qz78=
+github.com/ovn-kubernetes/libovsdb v0.9.1 h1:+ltjpC2PEObBWqg2UuG83fH+qdnTuAibXqGrWdUJ1Iw=
+github.com/ovn-kubernetes/libovsdb v0.9.1/go.mod h1:mbCZzSSa9FQ0YEFgeO2rXW/OYoMh6vqNrPGpsS2XYAo=
 github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY=
 github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
 github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
@@ -375,8 +375,8 @@ github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qq
 github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
 github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
 github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
-github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/pflag v1.1.0 h1:6W6MbVUrnMb8PEhjkqwsVLY7ryuzRfFHF4mQd7L94ag=
+github.com/spf13/pflag v1.1.0/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
 github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -485,8 +485,8 @@ golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
-golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
+golang.org/x/sync v0.25.0 h1:dJnifr0nMV8/SBnBvdo4eSGGnrc3O4FFx6IqvtLcmRs=
+golang.org/x/sync v0.25.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

internal/testkit/recordingstore/recordingstore.go

@@ -1,8 +1,9 @@
 // Package recordingstore provides an objectstore.ObjectStore fake that records
-// every key it is asked to read and every prefix it is asked to list. It exists
-// so tests can assert an access pattern rather than only an answer: that an
-// account-scoped listing never touched another account's prefix, which an
-// in-memory filter over a whole-cluster read would satisfy just as well.
+// every key it is asked to read, head, write or delete and every prefix it is
+// asked to list. It exists so tests can assert an access pattern rather than
+// only an answer: that an account-scoped listing never touched another
+// account's prefix, which an in-memory filter over a whole-cluster read would
+// satisfy just as well.
 package recordingstore
 
 import (
@@ -16,14 +17,17 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 )
 
-// Store wraps a MemoryObjectStore, appending to Gets and ListPrefixes before
-// delegating. Both slices are safe to read once the call under test returns.
+// Store wraps a MemoryObjectStore, recording each call's key or prefix before
+// delegating. The recordings are safe to read once the call under test returns.
 type Store struct {
 	*objectstore.MemoryObjectStore
 
 	mu           sync.Mutex
 	gets         []string
 	listPrefixes []string
+	heads        []string
+	puts         []string
+	deletes      []string
 }
 
 // New returns a recording store backed by an empty in-memory store.
@@ -47,6 +51,27 @@ func (s *Store) ListObjectsV2(ctx context.Context, input *s3.ListObjectsV2Input)
 	return s.MemoryObjectStore.ListObjectsV2(ctx, input)
 }
 
+func (s *Store) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
+	s.record(&s.heads, aws.StringValue(input.Key))
+	return s.MemoryObjectStore.HeadObject(ctx, input)
+}
+
+func (s *Store) PutObject(ctx context.Context, input *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
+	s.record(&s.puts, aws.StringValue(input.Key))
+	return s.MemoryObjectStore.PutObject(ctx, input)
+}
+
+func (s *Store) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+	s.record(&s.deletes, aws.StringValue(input.Key))
+	return s.MemoryObjectStore.DeleteObject(ctx, input)
+}
+
+func (s *Store) record(into *[]string, key string) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	*into = append(*into, key)
+}
+
 // Gets returns every key read so far.
 func (s *Store) Gets() []string {
 	s.mu.Lock()
@@ -61,12 +86,33 @@ func (s *Store) ListPrefixes() []string {
 	return append([]string(nil), s.listPrefixes...)
 }
 
+// Heads returns every key headed so far.
+func (s *Store) Heads() []string {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return append([]string(nil), s.heads...)
+}
+
+// Puts returns every key written so far.
+func (s *Store) Puts() []string {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return append([]string(nil), s.puts...)
+}
+
+// Deletes returns every key deleted so far.
+func (s *Store) Deletes() []string {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return append([]string(nil), s.deletes...)
+}
+
 // Reset drops what has been recorded, so a test can seed fixtures and then
 // assert only on the call it is actually exercising.
 func (s *Store) Reset() {
 	s.mu.Lock()
 	defer s.mu.Unlock()
-	s.gets, s.listPrefixes = nil, nil
+	s.gets, s.listPrefixes, s.heads, s.puts, s.deletes = nil, nil, nil, nil, nil
 }
 
 // TouchedPrefix reports whether any recorded get or listing named a key under

scripts/build-microvm-image.sh

@@ -209,6 +209,7 @@ echo "[build-microvm-image] nginx: binary + stream module present"
 #   virtio*.ko*    — virtio bus, net, blk, rng, console, and helpers
 #   *fw_cfg*.ko*   — QEMU fw_cfg sysfs driver (delivers boot blobs)
 #   *failover*.ko* — net_failover + failover (virtio_net deps for live migration)
+#   af_packet.ko*  — AF_PACKET sockets (lb-agent's per-NIC gateway ARP probe)
 # All other loadable modules are removed. Drivers compiled into the kernel
 # (CONFIG_VIRTIO_MMIO=y etc.) are unaffected — they live in vmlinuz, not here.
 echo "[build-microvm-image] stripping kernel modules to virtio+fw_cfg only..."
@@ -222,6 +223,7 @@ for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
     find "$kernel_dir" -name "virtio*.ko.gz" -o -name "virtio*.ko" \
                        -o -name "*fw_cfg*.ko.gz" -o -name "*fw_cfg*.ko" \
                        -o -name "*failover*.ko.gz" -o -name "*failover*.ko" \
+                       -o -name "af_packet.ko.gz" -o -name "af_packet.ko" \
         2>/dev/null | while read -r mod; do
         rel="${mod#"$kernel_dir/"}"
         dest_dir="$tmp_keep/$(dirname "$rel")"
@@ -240,6 +242,15 @@ for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
     done)
     rm -rf "$tmp_keep"
 
+    # lb-agent's data-path probe needs AF_PACKET; without it every LB would
+    # report a dead NIC. Accept a kernel with CONFIG_PACKET built in.
+    if ! find "$kernel_dir" -name "af_packet.ko*" | grep -q .; then
+        if ! grep -q "CONFIG_PACKET=y" "$KERNEL_CONFIG" 2>/dev/null; then
+            echo "ERROR: af_packet.ko missing from the kept modules — lb-agent's data-path probe cannot open its socket" >&2
+            exit 1
+        fi
+    fi
+
     # Decompress kept modules to plain .ko. Alpine ships .ko.gz, but a host
     # kmod that lacks gzip support (e.g. Debian trixie's kmod 34) silently
     # produces an EMPTY modules.dep from compressed modules — the guest's
@@ -310,12 +321,19 @@ echo "[build-microvm-image] building initramfs: $INITRAMFS_OUT"
 # Create the static device nodes and pack the archive inside a single fakeroot
 # session: mknod/cpio see faked device entries and write real char-device
 # records into the cpio, while the host filesystem never gets an actual node.
-fakeroot sh -c '
+# Pack to a .partial and list it before renaming, so a pack cut short (fakeroot
+# losing its daemon, a full disk) fails here and never leaves a stale archive.
+fakeroot bash -c '
+    set -euo pipefail
     cd "$1" || exit 1
     mknod -m 600 dev/console c 5 1
     mknod -m 666 dev/null    c 1 3
     find . | cpio --quiet -o -H newc | gzip -9 > "$2"
-' _ "$CHROOT_DIR" "$INITRAMFS_OUT"
+' _ "$CHROOT_DIR" "$INITRAMFS_OUT.partial" \
+    && gzip -t "$INITRAMFS_OUT.partial" \
+    && zcat "$INITRAMFS_OUT.partial" | cpio -t --quiet >/dev/null \
+    || { rm -f "$INITRAMFS_OUT.partial" "$INITRAMFS_OUT"; echo "ERROR: initramfs $INITRAMFS_OUT is incomplete or unreadable" >&2; exit 1; }
+mv -f "$INITRAMFS_OUT.partial" "$INITRAMFS_OUT"
 
 # --- Log artifact sizes ---
 echo ""

scripts/lib/microvm-image.sh

@@ -382,10 +382,17 @@ microvm_pack_initramfs() {
     # Create the static device nodes and pack the archive inside a single fakeroot
     # session: mknod/cpio see faked device entries and write real char-device
     # records into the cpio, while the host filesystem never gets an actual node.
-    fakeroot sh -c '
+    # Pack to a .partial and list it before renaming, so a pack cut short fails
+    # here and never leaves a stale archive.
+    fakeroot bash -c '
+        set -euo pipefail
         cd "$1" || exit 1
         mknod -m 600 dev/console c 5 1
         mknod -m 666 dev/null    c 1 3
         find . | cpio --quiet -o -H newc | gzip -9 > "$2"
-    ' _ "$CHROOT_DIR" "$INITRAMFS_OUT"
+    ' _ "$CHROOT_DIR" "$INITRAMFS_OUT.partial" \
+        && gzip -t "$INITRAMFS_OUT.partial" \
+        && zcat "$INITRAMFS_OUT.partial" | cpio -t --quiet >/dev/null \
+        || { rm -f "$INITRAMFS_OUT.partial" "$INITRAMFS_OUT"; echo "ERROR: initramfs $INITRAMFS_OUT is incomplete or unreadable" >&2; exit 1; }
+    mv -f "$INITRAMFS_OUT.partial" "$INITRAMFS_OUT"
 }

scripts/lib/microvm-image_test.sh

@@ -0,0 +1,77 @@
+#!/bin/sh
+# microvm-image_test.sh — microvm_pack_initramfs publishes only an archive that
+# lists completely, removes a rejected one, and succeeds again on a retry.
+#
+# The real pack step runs against a small chroot; fakeroot and mknod are
+# replaced so it needs no root, and cpio is wrapped to simulate a cut pack.
+set -eu
+
+SCRIPTS_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT
+
+fails=0
+pass() { echo "  ok   - $1"; }
+fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
+
+echo "-- microvm_pack_initramfs"
+
+if ! command -v cpio >/dev/null 2>&1 || ! command -v bash >/dev/null 2>&1; then
+    echo "  skip - cpio or bash not installed"
+    exit 0
+fi
+REAL_CPIO=$(command -v cpio)
+
+mkdir -p "$TMP/bin" "$TMP/chroot/dev" "$TMP/chroot/etc" "$TMP/stage"
+printf '#!/bin/sh\nexec "$@"\n' > "$TMP/bin/fakeroot"
+printf '#!/bin/sh\nexit 0\n' > "$TMP/bin/mknod"
+chmod +x "$TMP/bin/fakeroot" "$TMP/bin/mknod"
+# Large enough that a cut at 64 KiB loses whole entries and the trailer.
+head -c 400000 /dev/urandom > "$TMP/chroot/etc/blob"
+printf 'init\n' > "$TMP/chroot/init"
+
+# pack <cpio mode>: runs the real step; mode "cut" truncates the archive stream
+# and exits 0, mode "fail" writes it whole and exits 1, mode "ok" is untouched.
+pack() {
+    cat > "$TMP/bin/cpio" <<EOF
+#!/bin/bash
+case " \$* " in *" -o "*) ;; *) exec "$REAL_CPIO" "\$@" ;; esac
+case "$1" in
+    cut)  "$REAL_CPIO" "\$@" | head -c 65536; exit 0 ;;
+    fail) "$REAL_CPIO" "\$@"; exit 1 ;;
+    *)    exec "$REAL_CPIO" "\$@" ;;
+esac
+EOF
+    chmod +x "$TMP/bin/cpio"
+    PATH="$TMP/bin:$PATH" bash -c '
+        set -euo pipefail
+        . "$1/lib/microvm-image.sh"
+        TAG="[test]" CHROOT_DIR="$2" STAGE_DIR="$3"
+        microvm_pack_initramfs
+    ' _ "$SCRIPTS_DIR" "$TMP/chroot" "$TMP/stage" > "$TMP/log" 2>&1
+}
+
+OUT="$TMP/stage/initramfs.cpio.gz"
+
+for mode in cut fail; do
+    printf 'stale\n' > "$OUT"
+    if pack "$mode"; then
+        fail "$mode: pack succeeded, want failure"
+    elif [ -e "$OUT" ] || [ -e "$OUT.partial" ]; then
+        fail "$mode: rejected archive left behind"
+    else
+        pass "$mode: pack fails and leaves no archive"
+    fi
+done
+
+if ! pack ok; then
+    fail "retry after failure: $(tr '\n' ' ' < "$TMP/log")"
+elif [ -e "$OUT.partial" ]; then
+    fail "retry after failure: partial file left behind"
+elif ! zcat "$OUT" | "$REAL_CPIO" -t --quiet 2>/dev/null | grep -qx 'etc/blob'; then
+    fail "retry after failure: archive does not list etc/blob"
+else
+    pass "retry after failure publishes a complete archive"
+fi
+
+[ "$fails" -eq 0 ] || exit 1

spinifex/agents/elbv2/agent.go

@@ -21,6 +21,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"sync"
 	"syscall"
 	"time"
 
@@ -94,6 +95,18 @@ type Agent struct {
 	statsFn func(socketPath string) ([]ServerStatus, error)
 	// For testing: override the active health prober (nginx/NLB).
 	probeFn func(targets []HealthTarget) []ServerStatus
+
+	// Data-path probe: netcfg and sysfs locations, the ARP exchange, the
+	// instance ID lookup and per-NIC miss counters (overridable in tests).
+	netcfgPath   string
+	sysNetDir    string
+	probeWait    time.Duration
+	arpProbeFn   arpProbeFn
+	instanceIDFn func() (string, error)
+	instanceID   string
+	idSkip       int
+	probeMu      sync.Mutex
+	probeStates  map[string]*nicProbeState
 }
 
 // New creates a new LB agent for the given load balancer.
@@ -150,6 +163,12 @@ func New(lbID, gatewayURL, accessKey, secretKey, region string) (*Agent, error)
 		reloadNginxFn: reloadNginx,
 		statsFn:       queryHAProxyStats,
 		probeFn:       probeHealthTargets,
+		netcfgPath:    netcfgPath,
+		sysNetDir:     sysClassNet,
+		probeWait:     probeWait,
+		arpProbeFn:    arpProbe,
+		instanceIDFn:  fetchInstanceID,
+		probeStates:   map[string]*nicProbeState{},
 	}, nil
 }
 
@@ -207,7 +226,7 @@ func (a *Agent) tick() {
 		}
 	}
 
-	resp, err := a.sendHeartbeat(servers)
+	resp, err := a.sendHeartbeat(servers, a.dataPathEvidence())
 	if err != nil {
 		// Include the dial target so the serial console disambiguates a
 		// transport failure (never reached AWSGW — "send request: dial ...")
@@ -288,7 +307,7 @@ func (a *Agent) enginePaths(engine string) (configPath, pidPath, certDir string,
 }
 
 // sendHeartbeat sends a heartbeat with health report to the gateway.
-func (a *Agent) sendHeartbeat(servers []ServerStatus) (*heartbeatResponse, error) {
+func (a *Agent) sendHeartbeat(servers []ServerStatus, ev dataPathEvidence) (*heartbeatResponse, error) {
 	params := url.Values{}
 	params.Set("Action", "LBAgentHeartbeat")
 	params.Set("Version", "2015-12-01")
@@ -300,6 +319,7 @@ func (a *Agent) sendHeartbeat(servers []ServerStatus) (*heartbeatResponse, error
 		params.Set("Servers.member."+idx+".Server", s.Server)
 		params.Set("Servers.member."+idx+".Status", s.Status)
 	}
+	ev.addTo(params)
 
 	body, err := a.signedPost(params)
 	if err != nil {

spinifex/agents/elbv2/arp_linux.go

@@ -0,0 +1,62 @@
+//go:build linux
+
+package elbv2
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"net/netip"
+	"syscall"
+	"time"
+)
+
+// arpProbe sends one ARP request for gateway out of iface on an AF_PACKET
+// socket bound to it and waits for the gateway's reply on the same socket.
+func arpProbe(iface string, mac net.HardwareAddr, src, gateway netip.Addr, wait time.Duration) error {
+	ifi, err := net.InterfaceByName(iface)
+	if err != nil {
+		return fmt.Errorf("interface %s: %w", iface, err)
+	}
+
+	proto := htons(syscall.ETH_P_ARP)
+	fd, err := syscall.Socket(syscall.AF_PACKET, syscall.SOCK_RAW, int(proto))
+	if err != nil {
+		return fmt.Errorf("packet socket: %w", err)
+	}
+	defer syscall.Close(fd)
+
+	addr := &syscall.SockaddrLinklayer{Protocol: proto, Ifindex: ifi.Index}
+	if err := syscall.Bind(fd, addr); err != nil {
+		return fmt.Errorf("bind %s: %w", iface, err)
+	}
+
+	deadline := time.Now().Add(wait)
+	if err := syscall.Sendto(fd, buildARPRequest(mac, src, gateway), 0, addr); err != nil {
+		return fmt.Errorf("send ARP request: %w", err)
+	}
+
+	buf := make([]byte, 128)
+	for {
+		left := time.Until(deadline)
+		if left <= 0 {
+			return errARPNoReply
+		}
+		tv := syscall.NsecToTimeval(int64(left))
+		if err := syscall.SetsockoptTimeval(fd, syscall.SOL_SOCKET, syscall.SO_RCVTIMEO, &tv); err != nil {
+			return fmt.Errorf("set receive deadline: %w", err)
+		}
+		n, _, err := syscall.Recvfrom(fd, buf, 0)
+		if err != nil {
+			if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EINTR) {
+				continue
+			}
+			return fmt.Errorf("receive ARP reply: %w", err)
+		}
+		if isARPReplyFrom(buf[:n], mac, gateway) {
+			return nil
+		}
+	}
+}
+
+func htons(v uint16) uint16 { return v<<8 | v>>8 }

spinifex/agents/elbv2/arp_other.go

@@ -0,0 +1,15 @@
+//go:build !linux
+
+package elbv2
+
+import (
+	"errors"
+	"net"
+	"net/netip"
+	"time"
+)
+
+// arpProbe is only meaningful inside the Linux guest.
+func arpProbe(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error {
+	return errors.New("ARP probe is only supported on linux")
+}

spinifex/agents/elbv2/datapath.go

@@ -0,0 +1,303 @@
+package elbv2
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"net"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+const (
+	// netcfgPath is the fw_cfg blob the microVM init script reads to configure NICs.
+	netcfgPath = "/sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw"
+
+	sysClassNet = "/sys/class/net"
+
+	// probeFailureThreshold is the number of consecutive missed probes after
+	// which a NIC is reported as failing (about 15 s at the poll interval).
+	probeFailureThreshold = 3
+
+	// arpExchangeWait bounds one ARP exchange. probeWait bounds how long a tick
+	// waits for all probes, so a stuck probe never delays the heartbeat.
+	arpExchangeWait = time.Second
+	probeWait       = 1500 * time.Millisecond
+)
+
+// nicConfig is one NIC of the netcfg blob that has a gateway to probe.
+type nicConfig struct {
+	MAC     string
+	Addr    netip.Addr
+	Gateway netip.Addr
+}
+
+// NICReport is the per-NIC data-path evidence carried in the heartbeat.
+type NICReport struct {
+	MAC                 string
+	ProbeOK             bool
+	ConsecutiveFailures int64
+	RxPackets           int64
+	TxPackets           int64
+	RxDropped           int64
+	TxDropped           int64
+}
+
+// arpProbeFn sends one ARP request for gateway out of iface and returns nil
+// when the gateway's reply arrives within timeout.
+type arpProbeFn func(iface string, mac net.HardwareAddr, src, gateway netip.Addr, timeout time.Duration) error
+
+// parseNetcfg reads the KEY=value netcfg blob and returns the NICs that have a
+// gateway, in NIC order. The mgmt NIC carries no gateway and is skipped, as
+// is any NIC without a usable MAC, address or gateway.
+func parseNetcfg(blob []byte) []nicConfig {
+	type raw struct{ mac, cidr, gw string }
+	byIndex := map[int]*raw{}
+
+	sc := bufio.NewScanner(bytes.NewReader(blob))
+	for sc.Scan() {
+		line := strings.TrimSpace(sc.Text())
+		key, val, ok := strings.Cut(line, "=")
+		if !ok || !strings.HasPrefix(key, "NIC") {
+			continue
+		}
+		idxStr, field, ok := strings.Cut(strings.TrimPrefix(key, "NIC"), "_")
+		if !ok {
+			continue
+		}
+		idx, err := strconv.Atoi(idxStr)
+		if err != nil {
+			continue
+		}
+		r := byIndex[idx]
+		if r == nil {
+			r = &raw{}
+			byIndex[idx] = r
+		}
+		switch field {
+		case "MAC":
+			r.mac = strings.ToLower(val)
+		case "CIDR":
+			r.cidr = val
+		case "GW":
+			r.gw = val
+		}
+	}
+
+	idxs := make([]int, 0, len(byIndex))
+	for i := range byIndex {
+		idxs = append(idxs, i)
+	}
+	sort.Ints(idxs)
+
+	var out []nicConfig
+	for _, i := range idxs {
+		r := byIndex[i]
+		if r.mac == "" || r.cidr == "" || r.gw == "" {
+			continue
+		}
+		prefix, err := netip.ParsePrefix(r.cidr)
+		if err != nil || !prefix.Addr().Is4() {
+			continue
+		}
+		gw, err := netip.ParseAddr(r.gw)
+		if err != nil || !gw.Is4() {
+			continue
+		}
+		out = append(out, nicConfig{MAC: r.mac, Addr: prefix.Addr(), Gateway: gw})
+	}
+	return out
+}
+
+// ifaceByMAC returns the interface under sysNetDir whose address is mac.
+func ifaceByMAC(sysNetDir, mac string) (string, error) {
+	entries, err := os.ReadDir(sysNetDir)
+	if err != nil {
+		return "", err
+	}
+	for _, e := range entries {
+		b, err := os.ReadFile(filepath.Join(sysNetDir, e.Name(), "address"))
+		if err != nil {
+			continue
+		}
+		if strings.EqualFold(strings.TrimSpace(string(b)), mac) {
+			return e.Name(), nil
+		}
+	}
+	return "", fmt.Errorf("no interface with MAC %s", mac)
+}
+
+// readCounters fills the sysfs statistics of iface into r.
+func readCounters(sysNetDir, iface string, r *NICReport) {
+	read := func(name string) int64 {
+		b, err := os.ReadFile(filepath.Join(sysNetDir, iface, "statistics", name))
+		if err != nil {
+			return 0
+		}
+		v, _ := strconv.ParseInt(strings.TrimSpace(string(b)), 10, 64)
+		return v
+	}
+	r.RxPackets = read("rx_packets")
+	r.TxPackets = read("tx_packets")
+	r.RxDropped = read("rx_dropped")
+	r.TxDropped = read("tx_dropped")
+}
+
+var errARPNoReply = errors.New("no ARP reply from the gateway")
+
+// buildARPRequest returns the Ethernet frame asking who has target.
+func buildARPRequest(mac net.HardwareAddr, src, target netip.Addr) []byte {
+	f := make([]byte, 0, 42)
+	f = append(f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff)
+	f = append(f, mac...)
+	f = binary.BigEndian.AppendUint16(f, 0x0806)
+	f = binary.BigEndian.AppendUint16(f, 1)      // Ethernet
+	f = binary.BigEndian.AppendUint16(f, 0x0800) // IPv4
+	f = append(f, 6, 4)
+	f = binary.BigEndian.AppendUint16(f, 1) // request
+	f = append(f, mac...)
+	s := src.As4()
+	f = append(f, s[:]...)
+	f = append(f, 0, 0, 0, 0, 0, 0)
+	t := target.As4()
+	f = append(f, t[:]...)
+	return f
+}
+
+// isARPReplyFrom reports whether frame is an ARP reply sent by sender to mac.
+func isARPReplyFrom(frame []byte, mac net.HardwareAddr, sender netip.Addr) bool {
+	if len(frame) < 42 || binary.BigEndian.Uint16(frame[12:14]) != 0x0806 {
+		return false
+	}
+	if binary.BigEndian.Uint16(frame[20:22]) != 2 || !bytes.Equal(frame[32:38], mac) {
+		return false
+	}
+	s := sender.As4()
+	return bytes.Equal(frame[28:32], s[:])
+}
+
+// nicProbeState carries the miss counter of one NIC across ticks. A probe that
+// has not returned is still in flight, and counts as a miss without starting
+// another goroutine behind it.
+type nicProbeState struct {
+	mu       sync.Mutex
+	failures int64
+	inflight bool
+}
+
+type probeResult struct {
+	ok  bool
+	set bool
+}
+
+// probeNICs probes every gateway-bearing data NIC once, in parallel, and
+// returns within probeWait however long a probe blocks.
+func (a *Agent) probeNICs() []NICReport {
+	blob, err := os.ReadFile(a.netcfgPath)
+	if err != nil {
+		return nil
+	}
+	nics := parseNetcfg(blob)
+	if len(nics) == 0 {
+		return nil
+	}
+
+	reports := make([]NICReport, len(nics))
+	results := make([]chan probeResult, len(nics))
+	for i, n := range nics {
+		results[i] = make(chan probeResult, 1)
+		st := a.probeState(n.MAC)
+
+		st.mu.Lock()
+		busy := st.inflight
+		if !busy {
+			st.inflight = true
+		}
+		st.mu.Unlock()
+		if busy {
+			results[i] <- probeResult{set: true}
+			continue
+		}
+
+		go func(n nicConfig, st *nicProbeState, out chan<- probeResult) {
+			ok := a.probeOne(n)
+			st.mu.Lock()
+			st.inflight = false
+			st.mu.Unlock()
+			out <- probeResult{ok: ok, set: true}
+		}(n, st, results[i])
+	}
+
+	deadline := time.NewTimer(a.probeWait)
+	defer deadline.Stop()
+	expired := false
+	for i, n := range nics {
+		var res probeResult
+		if expired {
+			select {
+			case res = <-results[i]:
+			default:
+			}
+		} else {
+			select {
+			case res = <-results[i]:
+			case <-deadline.C:
+				// Out of time: an unfinished probe is a miss for this tick.
+				expired = true
+			}
+		}
+
+		st := a.probeState(n.MAC)
+		st.mu.Lock()
+		if res.ok {
+			st.failures = 0
+		} else {
+			st.failures++
+		}
+		failures := st.failures
+		st.mu.Unlock()
+
+		r := NICReport{
+			MAC:                 n.MAC,
+			ProbeOK:             failures < probeFailureThreshold,
+			ConsecutiveFailures: failures,
+		}
+		if iface, err := ifaceByMAC(a.sysNetDir, n.MAC); err == nil {
+			readCounters(a.sysNetDir, iface, &r)
+		}
+		reports[i] = r
+	}
+	return reports
+}
+
+// probeOne runs one ARP exchange for n, false on any failure.
+func (a *Agent) probeOne(n nicConfig) bool {
+	iface, err := ifaceByMAC(a.sysNetDir, n.MAC)
+	if err != nil {
+		return false
+	}
+	hw, err := net.ParseMAC(n.MAC)
+	if err != nil {
+		return false
+	}
+	return a.arpProbeFn(iface, hw, n.Addr, n.Gateway, arpExchangeWait) == nil
+}
+
+func (a *Agent) probeState(mac string) *nicProbeState {
+	a.probeMu.Lock()
+	defer a.probeMu.Unlock()
+	st := a.probeStates[mac]
+	if st == nil {
+		st = &nicProbeState{}
+		a.probeStates[mac] = st
+	}
+	return st
+}

spinifex/agents/elbv2/datapath_test.go

@@ -0,0 +1,484 @@
+package elbv2
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/netip"
+	"net/url"
+	"os"
+	"path/filepath"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+const (
+	testDataMAC = "02:00:00:aa:00:01"
+	testMgmtMAC = "02:00:00:aa:00:02"
+	testNetcfg  = "NIC0_MAC=" + testDataMAC + "\nNIC0_CIDR=10.0.1.5/24\nNIC0_GW=10.0.1.1\nNIC0_DEFAULT=1\n" +
+		"NIC1_MAC=" + testMgmtMAC + "\nNIC1_CIDR=10.255.0.9/24\nNIC1_DEFAULT=0\n"
+)
+
+// datapathAgent returns a test agent whose netcfg and sysfs live in a temp dir.
+func datapathAgent(t *testing.T, gwURL, netcfg string) *Agent {
+	t.Helper()
+	a := newTestAgent(t, gwURL)
+	dir := t.TempDir()
+
+	a.netcfgPath = filepath.Join(dir, "netcfg")
+	if err := os.WriteFile(a.netcfgPath, []byte(netcfg), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	a.sysNetDir = filepath.Join(dir, "net")
+	for name, mac := range map[string]string{"ens3": testDataMAC, "ens4": testMgmtMAC} {
+		stats := filepath.Join(a.sysNetDir, name, "statistics")
+		if err := os.MkdirAll(stats, 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(filepath.Join(a.sysNetDir, name, "address"), []byte(mac+"\n"), 0o600); err != nil {
+			t.Fatal(err)
+		}
+		for file, v := range map[string]string{"rx_packets": "100", "tx_packets": "200", "rx_dropped": "3", "tx_dropped": "4"} {
+			if err := os.WriteFile(filepath.Join(stats, file), []byte(v+"\n"), 0o600); err != nil {
+				t.Fatal(err)
+			}
+		}
+	}
+
+	a.instanceIDFn = func() (string, error) { return "i-0123456789abcdef0", nil }
+	return a
+}
+
+func TestParseNetcfg_SkipsMgmtNICAndMissingGateway(t *testing.T) {
+	blob := testNetcfg +
+		"NIC2_MAC=02:00:00:aa:00:03\nNIC2_CIDR=10.0.2.5/24\nNIC2_GW=10.0.2.1\nNIC2_DEFAULT=0\n" +
+		"NIC3_MAC=02:00:00:aa:00:04\nNIC3_DHCP=1\nNIC3_DEFAULT=0\n" +
+		"NIC4_CIDR=10.0.4.5/24\nNIC4_GW=10.0.4.1\n" +
+		"NIC5_MAC=02:00:00:aa:00:06\nNIC5_CIDR=not-a-cidr\nNIC5_GW=10.0.5.1\n"
+
+	got := parseNetcfg([]byte(blob))
+	if len(got) != 2 {
+		t.Fatalf("got %d NICs, want the 2 with a gateway: %+v", len(got), got)
+	}
+	if got[0].MAC != testDataMAC || got[0].Gateway != netip.MustParseAddr("10.0.1.1") || got[0].Addr != netip.MustParseAddr("10.0.1.5") {
+		t.Errorf("first NIC = %+v", got[0])
+	}
+	if got[1].MAC != "02:00:00:aa:00:03" || got[1].Gateway != netip.MustParseAddr("10.0.2.1") {
+		t.Errorf("second NIC = %+v", got[1])
+	}
+}
+
+func TestParseNetcfg_OrdersByNICIndexAndLowercasesMAC(t *testing.T) {
+	blob := "NIC10_MAC=02:00:00:AA:00:0A\nNIC10_CIDR=10.0.9.5/24\nNIC10_GW=10.0.9.1\n" +
+		"NIC2_MAC=02:00:00:AA:00:02\nNIC2_CIDR=10.0.2.5/24\nNIC2_GW=10.0.2.1\n"
+	got := parseNetcfg([]byte(blob))
+	if len(got) != 2 || got[0].MAC != "02:00:00:aa:00:02" || got[1].MAC != "02:00:00:aa:00:0a" {
+		t.Fatalf("order or case wrong: %+v", got)
+	}
+}
+
+func TestIfaceByMACAndCounters(t *testing.T) {
+	a := datapathAgent(t, "http://unused", testNetcfg)
+
+	iface, err := ifaceByMAC(a.sysNetDir, "02:00:00:AA:00:01")
+	if err != nil || iface != "ens3" {
+		t.Fatalf("iface = %q, err = %v", iface, err)
+	}
+	if _, err := ifaceByMAC(a.sysNetDir, "02:00:00:ff:ff:ff"); err == nil {
+		t.Error("an unknown MAC must not resolve to an interface")
+	}
+
+	var r NICReport
+	readCounters(a.sysNetDir, iface, &r)
+	if r.RxPackets != 100 || r.TxPackets != 200 || r.RxDropped != 3 || r.TxDropped != 4 {
+		t.Errorf("counters = %+v", r)
+	}
+}
+
+func TestARPFrames_RequestShapeAndReplyMatching(t *testing.T) {
+	mac, _ := net.ParseMAC(testDataMAC)
+	src, gw := netip.MustParseAddr("10.0.1.5"), netip.MustParseAddr("10.0.1.1")
+
+	req := buildARPRequest(mac, src, gw)
+	if len(req) != 42 || req[12] != 0x08 || req[13] != 0x06 || req[21] != 1 {
+		t.Fatalf("request frame malformed: % x", req)
+	}
+	if string(req[38:42]) != string(gw.AsSlice()) {
+		t.Errorf("target IP not the gateway: % x", req[38:42])
+	}
+
+	reply := make([]byte, 42)
+	copy(reply[0:6], mac)
+	reply[12], reply[13] = 0x08, 0x06
+	reply[21] = 2
+	copy(reply[28:32], gw.AsSlice())
+	copy(reply[32:38], mac)
+	if !isARPReplyFrom(reply, mac, gw) {
+		t.Error("a reply from the gateway to our MAC must match")
+	}
+	if isARPReplyFrom(reply, mac, netip.MustParseAddr("10.0.1.99")) {
+		t.Error("a reply from another host must not match")
+	}
+	reply[21] = 1
+	if isARPReplyFrom(reply, mac, gw) {
+		t.Error("a request must not count as a reply")
+	}
+	if isARPReplyFrom(reply[:20], mac, gw) {
+		t.Error("a short frame must not match")
+	}
+}
+
+func TestProbeNICs_ProbesOnlyTheGatewayNIC(t *testing.T) {
+	a := datapathAgent(t, "http://unused", testNetcfg)
+	var mu sync.Mutex
+	var probed []string
+	a.arpProbeFn = func(iface string, _ net.HardwareAddr, src, gw netip.Addr, _ time.Duration) error {
+		mu.Lock()
+		defer mu.Unlock()
+		probed = append(probed, fmt.Sprintf("%s %s>%s", iface, src, gw))
+		return nil
+	}
+
+	reports := a.probeNICs()
+	if len(reports) != 1 || reports[0].MAC != testDataMAC || !reports[0].ProbeOK || reports[0].ConsecutiveFailures != 0 {
+		t.Fatalf("reports = %+v", reports)
+	}
+	if len(probed) != 1 || probed[0] != "ens3 10.0.1.5>10.0.1.1" {
+		t.Errorf("probed = %v, want only the data NIC", probed)
+	}
+	if reports[0].RxPackets != 100 || reports[0].TxDropped != 4 {
+		t.Errorf("counters missing from report: %+v", reports[0])
+	}
+}
+
+func TestProbeNICs_TwoMissesThenSuccessNeverReportsFailing(t *testing.T) {
+	a := datapathAgent(t, "http://unused", testNetcfg)
+	results := []error{errARPNoReply, errARPNoReply, nil, errARPNoReply, errARPNoReply, nil}
+	var n atomic.Int32
+	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error {
+		return results[n.Add(1)-1]
+	}
+
+	for i, want := range []int64{1, 2, 0, 1, 2, 0} {
+		r := a.probeNICs()
+		if len(r) != 1 || !r[0].ProbeOK || r[0].ConsecutiveFailures != want {
+			t.Fatalf("tick %d: report = %+v, want ok with %d failures", i, r, want)
+		}
+	}
+}
+
+func TestProbeNICs_ThreeMissesReportFailingAndRecoveryClears(t *testing.T) {
+	a := datapathAgent(t, "http://unused", testNetcfg)
+	var fail atomic.Bool
+	fail.Store(true)
+	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error {
+		if fail.Load() {
+			return errARPNoReply
+		}
+		return nil
+	}
+
+	for i := 1; i <= 4; i++ {
+		r := a.probeNICs()
+		wantOK := i < 3 // failing from the third consecutive miss
+		if r[0].ProbeOK != wantOK || r[0].ConsecutiveFailures != int64(i) {
+			t.Fatalf("miss %d: report = %+v, want ProbeOK=%v", i, r[0], wantOK)
+		}
+	}
+
+	fail.Store(false)
+	if r := a.probeNICs(); !r[0].ProbeOK || r[0].ConsecutiveFailures != 0 {
+		t.Errorf("a successful probe must clear the failure: %+v", r[0])
+	}
+}
+
+func TestProbeNICs_BlockedProbeIsBoundedAndCountsAsMiss(t *testing.T) {
+	a := datapathAgent(t, "http://unused", testNetcfg)
+	a.probeWait = 50 * time.Millisecond
+	release := make(chan struct{})
+	defer close(release)
+	var calls atomic.Int32
+	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error {
+		calls.Add(1)
+		<-release
+		return nil
+	}
+
+	for i := 1; i <= 3; i++ {
+		start := time.Now()
+		r := a.probeNICs()
+		if d := time.Since(start); d > time.Second {
+			t.Fatalf("tick %d waited %s on a stuck probe", i, d)
+		}
+		if len(r) != 1 || r[0].ConsecutiveFailures != int64(i) {
+			t.Fatalf("tick %d: report = %+v, want %d misses", i, r, i)
+		}
+	}
+	if got := calls.Load(); got != 1 {
+		t.Errorf("a stuck probe was started %d times, want 1 (no goroutine pile-up)", got)
+	}
+}
+
+func TestTick_BlockedProbeDoesNotDelayHeartbeatBeyondOneTick(t *testing.T) {
+	var got atomic.Pointer[url.Values]
+	gw := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_ = r.ParseForm()
+		if r.FormValue("Action") == "LBAgentHeartbeat" {
+			v := r.Form
+			got.Store(&v)
+		}
+		fmt.Fprint(w, `<LBAgentHeartbeatResponse><LBAgentHeartbeatResult><Status>active</Status><ConfigHash>h1</ConfigHash></LBAgentHeartbeatResult></LBAgentHeartbeatResponse>`)
+	}))
+	defer gw.Close()
+
+	a := datapathAgent(t, gw.URL, testNetcfg)
+	a.localConfigHash = "h1"
+	a.probeWait = 50 * time.Millisecond
+	release := make(chan struct{})
+	defer close(release)
+	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error {
+		<-release
+		return nil
+	}
+
+	start := time.Now()
+	a.tick()
+	if d := time.Since(start); d >= pollInterval {
+		t.Fatalf("tick took %s with a stuck probe, want under one %s tick", d, pollInterval)
+	}
+	if got.Load() == nil {
+		t.Fatal("the heartbeat was never sent")
+	}
+}
+
+func TestHeartbeat_EncodesEvidenceWhenPresent(t *testing.T) {
+	var got atomic.Pointer[url.Values]
+	gw := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_ = r.ParseForm()
+		v := r.Form
+		got.Store(&v)
+		fmt.Fprint(w, `<LBAgentHeartbeatResponse><LBAgentHeartbeatResult><Status>active</Status><ConfigHash>h1</ConfigHash></LBAgentHeartbeatResult></LBAgentHeartbeatResponse>`)
+	}))
+	defer gw.Close()
+
+	a := datapathAgent(t, gw.URL, testNetcfg)
+	a.localConfigHash = "h1"
+	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error { return errARPNoReply }
+
+	a.tick()
+	v := got.Load()
+	if v == nil {
+		t.Fatal("no heartbeat received")
+	}
+	want := map[string]string{
+		"InstanceID":                        "i-0123456789abcdef0",
+		"NICs.member.1.MAC":                 testDataMAC,
+		"NICs.member.1.ProbeOK":             "true",
+		"NICs.member.1.ConsecutiveFailures": "1",
+		"NICs.member.1.RxPackets":           "100",
+		"NICs.member.1.TxPackets":           "200",
+		"NICs.member.1.RxDropped":           "3",
+		"NICs.member.1.TxDropped":           "4",
+	}
+	for k, w := range want {
+		if g := v.Get(k); g != w {
+			t.Errorf("%s = %q, want %q", k, g, w)
+		}
+	}
+	if v.Get("NICs.member.2.MAC") != "" {
+		t.Error("the mgmt NIC must not be reported")
+	}
+}
+
+func TestHeartbeat_OmitsEvidenceWithoutNetcfg(t *testing.T) {
+	var got atomic.Pointer[url.Values]
+	gw := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_ = r.ParseForm()
+		v := r.Form
+		got.Store(&v)
+		fmt.Fprint(w, `<LBAgentHeartbeatResponse><LBAgentHeartbeatResult><Status>active</Status><ConfigHash>h1</ConfigHash></LBAgentHeartbeatResult></LBAgentHeartbeatResponse>`)
+	}))
+	defer gw.Close()
+
+	a := datapathAgent(t, gw.URL, testNetcfg)
+	a.localConfigHash = "h1"
+	a.netcfgPath = filepath.Join(t.TempDir(), "absent")
+	a.instanceIDFn = func() (string, error) { return "", errors.New("must not be asked without NICs") }
+
+	a.tick()
+	v := got.Load()
+	if v == nil {
+		t.Fatal("no heartbeat received")
+	}
+	for k := range *v {
+		if k == "InstanceID" || len(k) >= 4 && k[:4] == "NICs" {
+			t.Errorf("unexpected evidence parameter %s", k)
+		}
+	}
+}
+
+func TestDataPathEvidence_InstanceIDRetriedUntilKnown(t *testing.T) {
+	a := datapathAgent(t, "http://unused", testNetcfg)
+	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error { return nil }
+	var calls atomic.Int32
+	a.instanceIDFn = func() (string, error) {
+		if calls.Add(1) < 3 {
+			return "", errors.New("metadata not ready")
+		}
+		return "i-late", nil
+	}
+
+	// A failure backs the lookup off for imdsRetryTicks-1 ticks.
+	ticks := func(n int) string {
+		var id string
+		for range n {
+			id = a.dataPathEvidence().InstanceID
+		}
+		return id
+	}
+	if got := ticks(imdsRetryTicks); got != "" || calls.Load() != 1 {
+		t.Fatalf("after %d ticks: id %q, calls %d; want no id and one lookup", imdsRetryTicks, got, calls.Load())
+	}
+	if got := ticks(imdsRetryTicks); got != "" || calls.Load() != 2 {
+		t.Fatalf("second window: id %q, calls %d; want no id and two lookups", got, calls.Load())
+	}
+	if got := ticks(imdsRetryTicks); got != "i-late" || calls.Load() != 3 {
+		t.Fatalf("third window: id %q, calls %d; want i-late and three lookups", got, calls.Load())
+	}
+	ticks(imdsRetryTicks)
+	if calls.Load() != 3 {
+		t.Errorf("metadata asked %d times, want 3 (stop once known)", calls.Load())
+	}
+}
+
+func TestTick_BlockedInstanceIDLookupDoesNotDelayHeartbeat(t *testing.T) {
+	var got atomic.Pointer[url.Values]
+	gw := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_ = r.ParseForm()
+		if r.FormValue("Action") == "LBAgentHeartbeat" {
+			v := r.Form
+			got.Store(&v)
+		}
+		fmt.Fprint(w, `<LBAgentHeartbeatResponse><LBAgentHeartbeatResult><Status>active</Status><ConfigHash>h1</ConfigHash></LBAgentHeartbeatResult></LBAgentHeartbeatResponse>`)
+	}))
+	defer gw.Close()
+
+	a := datapathAgent(t, gw.URL, testNetcfg)
+	a.localConfigHash = "h1"
+	a.arpProbeFn = func(string, net.HardwareAddr, netip.Addr, netip.Addr, time.Duration) error { return nil }
+	release := make(chan struct{})
+	defer close(release)
+	var calls atomic.Int32
+	a.instanceIDFn = func() (string, error) {
+		calls.Add(1)
+		<-release
+		return "", errors.New("unreachable")
+	}
+
+	for i := range 3 {
+		start := time.Now()
+		a.tick()
+		if d := time.Since(start); d >= pollInterval {
+			t.Fatalf("tick %d took %s with IMDS blocked, want under one %s tick", i, d, pollInterval)
+		}
+	}
+	v := got.Load()
+	if v == nil {
+		t.Fatal("the heartbeat was never sent")
+	}
+	if v.Get("InstanceID") != "" {
+		t.Error("InstanceID must be omitted while the lookup fails")
+	}
+	if v.Get("NICs.member.1.MAC") == "" {
+		t.Error("NIC evidence must still be sent")
+	}
+	if calls.Load() != 1 {
+		t.Errorf("blocked lookup started %d times in 3 ticks, want 1 (backoff)", calls.Load())
+	}
+}
+
+func TestFetchInstanceID_SharesOneDeadline(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		time.Sleep(3 * imdsWait)
+	}))
+	defer srv.Close()
+	old := imdsBase
+	imdsBase = srv.URL + "/latest"
+	defer func() { imdsBase = old }()
+
+	start := time.Now()
+	if _, err := fetchInstanceID(); err == nil {
+		t.Fatal("a hung metadata service must fail the lookup")
+	}
+	if d := time.Since(start); d > imdsWait+imdsWait/2 {
+		t.Fatalf("lookup took %s, want about one %s deadline", d, imdsWait)
+	}
+}
+
+func TestARPProbe_NoResponderFailsClosed(t *testing.T) {
+	mac, _ := net.ParseMAC(testDataMAC)
+	// Nobody answers an ARP request sent on loopback, so a probe that returned
+	// nil here would mask a dead data path. It fails either on the socket (no
+	// CAP_NET_RAW) or on the missing reply.
+	err := arpProbe("lo", mac, netip.MustParseAddr("10.9.9.2"), netip.MustParseAddr("10.9.9.1"), 100*time.Millisecond)
+	if err == nil {
+		t.Fatal("a probe with no responder must fail")
+	}
+	if err := arpProbe("no-such-iface0", mac, netip.MustParseAddr("10.9.9.2"), netip.MustParseAddr("10.9.9.1"), 100*time.Millisecond); err == nil {
+		t.Fatal("a probe on a missing interface must fail")
+	}
+}
+
+func TestFetchInstanceID(t *testing.T) {
+	serve := func(tokenStatus, idStatus int, wantToken string) *httptest.Server {
+		return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			switch {
+			case r.Method == http.MethodPut && r.URL.Path == "/latest/api/token":
+				w.WriteHeader(tokenStatus)
+				fmt.Fprint(w, "tok123")
+			case r.URL.Path == "/latest/meta-data/instance-id":
+				if got := r.Header.Get("X-Aws-Ec2-Metadata-Token"); got != wantToken {
+					t.Errorf("token header = %q, want %q", got, wantToken)
+				}
+				w.WriteHeader(idStatus)
+				fmt.Fprint(w, " i-abc123 \n")
+			default:
+				http.NotFound(w, r)
+			}
+		}))
+	}
+	saved := imdsBase
+	defer func() { imdsBase = saved }()
+
+	for _, tc := range []struct {
+		name                  string
+		tokenStatus, idStatus int
+		wantToken             string
+		wantID                string
+		wantErr               bool
+	}{
+		{"token then id", http.StatusOK, http.StatusOK, "tok123", "i-abc123", false},
+		{"token refused falls back to v1", http.StatusNotFound, http.StatusOK, "", "i-abc123", false},
+		{"id endpoint error", http.StatusOK, http.StatusServiceUnavailable, "tok123", "", true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			srv := serve(tc.tokenStatus, tc.idStatus, tc.wantToken)
+			defer srv.Close()
+			imdsBase = srv.URL + "/latest"
+			id, err := fetchInstanceID()
+			if (err != nil) != tc.wantErr || id != tc.wantID {
+				t.Fatalf("id = %q, err = %v", id, err)
+			}
+		})
+	}
+
+	imdsBase = "http://127.0.0.1:1/latest"
+	if _, err := fetchInstanceID(); err == nil {
+		t.Error("an unreachable metadata service must fail")
+	}
+}

spinifex/agents/elbv2/evidence.go

@@ -0,0 +1,133 @@
+package elbv2
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	imdsWait       = time.Second
+	imdsRetryTicks = 6
+)
+
+// imdsBase is the in-VM metadata endpoint (a variable so tests can point it at a fake).
+var imdsBase = "http://169.254.169.254/latest"
+
+// dataPathEvidence is the optional part of a heartbeat: which VM the agent runs
+// in and what each data NIC's gateway probe saw. An empty value adds nothing,
+// which is what an agent without a netcfg blob sends.
+type dataPathEvidence struct {
+	InstanceID string
+	NICs       []NICReport
+}
+
+// addTo encodes the evidence as additional heartbeat query parameters.
+func (e dataPathEvidence) addTo(params url.Values) {
+	if e.InstanceID != "" {
+		params.Set("InstanceID", e.InstanceID)
+	}
+	for i, n := range e.NICs {
+		p := "NICs.member." + strconv.Itoa(i+1) + "."
+		params.Set(p+"MAC", n.MAC)
+		params.Set(p+"ProbeOK", strconv.FormatBool(n.ProbeOK))
+		params.Set(p+"ConsecutiveFailures", strconv.FormatInt(n.ConsecutiveFailures, 10))
+		params.Set(p+"RxPackets", strconv.FormatInt(n.RxPackets, 10))
+		params.Set(p+"TxPackets", strconv.FormatInt(n.TxPackets, 10))
+		params.Set(p+"RxDropped", strconv.FormatInt(n.RxDropped, 10))
+		params.Set(p+"TxDropped", strconv.FormatInt(n.TxDropped, 10))
+	}
+}
+
+// dataPathEvidence gathers the probe results and instance ID for one heartbeat.
+// Neither step can fail the heartbeat: missing evidence is simply omitted.
+func (a *Agent) dataPathEvidence() dataPathEvidence {
+	nics := a.probeNICs()
+	if len(nics) == 0 {
+		return dataPathEvidence{}
+	}
+	if a.instanceID == "" {
+		a.lookupInstanceID()
+	}
+	return dataPathEvidence{InstanceID: a.instanceID, NICs: nics}
+}
+
+// lookupInstanceID asks for the instance ID without ever holding up a tick: the
+// call is bounded by imdsWait and, after a failure, retried only every
+// imdsRetryTicks ticks.
+func (a *Agent) lookupInstanceID() {
+	if a.idSkip > 0 {
+		a.idSkip--
+		return
+	}
+	type result struct {
+		id  string
+		err error
+	}
+	ch := make(chan result, 1)
+	go func() {
+		id, err := a.instanceIDFn()
+		ch <- result{id, err}
+	}()
+	timer := time.NewTimer(imdsWait)
+	defer timer.Stop()
+	select {
+	case r := <-ch:
+		if r.err == nil && r.id != "" {
+			a.instanceID = r.id
+			return
+		}
+	case <-timer.C:
+	}
+	a.idSkip = imdsRetryTicks - 1
+}
+
+// fetchInstanceID reads the instance ID from the in-VM metadata service. The
+// token request and the read share one deadline.
+func fetchInstanceID() (string, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), imdsWait)
+	defer cancel()
+	c := &http.Client{}
+
+	var token string
+	if req, err := http.NewRequestWithContext(ctx, http.MethodPut, imdsBase+"/api/token", nil); err == nil {
+		req.Header.Set("X-Aws-Ec2-Metadata-Token-Ttl-Seconds", "60")
+		if resp, err := c.Do(req); err == nil {
+			b, _ := io.ReadAll(io.LimitReader(resp.Body, 256))
+			resp.Body.Close()
+			if resp.StatusCode == http.StatusOK {
+				token = strings.TrimSpace(string(b))
+			}
+		}
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, imdsBase+"/meta-data/instance-id", nil)
+	if err != nil {
+		return "", err
+	}
+	if token != "" {
+		req.Header.Set("X-Aws-Ec2-Metadata-Token", token)
+	}
+	resp, err := c.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("instance-id: status %d", resp.StatusCode)
+	}
+	b, err := io.ReadAll(io.LimitReader(resp.Body, 256))
+	if err != nil {
+		return "", err
+	}
+	id := strings.TrimSpace(string(b))
+	if id == "" {
+		return "", fmt.Errorf("instance-id: empty")
+	}
+	return id, nil
+}

spinifex/agents/lambda/hostlink/configdisk.go

@@ -0,0 +1,71 @@
+package hostlink
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+
+	lambdav1 "github.com/mulgadc/spinifex/contracts/lambda/v1"
+	"golang.org/x/sys/unix"
+)
+
+// On a VMM without fw_cfg the executor attaches the same entries as a tar
+// archive on a second disk, which the runtime image's init extracts here in
+// the fw_cfg sysfs layout, so the fw_cfg readers work unchanged.
+const (
+	ConfigDiskRoot   = "/spinifex-cfg"
+	ConfigDiskDir    = ConfigDiskRoot + "/by_name/" + lambdav1.FwCfgPrefix
+	ConfigDiskDevice = "/dev/vdb"
+)
+
+// zeroChunk bounds each write while erasing the configuration disk.
+const zeroChunk = 64 << 10
+
+// unmount is a var so tests without privilege can stand in for the tmpfs.
+var unmount = unix.Unmount
+
+// SealConfigDisk overwrites the whole configuration disk with zeros, then
+// removes the extracted tree. The tree stays if the disk cannot be erased, so
+// SealFwCfg's check on root fails closed.
+func SealConfigDisk(device, root string) error {
+	if err := zeroDevice(device); err != nil {
+		return fmt.Errorf("hostlink: erase configuration disk: %w", err)
+	}
+	if err := unmount(root, 0); err != nil && !errors.Is(err, unix.EINVAL) {
+		return fmt.Errorf("hostlink: unmount %s: %w", root, err)
+	}
+	if err := os.RemoveAll(root); err != nil {
+		return fmt.Errorf("hostlink: remove %s: %w", root, err)
+	}
+	return nil
+}
+
+func zeroDevice(device string) error {
+	f, err := os.OpenFile(device, os.O_WRONLY, 0)
+	if err != nil {
+		return err
+	}
+	size, err := f.Seek(0, io.SeekEnd)
+	if err == nil {
+		_, err = f.Seek(0, io.SeekStart)
+	}
+	if err != nil {
+		_ = f.Close()
+		return err
+	}
+	zeros := make([]byte, zeroChunk)
+	for left := size; left > 0; {
+		n := min(left, int64(len(zeros)))
+		if _, err := f.Write(zeros[:n]); err != nil {
+			_ = f.Close()
+			return err
+		}
+		left -= n
+	}
+	if err := f.Sync(); err != nil {
+		_ = f.Close()
+		return err
+	}
+	return f.Close()
+}

spinifex/agents/lambda/hostlink/configdisk_test.go

@@ -0,0 +1,63 @@
+//test:in-package reaches the unexported configDisk and fwCfgTree readers to seal and read a real archive
+
+package hostlink
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/unix"
+)
+
+// configDisk writes a fake disk holding secret and an extracted tree; the
+// stand-in unmount reports the tree is not a mount, as on a plain directory.
+func configDisk(t *testing.T, secret []byte) (device, root string) {
+	t.Helper()
+	dir := t.TempDir()
+	device = filepath.Join(dir, "vdb")
+	require.NoError(t, os.WriteFile(device, append(bytes.Repeat([]byte{0xa5}, 3*zeroChunk+7), secret...), 0o600))
+	root = fwCfgTree(t)
+	require.NoError(t, os.WriteFile(filepath.Join(root, "by_name", "key"), secret, 0o600))
+	old := unmount
+	unmount = func(string, int) error { return unix.EINVAL }
+	t.Cleanup(func() { unmount = old })
+	return device, root
+}
+
+func TestSealConfigDisk(t *testing.T) {
+	secret := []byte("guest-key-material")
+
+	t.Run("erases the disk and removes the tree", func(t *testing.T) {
+		device, root := configDisk(t, secret)
+		before, err := os.Stat(device)
+		require.NoError(t, err)
+		require.NoError(t, SealFwCfg(root, func() error { return SealConfigDisk(device, root) }))
+
+		disk, err := os.ReadFile(device)
+		require.NoError(t, err)
+		assert.Len(t, disk, int(before.Size()))
+		assert.Equal(t, make([]byte, len(disk)), disk, "configuration disk not fully erased")
+		_, err = os.Stat(root)
+		assert.ErrorIs(t, err, os.ErrNotExist)
+	})
+	t.Run("an unwritable disk keeps the tree and fails closed", func(t *testing.T) {
+		device, root := configDisk(t, secret)
+		require.NoError(t, os.Chmod(device, 0o400))
+		err := SealFwCfg(root, func() error { return SealConfigDisk(device, root) })
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "still readable")
+		_, err = os.Stat(root)
+		assert.NoError(t, err)
+	})
+	t.Run("a failed unmount keeps the tree", func(t *testing.T) {
+		device, root := configDisk(t, secret)
+		unmount = func(string, int) error { return unix.EBUSY }
+		require.Error(t, SealFwCfg(root, func() error { return SealConfigDisk(device, root) }))
+		_, err := os.Stat(root)
+		assert.NoError(t, err)
+	})
+}

spinifex/agents/lambda/hostlink/link.go

@@ -28,11 +28,12 @@ type Request struct {
 	Credential *Credential `json:"credential,omitempty"`
 }
 
-// Credential is one execution-role session. It is never logged or stored.
+// Credential holds one execution-role session and its function environment.
 type Credential struct {
-	AccessKeyID     string `json:"access_key_id"`
-	SecretAccessKey string `json:"secret_access_key"`
-	SessionToken    string `json:"session_token"`
+	AccessKeyID     string            `json:"access_key_id"`
+	SecretAccessKey string            `json:"secret_access_key"`
+	SessionToken    string            `json:"session_token"`
+	Environment     map[string]string `json:"environment,omitempty"`
 }
 
 // LogValue keeps every value out of logs.

spinifex/agents/lambda/hostlink/protocol_testlink.go

@@ -4,5 +4,5 @@ package hostlink
 
 // Protocol is the only ALPN protocol a test-link host and guest offer. It
 // differs from the release protocol because the first frame carries the
-// execution credential, which an older guest would ignore.
-const Protocol = "spinifex-lambda-testlink/2"
+// execution credential and function environment, which an older guest would ignore.
+const Protocol = "spinifex-lambda-testlink/3"

spinifex/domains/acm/store.go

@@ -14,7 +14,6 @@ import (
 	iamdomain "github.com/mulgadc/spinifex/spinifex/domains/iam"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/arn"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
 )
@@ -184,7 +183,7 @@ func NewStore(ctx context.Context, nc *nats.Conn, masterKey []byte) (*Store, err
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketACM, KVBucketACMVersion)
+	kv, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketACM, KVBucketACMVersion)
 	if err != nil {
 		return nil, err
 	}

spinifex/domains/admission/quota/bedrock.go

@@ -10,7 +10,7 @@ import (
 	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 	"golang.org/x/time/rate"
 )
@@ -171,7 +171,7 @@ func (s *Service) RunBedrockRPMSync(ctx context.Context, js jetstream.JetStream)
 	if s == nil || !s.limits.RequestsPerMinuteEnabled {
 		return
 	}
-	kv, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketBedrockRPMUsage, 1)
+	kv, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketBedrockRPMUsage, 1)
 	if err != nil {
 		slog.Warn("bedrock rpm sync: open usage bucket failed, sync disabled", "err", err)
 		return

spinifex/domains/admission/quota/override.go

@@ -9,7 +9,7 @@ import (
 	"time"
 
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -187,7 +187,7 @@ func (s *Service) PutAccountQuota(ctx context.Context, accountID string, over Ov
 	over.UpdatedAt = time.Now().UTC().Format(time.RFC3339)
 	over.UpdatedBy = updatedBy
 
-	_, err := kvutil.Update(ctx, s.overrides, accountID, kvutil.CASConfig{
+	_, err := kvstore.Update(ctx, s.overrides, accountID, kvstore.CASConfig{
 		CreateIfAbsent: true,
 		Attempts:       overrideMaxAttempts,
 		Exhausted:      func(string, int) error { return errOverridesContended },
@@ -230,7 +230,7 @@ func (s *Service) clearAccountQuota(ctx context.Context, accountID string) error
 			return nil
 		case errors.Is(err, jetstream.ErrKeyNotFound):
 			return nil
-		case kvutil.IsRevisionConflict(err):
+		case kvstore.IsRevisionConflict(err):
 			// A concurrent write landed between the read and the guarded
 			// delete; retry against whatever revision now holds.
 			continue

spinifex/domains/admission/quota/quota.go

@@ -22,10 +22,10 @@ const ReconcileInterval = 30 * time.Second
 // dimension; the live-counted dimensions need no stored state.
 const KVBucketAccountUsage = "spinifex-account-usage"
 
-// KVBucketQuotaReconcile is the leader-lock bucket for the gateway quota
-// reconcile. It is deliberately distinct from vpcd's network-reconcile lock so
-// the two independent loops never share one mutex and block each other.
-const KVBucketQuotaReconcile = "spinifex-quota-reconcile"
+// KeyReconcile is the sync-bucket leader lease for the gateway quota reconcile.
+// It is deliberately distinct from vpcd's network-reconcile lease so the two
+// independent loops never share one mutex and block each other.
+const KeyReconcile = "quota.reconcile"
 
 // KVBucketAccountQuota holds the per-account limit overrides, keyed by
 // accountID. It is separate from the accounts bucket so a quota write never

spinifex/domains/admission/quota/seal.go

@@ -8,7 +8,7 @@ import (
 
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -73,7 +73,7 @@ func (s *Service) sealBucket(ctx context.Context) (jetstream.KeyValue, error) {
 		slog.ErrorContext(ctx, "Quota override seal has no JetStream client wired to open the override bucket")
 		return nil, errors.New(awserrors.ErrorInternalError)
 	}
-	kv, err := kvutil.GetOrCreateBucket(ctx, s.js, KVBucketAccountQuota, 1)
+	kv, err := kvstore.GetOrCreateBucket(ctx, s.js, KVBucketAccountQuota, 1)
 	if err != nil {
 		slog.ErrorContext(ctx, "Failed to open quota override bucket for seal", "err", err)
 		return nil, errors.New(awserrors.ErrorInternalError)
@@ -136,7 +136,7 @@ func (s *Service) SealAccountOverrides(ctx context.Context, accountID string) (*
 		switch {
 		case writeErr == nil:
 			return &OverrideSeal{Revision: newRev}, nil
-		case kvutil.IsRevisionConflict(writeErr):
+		case kvstore.IsRevisionConflict(writeErr):
 			continue
 		default:
 			slog.ErrorContext(ctx, "Failed to write sealed quota override record", "accountID", accountID, "err", writeErr)

spinifex/domains/dns/delegation.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	nsconfig "github.com/mulgadc/northstar/pkg/config"
-	"github.com/nats-io/nats.go"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 
 	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
 )
@@ -48,7 +48,7 @@ func (w *Writer) PlatformDelegation(name string) (servers []string, zone string,
 // Delegate writes, or with remove withdraws, an NS set in the base zone that
 // delegates each name to this cluster's nameservers. It takes the zone-writer
 // election first, so it never races the reconcile's own writes to the zone.
-func (w *Writer) Delegate(ctx context.Context, nc *nats.Conn, holder string, names []string, remove bool) (*ChangeResult, error) {
+func (w *Writer) Delegate(ctx context.Context, lease *kvsync.Lease, names []string, remove bool) (*ChangeResult, error) {
 	if !w.Enabled() {
 		return nil, errors.New("northstar is not configured on this node")
 	}
@@ -77,7 +77,7 @@ func (w *Writer) Delegate(ctx context.Context, nc *nats.Conn, holder string, nam
 	if len(batch.Changes) == 0 {
 		return &ChangeResult{}, nil
 	}
-	release, err := w.awaitWriterElection(ctx, nc, holder)
+	release, err := w.awaitWriterElection(ctx, lease)
 	if err != nil {
 		return nil, err
 	}
@@ -85,19 +85,19 @@ func (w *Writer) Delegate(ctx context.Context, nc *nats.Conn, holder string, nam
 	return w.ApplyBatch(batch)
 }
 
-func (w *Writer) awaitWriterElection(ctx context.Context, nc *nats.Conn, holder string) (func(), error) {
-	if nc == nil {
+func (w *Writer) awaitWriterElection(ctx context.Context, lease *kvsync.Lease) (func(), error) {
+	if lease == nil {
 		return func() {}, nil
 	}
 	tick := time.NewTicker(delegateLockPoll)
 	defer tick.Stop()
 	for {
-		if release, ok := reconcilelock.AcquireLeader(ctx, nc, KVBucketDNSReconcile, holder); ok {
+		if release, ok := reconcilelock.AcquireLeader(ctx, lease); ok {
 			return release, nil
 		}
 		select {
 		case <-ctx.Done():
-			return nil, fmt.Errorf("another node is writing the zone: %w", ctx.Err())
+			return nil, fmt.Errorf("another writer has the zone: %w", ctx.Err())
 		case <-tick.C:
 		}
 	}

spinifex/domains/dns/delegation_test.go

@@ -14,7 +14,7 @@ import (
 
 func TestDelegateWritesAndWithdrawsPlatformDelegations(t *testing.T) {
 	w, _ := newMultiNodeTestWriter(t)
-	_, err := w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net", "B.r53e2e.spx3.net."}, false)
+	_, err := w.Delegate(t.Context(), nil, []string{"a.r53e2e.spx3.net", "B.r53e2e.spx3.net."}, false)
 	require.NoError(t, err)
 
 	for _, name := range []string{"a.r53e2e.spx3.net.", "b.r53e2e.spx3.net"} {
@@ -29,7 +29,7 @@ func TestDelegateWritesAndWithdrawsPlatformDelegations(t *testing.T) {
 	assert.True(t, ok, "a name below the base domain is decided by the base zone")
 	assert.Empty(t, servers)
 
-	_, err = w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net"}, true)
+	_, err = w.Delegate(t.Context(), nil, []string{"a.r53e2e.spx3.net"}, true)
 	require.NoError(t, err)
 	servers, _, _, err = w.PlatformDelegation("a.r53e2e.spx3.net")
 	require.NoError(t, err)
@@ -43,11 +43,11 @@ func TestDelegateWritesAndWithdrawsPlatformDelegations(t *testing.T) {
 // servers written while the cluster published a larger set.
 func TestDelegateRemoveWithdrawsServersFromAnEarlierSet(t *testing.T) {
 	w, _ := newMultiNodeTestWriter(t)
-	_, err := w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net"}, false)
+	_, err := w.Delegate(t.Context(), nil, []string{"a.r53e2e.spx3.net"}, false)
 	require.NoError(t, err)
 
 	w.nameservers = w.nameservers[:1]
-	_, err = w.Delegate(t.Context(), nil, "node1", []string{"a.r53e2e.spx3.net"}, true)
+	_, err = w.Delegate(t.Context(), nil, []string{"a.r53e2e.spx3.net"}, true)
 	require.NoError(t, err)
 	servers, _, _, err := w.PlatformDelegation("a.r53e2e.spx3.net")
 	require.NoError(t, err)
@@ -66,7 +66,7 @@ func TestPlatformDelegationIgnoresNamesOutsideTheBaseDomain(t *testing.T) {
 func TestDelegateRefusesPlatformAndForeignNames(t *testing.T) {
 	w, _ := newMultiNodeTestWriter(t)
 	for _, name := range []string{"ns1.spx3.net", "x.elb.spx3.net", "compute.spx3.net", "example.com", "spx3.net"} {
-		_, err := w.Delegate(t.Context(), nil, "node1", []string{"ok.spx3.net", name}, false)
+		_, err := w.Delegate(t.Context(), nil, []string{"ok.spx3.net", name}, false)
 		require.Error(t, err, name)
 	}
 	servers, _, _, err := w.PlatformDelegation("ok.spx3.net")
@@ -76,22 +76,45 @@ func TestDelegateRefusesPlatformAndForeignNames(t *testing.T) {
 
 func TestDelegateWaitsForTheZoneWriterElection(t *testing.T) {
 	w, _ := newMultiNodeTestWriter(t)
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	prev := delegateLockPoll
 	delegateLockPoll = 10 * time.Millisecond
 	t.Cleanup(func() { delegateLockPoll = prev })
 
-	release, ok := reconcilelock.AcquireLeader(t.Context(), nc, KVBucketDNSReconcile, "node2")
+	release, ok := reconcilelock.AcquireLeader(t.Context(), dnsLease(js, "node2"))
 	require.True(t, ok)
 	ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
 	defer cancel()
-	_, err := w.Delegate(ctx, nc, "node1", []string{"a.spx3.net"}, false)
-	require.ErrorContains(t, err, "another node is writing the zone")
+	node1 := dnsLease(js, "node1")
+	_, err := w.Delegate(ctx, node1, []string{"a.spx3.net"}, false)
+	require.ErrorContains(t, err, "another writer has the zone")
 
 	release()
-	_, err = w.Delegate(t.Context(), nc, "node1", []string{"a.spx3.net"}, false)
+	_, err = w.Delegate(t.Context(), node1, []string{"a.spx3.net"}, false)
 	require.NoError(t, err)
 	servers, _, _, err := w.PlatformDelegation("a.spx3.net")
 	require.NoError(t, err)
 	assert.Len(t, servers, 2)
 }
+
+// The reconcile pass and a delegation on one node share the DNS lease, so a
+// delegation waits for this node's own pass as it would for another node's.
+func TestDelegateWaitsForThisNodesOwnReconcilePass(t *testing.T) {
+	w, _ := newMultiNodeTestWriter(t)
+	_, _, js := testutil.StartTestJetStream(t)
+	prev := delegateLockPoll
+	delegateLockPoll = 10 * time.Millisecond
+	t.Cleanup(func() { delegateLockPoll = prev })
+
+	shared := dnsLease(js, "node1")
+	release, ok := reconcilelock.AcquireLeader(t.Context(), shared)
+	require.True(t, ok, "the reconcile pass holds the lease")
+	ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
+	defer cancel()
+	_, err := w.Delegate(ctx, shared, []string{"a.spx3.net"}, false)
+	require.ErrorContains(t, err, "another writer has the zone")
+
+	release()
+	_, err = w.Delegate(t.Context(), shared, []string{"a.spx3.net"}, false)
+	require.NoError(t, err)
+}

spinifex/domains/dns/reconcile.go

@@ -11,7 +11,7 @@ import (
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/reconciler"
-	"github.com/nats-io/nats.go"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
 )
 
 // DefaultReconcileInterval is how often the drift backstop rebuilds managed
@@ -33,6 +33,12 @@ const DefaultReconcileInterval = reconciler.DefaultResync
 // costs a cycle for nothing.
 const DefaultComputeTimeout = 2 * time.Minute
 
+// NewReconcileLease returns the zone writer's election lease, shared by the
+// drift reconcile and Delegate so the two never write the zone together.
+func NewReconcileLease(store kvsync.KVStore, holder string) *kvsync.Lease {
+	return reconcilelock.NewLeaderLease(store, KeyReconcile, holder)
+}
+
 // DesiredFunc returns the full desired managed record set built from the live
 // resource inventory across all tenants. The daemon supplies it by enumerating
 // instances, load balancers, and EKS clusters.
@@ -102,7 +108,8 @@ type Reconciler struct {
 	// lands in. Distinct from both baseDomain and internalDomain, so pruning it
 	// needs its own read the same way internalDomain does for EC2.
 	serviceZone string
-	nc          *nats.Conn
+	// lease elects the one node that writes the zones; nil skips the election.
+	lease *kvsync.Lease
 	// writer applies the converged batch. The reconcile owns the zone object on
 	// the node that won the election, so the batch never leaves this process.
 	writer   *Writer
@@ -111,7 +118,6 @@ type Reconciler struct {
 	// computeTimeout bounds how long one pass spends building the desired set.
 	// A field rather than a constant so a test can shrink it.
 	computeTimeout time.Duration
-	holder         string
 	sources        []reconciler.Source
 }
 
@@ -121,18 +127,15 @@ type Reconciler struct {
 // changes should wake the loop. They are optional: with none supplied the loop
 // falls back to the interval alone, which is the behaviour that predates the
 // watch.
-func NewReconciler(cfg *config.Config, cluster *config.ClusterConfig, nc *nats.Conn, writer *Writer, desired DesiredFunc, sources ...reconciler.Source) *Reconciler {
+func NewReconciler(cfg *config.Config, cluster *config.ClusterConfig, lease *kvsync.Lease, writer *Writer, desired DesiredFunc, sources ...reconciler.Source) *Reconciler {
 	r := &Reconciler{
-		nc:             nc,
+		lease:          lease,
 		writer:         writer,
 		desired:        desired,
 		interval:       DefaultReconcileInterval,
 		computeTimeout: DefaultComputeTimeout,
 		sources:        sources,
 	}
-	if cfg != nil {
-		r.holder = cfg.Node
-	}
 	zoneCfg, ok := zoneS3Config(cfg)
 	if !ok || desired == nil || !writer.Enabled() {
 		return r
@@ -166,25 +169,24 @@ func (r *Reconciler) Run(ctx context.Context) {
 		// No revisit deadline: the desired set is a function of the buckets
 		// being watched, so a change is the only reason to run early.
 		Reconcile: func(ctx context.Context) (time.Duration, error) {
-			r.reconcileOnce(ctx)
-			return 0, nil
+			return r.reconcileOnce(ctx), nil
 		},
 		Resync: r.interval,
 	})
 }
 
 // reconcileOnce computes the converging batch and applies it, on whichever node
-// wins this cycle's election. The read that finds the stale records and the
+// wins this cycle's election, and returns the election's revisit when it lost. The read that finds the stale records and the
 // write that removes them now happen in one process, so a record re-added
 // between them is no longer deleted by a batch that predates it.
-func (r *Reconciler) reconcileOnce(ctx context.Context) {
+func (r *Reconciler) reconcileOnce(ctx context.Context) time.Duration {
 	if !r.enabled {
-		return
+		return 0
 	}
-	if r.nc != nil {
-		release, elected := reconcilelock.AcquireLeader(ctx, r.nc, KVBucketDNSReconcile, r.holder)
+	if r.lease != nil {
+		release, retry, elected := reconcilelock.ElectPass(ctx, r.lease)
 		if !elected {
-			return
+			return retry
 		}
 		defer release()
 	}
@@ -193,18 +195,19 @@ func (r *Reconciler) reconcileOnce(ctx context.Context) {
 		// A corrupt zone is handled in readZone; anything reaching here left the
 		// desired state unapplied, so surface it rather than burying it in WARN.
 		slog.Error("dns reconcile: compute batch failed, retrying next cycle", "error", err)
-		return
+		return 0
 	}
 	if len(batch) == 0 {
-		return
+		return 0
 	}
 	slog.Debug("dns reconcile: converging", "changes", len(batch))
 	res, err := r.writer.ApplyBatch(&ChangeBatch{Changes: batch})
 	if err != nil {
 		slog.Warn("dns reconcile: apply failed, retrying next cycle", "changes", len(batch), "error", err)
-		return
+		return 0
 	}
 	slog.Debug("dns reconcile: converged", "changes", res.Applied, "zones", res.Zones)
+	return 0
 }
 
 // computeBatchBounded runs computeBatch with a deadline, so a pass that never

spinifex/domains/dns/reconcile_election_test.go

@@ -8,41 +8,50 @@ import (
 
 	"github.com/mulgadc/spinifex/internal/testkit"
 	reconcilelock "github.com/mulgadc/spinifex/spinifex/domains/network/reconcile"
-	"github.com/nats-io/nats.go"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvsync"
+	"github.com/nats-io/nats.go/jetstream"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-// electingReconcilerFor is reconcilerFor with a live connection, so a pass runs
-// the election instead of skipping it.
-func electingReconcilerFor(w *Writer, nc *nats.Conn, holder string, desired func() DesiredSet) *Reconciler {
+// electingReconcilerFor is reconcilerFor with a lease, so a pass runs the
+// election instead of skipping it.
+func electingReconcilerFor(w *Writer, lease *kvsync.Lease, desired func() DesiredSet) *Reconciler {
 	r := reconcilerFor(w, desired)
-	r.nc = nc
-	r.holder = holder
+	r.lease = lease
 	return r
 }
 
+// dnsLease returns holder's DNS lease on a store of its own, standing in for
+// a separate process.
+func dnsLease(js jetstream.JetStream, holder string) *kvsync.Lease {
+	return NewReconcileLease(kvstore.OpenSyncBucket(js), holder)
+}
+
 // The zone lock is gone, and what replaces it is that a zone has one writer:
 // the node holding the reconcile lease. A node that loses the election must
 // therefore not touch the object at all — if it did, two nodes could be
 // read-modify-writing the same zone with nothing serialising them.
 func TestReconcileOnce_ALoserOfTheElectionDoesNotWriteTheZone(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	w, objects := newTestWriter(t)
 	before := objects[testBase+".toml"]
 	require.NotEmpty(t, before)
 
 	// A peer holds the lease for the whole pass, so this node cannot win it.
-	release, elected := reconcilelock.AcquireLeader(t.Context(), nc, KVBucketDNSReconcile, "peer")
+	release, elected := reconcilelock.AcquireLeader(t.Context(), dnsLease(js, "peer"))
 	require.True(t, elected, "the peer has to hold the lease or this test proves nothing")
 
-	r := electingReconcilerFor(w, nc, "node1", func() DesiredSet {
+	r := electingReconcilerFor(w, dnsLease(js, "node1"), func() DesiredSet {
 		return DesiredSet{Changes: []Change{upsert("lb-1.elb."+testBase, "10.200.1.9")}}
 	})
-	r.reconcileOnce(t.Context())
+	revisit := r.reconcileOnce(t.Context())
 
 	assert.Equal(t, before, objects[testBase+".toml"],
 		"a node that did not win the election must leave the zone object exactly as it found it")
+	assert.Positive(t, revisit, "the loser asks to retry so a dead holder is taken over")
+	assert.LessOrEqual(t, revisit, reconcilelock.LeaderTTL/3, "the retry comes within a renewal interval")
 
 	release()
 }
@@ -50,10 +59,10 @@ func TestReconcileOnce_ALoserOfTheElectionDoesNotWriteTheZone(t *testing.T) {
 // The counterpart, so the test above is not passing because the pass was broken
 // for some other reason: with the lease free, the same reconciler writes.
 func TestReconcileOnce_TheWinnerOfTheElectionWritesTheZone(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	w, objects := newTestWriter(t)
 
-	r := electingReconcilerFor(w, nc, "node1", func() DesiredSet {
+	r := electingReconcilerFor(w, dnsLease(js, "node1"), func() DesiredSet {
 		return DesiredSet{Changes: []Change{upsert("lb-1.elb."+testBase, "10.200.1.9")}}
 	})
 	r.reconcileOnce(t.Context())

spinifex/domains/dns/reconcile_test.go

@@ -224,7 +224,7 @@ func TestReconciler_RunReturnsWhenDisabled(t *testing.T) {
 // path of Run: the startup pass must fire before any watch or resync activity,
 // and cancelling ctx must still let the loop return.
 func TestReconciler_RunPerformsStartupPassThenStopsOnCancel(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	endpoint, _ := fakeS3(t, "northstar")
 
 	var passes atomic.Int64
@@ -235,8 +235,7 @@ func TestReconciler_RunPerformsStartupPassThenStopsOnCancel(t *testing.T) {
 			Endpoint: endpoint, Bucket: "northstar", Region: "us-east-1",
 			AccessKey: "SYSTEM", SecretKey: "SYSTEMSECRET",
 		},
-		nc:     nc,
-		holder: "test-node",
+		lease: dnsLease(js, "test-node"),
 		desired: func() DesiredSet {
 			passes.Add(1)
 			return DesiredSet{}
@@ -268,7 +267,7 @@ func TestReconciler_RunPerformsStartupPassThenStopsOnCancel(t *testing.T) {
 // acquirable afterwards, not merely on the call returning, since it is the
 // leadership and not the pass that the rest of the cluster waits on.
 func TestReconcileOnce_AStuckPassSurrendersLeadership(t *testing.T) {
-	_, nc, _ := testutil.StartTestJetStream(t)
+	_, _, js := testutil.StartTestJetStream(t)
 	endpoint, _ := fakeS3(t, "northstar")
 
 	blocked := make(chan struct{})
@@ -281,8 +280,7 @@ func TestReconcileOnce_AStuckPassSurrendersLeadership(t *testing.T) {
 			Endpoint: endpoint, Bucket: "northstar", Region: "us-east-1",
 			AccessKey: "SYSTEM", SecretKey: "SYSTEMSECRET",
 		},
-		nc:             nc,
-		holder:         "stuck-node",
+		lease:          dnsLease(js, "stuck-node"),
 		computeTimeout: 100 * time.Millisecond,
 		desired: func() DesiredSet {
 			<-blocked
@@ -302,7 +300,7 @@ func TestReconcileOnce_AStuckPassSurrendersLeadership(t *testing.T) {
 		t.Fatal("reconcileOnce never returned: a stuck pass still holds the lease forever")
 	}
 
-	release, elected := reconcilelock.AcquireLeader(t.Context(), nc, KVBucketDNSReconcile, "next-node")
+	release, elected := reconcilelock.AcquireLeader(t.Context(), dnsLease(js, "next-node"))
 	require.True(t, elected, "another node must be able to take the next cycle")
 	release()
 }

spinifex/domains/dns/types.go

@@ -11,9 +11,9 @@ const (
 	// PUT so every northstar instance reloads just that zone immediately, instead
 	// of waiting for the next S3 sync poll. No queue group: all servers consume.
 	SubjectZoneReload = "dns.zone.reload"
-	// KVBucketDNSReconcile elects the one node that writes the zones for a drift
-	// cycle. Its own bucket, so the election never blocks another reconcile loop.
-	KVBucketDNSReconcile = "spinifex-dns-reconcile"
+	// KeyReconcile elects the one node that writes the zones for a drift cycle.
+	// Its own key, so the election never blocks another reconcile loop.
+	KeyReconcile = "dns.reconcile"
 	// PrivateZone is the fixed AWS-parity private DNS zone (IMDS synthHostname).
 	PrivateZone = "compute.internal"
 	// DefaultTTL is applied when a change omits a TTL.

spinifex/domains/ec2/account/seal.go

@@ -8,7 +8,7 @@ import (
 
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -109,7 +109,7 @@ func (s *AccountSettingsServiceImpl) SealAccount(ctx context.Context, accountID
 		switch {
 		case writeErr == nil:
 			return &AccountSettingsSeal{Revision: newRev}, nil
-		case kvutil.IsRevisionConflict(writeErr):
+		case kvstore.IsRevisionConflict(writeErr):
 			continue
 		default:
 			slog.ErrorContext(ctx, "Failed to write sealed account settings record", "accountID", accountID, "err", writeErr)

spinifex/domains/ec2/account/service_impl.go

@@ -13,7 +13,7 @@ import (
 	ebsencryption "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/encryption"
 	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -57,7 +57,7 @@ func NewAccountSettingsServiceImplWithNATS(ctx context.Context, cfg *config.Conf
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	settingsKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketAccountSettings, 10)
+	settingsKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketAccountSettings, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create account settings KV bucket: %w", err)
 	}
@@ -115,13 +115,13 @@ func mapSettingsError(ctx context.Context, accountID string, err error) error {
 		return errors.New(awserrors.ErrorBlocked)
 	case errors.Is(err, errSettingsContended):
 		slog.ErrorContext(ctx, "Account settings CAS retries exhausted under contention", "accountID", accountID)
-	case errors.Is(err, kvutil.ErrRead):
+	case errors.Is(err, kvstore.ErrRead):
 		slog.ErrorContext(ctx, "Failed to read account settings from KV", "accountID", accountID, "err", err)
-	case errors.Is(err, kvutil.ErrDecode):
+	case errors.Is(err, kvstore.ErrDecode):
 		slog.ErrorContext(ctx, "Corrupt account settings record in KV", "accountID", accountID, "err", err)
-	case errors.Is(err, kvutil.ErrEncode):
+	case errors.Is(err, kvstore.ErrEncode):
 		slog.ErrorContext(ctx, "Failed to marshal account settings record", "accountID", accountID, "err", err)
-	case errors.Is(err, kvutil.ErrWrite):
+	case errors.Is(err, kvstore.ErrWrite):
 		slog.ErrorContext(ctx, "Failed to write account settings to KV", "accountID", accountID, "err", err)
 	default:
 		slog.ErrorContext(ctx, "Account settings operation failed", "accountID", accountID, "err", err)
@@ -133,7 +133,7 @@ func mapSettingsError(ctx context.Context, accountID string, err error) error {
 // changed is always true, so every success is a leader-accepted write over an
 // active value; a seal landing between the read and the write fails the CAS.
 func (s *AccountSettingsServiceImpl) mutateSettings(ctx context.Context, accountID string, set func(*AccountSettingsRecord)) error {
-	_, err := kvutil.Update(ctx, s.settingsKV, settingsKey(accountID), kvutil.CASConfig{
+	_, err := kvstore.Update(ctx, s.settingsKV, settingsKey(accountID), kvstore.CASConfig{
 		CreateIfAbsent: true,
 		Exhausted:      func(string, int) error { return errSettingsContended },
 	}, func(rec *AccountSettingsRecord) (bool, error) {

spinifex/domains/ec2/accountcleanup/remainder.go

@@ -0,0 +1,155 @@
+package accountcleanup
+
+import (
+	"context"
+	"log/slog"
+	"slices"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
+)
+
+// ReasonRecordRemains marks a resource whose record the owner still holds.
+const ReasonRecordRemains = "record-remains"
+
+// ComputeRemainder reads what compute the account still holds, changing nothing.
+type ComputeRemainder interface {
+	ComputeRemaining(ctx context.Context, accountID string) (*Outcome, error)
+}
+
+// StorageRemainder reads what volumes and tag residue the account still holds,
+// changing nothing.
+type StorageRemainder interface {
+	StorageRemaining(ctx context.Context, accountID string) (*Outcome, error)
+}
+
+var (
+	_ ComputeRemainder = (*ComputeCleanup)(nil)
+	_ StorageRemainder = (*StorageCleanup)(nil)
+)
+
+func validRemainderAccount(accountID string) error {
+	if accountID == "" {
+		return awserrors.Errorf(awserrors.ErrorMissingParameter, "The request must specify an account ID.")
+	}
+	if !awsidentifiers.IsAccountID(accountID) {
+		return awserrors.Errorf(awserrors.ErrorInvalidParameterValue, "The account ID must be 12 digits.")
+	}
+	return nil
+}
+
+// ComputeRemaining lists every instance the account still holds and every spot
+// request that can still launch. A read that does not complete is an incomplete
+// reason, never a shorter list. The result is a snapshot, not evidence of absence.
+func (c *ComputeCleanup) ComputeRemaining(ctx context.Context, accountID string) (*Outcome, error) {
+	if err := validRemainderAccount(accountID); err != nil {
+		return nil, err
+	}
+	if c == nil || c.spot == nil || c.instances == nil || c.teardown == nil {
+		return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "Compute cleanup is not fully wired.")
+	}
+	res := &Outcome{AccountID: accountID}
+	defer res.finish()
+
+	scan, err := c.spot.ScanAccount(ctx, accountID)
+	if err != nil {
+		slog.WarnContext(ctx, "ComputeRemaining: spot scan failed", "account_id", accountID, "err", err)
+		res.incomplete(ReasonSpotInventoryIncomplete)
+	} else {
+		for _, r := range slices.Concat(scan.Active, scan.Terminal) {
+			if !terminalSpotState(r.State) {
+				res.Items = append(res.Items, Item{ResourceID: r.RequestID, Kind: KindSpotRequest, State: r.State, Disposition: DispositionResidue, Reason: ReasonRecordRemains})
+			}
+		}
+	}
+
+	inventory, err := c.instances.AccountInstances(ctx, accountID)
+	if err != nil || !inventoryUsable(inventory) {
+		slog.WarnContext(ctx, "ComputeRemaining: instance inventory incomplete", "account_id", accountID, "err", err)
+		res.incomplete(ReasonInstanceInventoryIncomplete)
+	} else {
+		for _, reservation := range inventory.Reservations {
+			for _, in := range reservation.Instances {
+				state := ""
+				if in.State != nil {
+					state = aws.StringValue(in.State.Name)
+				}
+				res.Items = append(res.Items, Item{ResourceID: aws.StringValue(in.InstanceId), Kind: KindInstance, State: state, Disposition: DispositionResidue, Reason: ReasonRecordRemains})
+			}
+		}
+	}
+
+	outstanding, err := c.teardown.AccountTeardownOutstanding(accountID)
+	if err != nil {
+		slog.WarnContext(ctx, "ComputeRemaining: teardown unread", "account_id", accountID, "err", err)
+		res.incomplete(ReasonTeardownUnread)
+	} else {
+		for _, o := range outstanding {
+			if !res.hasInstance(o.InstanceID) {
+				res.Items = append(res.Items, Item{ResourceID: o.InstanceID, Kind: KindInstance, State: o.State, Disposition: DispositionResidue, Reason: ReasonTerminating})
+			}
+		}
+	}
+	return res, nil
+}
+
+// StorageRemaining lists every volume record the account still holds, deleting
+// ones included, and the tag entries of resources whose owner holds no record.
+func (c *StorageCleanup) StorageRemaining(ctx context.Context, accountID string) (*Outcome, error) {
+	if err := validRemainderAccount(accountID); err != nil {
+		return nil, err
+	}
+	if c == nil || c.volumes == nil || c.instances == nil || c.teardown == nil || c.spot == nil || c.tags == nil {
+		return nil, awserrors.Errorf(awserrors.ErrorServerInternal, "Storage cleanup is not fully wired.")
+	}
+	res := &Outcome{AccountID: accountID}
+	defer res.finish()
+
+	volumes, err := c.volumes.VolumeInventory(ctx, accountID)
+	if err != nil {
+		slog.WarnContext(ctx, "StorageRemaining: volume inventory incomplete", "account_id", accountID, "err", err)
+		res.incomplete(ReasonVolumeInventoryIncomplete)
+	}
+	for _, v := range volumes {
+		reason := ReasonRecordRemains
+		if v.State == ec2.VolumeStateDeleting {
+			reason = ReasonVolumeDeleting
+		}
+		res.Items = append(res.Items, Item{ResourceID: v.VolumeID, Kind: KindVolume, State: v.State, Disposition: DispositionResidue, Reason: reason})
+	}
+
+	live, err := c.instances.AccountInstances(ctx, accountID)
+	if err != nil || !inventoryUsable(live) {
+		slog.WarnContext(ctx, "StorageRemaining: no complete instance view", "account_id", accountID, "err", err)
+		res.incomplete(ReasonHolderProofIncomplete)
+		return res, nil
+	}
+	outstanding, err := c.teardown.AccountTeardownOutstanding(accountID)
+	if err != nil {
+		slog.WarnContext(ctx, "StorageRemaining: teardown unread", "account_id", accountID, "err", err)
+		res.incomplete(ReasonHolderProofIncomplete)
+		return res, nil
+	}
+
+	tagged, err := c.tags.ScanAccountResourceIDs(ctx, accountID, storageTagPrefixes...)
+	if err != nil {
+		slog.WarnContext(ctx, "StorageRemaining: tag index read failed", "account_id", accountID, "err", err)
+		res.incomplete(ReasonTagIndexUnread)
+		return res, nil
+	}
+	owned, resolved := c.ownedResourceIDs(ctx, accountID, live, outstanding)
+	for _, id := range tagged {
+		switch {
+		case !resolved[tagPrefix(id)]:
+			res.incomplete(ReasonTagOwnerUnresolved)
+			res.Items = append(res.Items, Item{ResourceID: id, Kind: KindTag, Disposition: DispositionResidue, Reason: ReasonOwnerRecordsUnresolved})
+		case owned[id]:
+			// The resource itself is reported; its tags are not residue.
+		default:
+			res.Items = append(res.Items, Item{ResourceID: id, Kind: KindTag, Disposition: DispositionResidue, Reason: ReasonRecordRemains})
+		}
+	}
+	return res, nil
+}

spinifex/domains/ec2/accountcleanup/remainder_test.go

@@ -0,0 +1,97 @@
+package accountcleanup
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/accountinventory"
+	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
+	ec2spotinstance "github.com/mulgadc/spinifex/spinifex/domains/ec2/spotinstance"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestComputeRemainingReportsEveryHeldInstanceAndOpenSpot(t *testing.T) {
+	r := newComputeRig()
+	r.spot.scans = []*ec2spotinstance.AccountScan{{
+		Active:   []ec2spotinstance.AccountSpotRequest{{RequestID: "sir-0aaaaaaaaaaaaaaa1", State: ec2.SpotInstanceStateOpen}},
+		Terminal: []ec2spotinstance.AccountSpotRequest{{RequestID: "sir-0aaaaaaaaaaaaaaa2", State: ec2.SpotInstanceStateClosed}},
+	}}
+	r.teardown.outstanding = []ec2instance.OutstandingInstance{{InstanceID: "i-0aaaaaaaaaaaaaa99", State: ec2.InstanceStateNameTerminated}}
+
+	res, err := r.cleanup().ComputeRemaining(t.Context(), testAccount)
+	require.NoError(t, err)
+
+	assert.Equal(t, StatusIncomplete, res.Status)
+	assert.NotNil(t, itemFor(res, runningID))
+	assert.NotNil(t, itemFor(res, stoppedID))
+	assert.NotNil(t, itemFor(res, "i-0aaaaaaaaaaaaaa99"))
+	assert.NotNil(t, itemFor(res, "sir-0aaaaaaaaaaaaaaa1"))
+	assert.Nil(t, itemFor(res, "sir-0aaaaaaaaaaaaaaa2"), "a closed request launches nothing")
+	assert.Empty(t, r.terminator.terminated, "a re-read changes nothing")
+}
+
+func TestComputeRemainingCleanWhenNothingHeld(t *testing.T) {
+	r := newComputeRig()
+	r.inventory.out = listing(nil)
+	res, err := r.cleanup().ComputeRemaining(t.Context(), testAccount)
+	require.NoError(t, err)
+	assert.Equal(t, StatusClean, res.Status)
+}
+
+func TestComputeRemainingUnreadStoreIsIncompleteNeverClean(t *testing.T) {
+	for name, mutate := range map[string]func(*computeRig){
+		"instances": func(r *computeRig) { r.inventory.out, r.inventory.err = nil, errors.New("node silent") },
+		"spot":      func(r *computeRig) { r.spot.scanErr = errors.New("spot unreadable") },
+		"teardown":  func(r *computeRig) { r.teardown.err = errors.New("no responder") },
+	} {
+		t.Run(name, func(t *testing.T) {
+			r := newComputeRig()
+			r.inventory.out = listing(nil)
+			mutate(r)
+			res, err := r.cleanup().ComputeRemaining(t.Context(), testAccount)
+			require.NoError(t, err)
+			assert.Equal(t, StatusIncomplete, res.Status)
+			assert.NotEmpty(t, res.IncompleteReasons)
+		})
+	}
+}
+
+func TestStorageRemainingReportsVolumeRecordsAndTagResidue(t *testing.T) {
+	r := newStorageRig()
+	r.volumes.vols = []accountinventory.VolumeItem{
+		volume(freeVol, ec2.VolumeStateAvailable, ""),
+		volume("vol-0aaaaaaaaaaaaaaa2", ec2.VolumeStateDeleting, ""),
+	}
+	r.tags.ids = []string{"i-0aaaaaaaaaaaaaaaf", freeVol}
+
+	res, err := r.cleanup().StorageRemaining(t.Context(), testAccount)
+	require.NoError(t, err)
+
+	assert.Equal(t, StatusIncomplete, res.Status)
+	assert.Equal(t, ReasonRecordRemains, itemFor(res, freeVol).Reason)
+	assert.Equal(t, ReasonVolumeDeleting, itemFor(res, "vol-0aaaaaaaaaaaaaaa2").Reason)
+	assert.Equal(t, KindTag, itemFor(res, "i-0aaaaaaaaaaaaaaaf").Kind, "tags of a resource with no record are residue")
+	assert.Empty(t, r.deleter.deleted, "a re-read deletes nothing")
+	assert.Empty(t, r.tags.deleted, "a re-read clears no tag")
+}
+
+func TestStorageRemainingUnreadStoreIsIncompleteNeverClean(t *testing.T) {
+	for name, mutate := range map[string]func(*storageRig){
+		"volumes":   func(r *storageRig) { r.volumes.failAt = -1 },
+		"instances": func(r *storageRig) { r.instances.out, r.instances.err = nil, errors.New("node silent") },
+		"teardown":  func(r *storageRig) { r.teardown.err = errors.New("no responder") },
+		"tags":      func(r *storageRig) { r.tags.scanErr = errors.New("tag index unreadable") },
+	} {
+		t.Run(name, func(t *testing.T) {
+			r := newStorageRig()
+			r.volumes.vols = nil
+			mutate(r)
+			res, err := r.cleanup().StorageRemaining(t.Context(), testAccount)
+			require.NoError(t, err)
+			assert.Equal(t, StatusIncomplete, res.Status)
+			assert.NotEmpty(t, res.IncompleteReasons)
+		})
+	}
+}

spinifex/domains/ec2/awsapi/instance/AccountTeardownOutstanding.go

@@ -0,0 +1,28 @@
+package instance
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
+	natsmsg "github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
+	"github.com/nats-io/nats.go"
+)
+
+// AccountTeardownOutstandingSubject serves the instances of an account whose
+// teardown is not complete, from the shared record store.
+const AccountTeardownOutstandingSubject = "ec2.AccountTeardownOutstanding"
+
+var accountTeardownOutstandingTimeout = 15 * time.Second
+
+// AccountTeardownOutstanding reads the account's unfinished instance teardowns.
+// No responder, a timeout or an error reply is an error, never an empty list.
+func AccountTeardownOutstanding(ctx context.Context, natsConn *nats.Conn, accountID string) ([]ec2instance.OutstandingInstance, error) {
+	out, err := natsmsg.NATSRequest[ec2instance.TeardownOutstandingOutput](ctx, natsConn, AccountTeardownOutstandingSubject,
+		&ec2instance.TeardownOutstandingInput{}, accountTeardownOutstandingTimeout, accountID)
+	if err != nil {
+		return nil, fmt.Errorf("read outstanding teardown: %w", err)
+	}
+	return out.Instances, nil
+}

spinifex/domains/ec2/awsapi/instance/AccountTeardownOutstanding_test.go

@@ -0,0 +1,88 @@
+package instance
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	ec2instance "github.com/mulgadc/spinifex/spinifex/domains/ec2/instance"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func serveTeardownOutstanding(t *testing.T, nc *nats.Conn, reply []byte) {
+	t.Helper()
+	_, err := nc.Subscribe(AccountTeardownOutstandingSubject, func(msg *nats.Msg) { _ = msg.Respond(reply) })
+	require.NoError(t, err)
+	require.NoError(t, nc.Flush())
+}
+
+func TestAccountTeardownOutstanding_ReturnsTheServedInstances(t *testing.T) {
+	t.Parallel()
+	_, nc := startTestNATSServer(t)
+	reply, err := json.Marshal(ec2instance.TeardownOutstandingOutput{Instances: []ec2instance.OutstandingInstance{{InstanceID: "i-1"}}})
+	require.NoError(t, err)
+	serveTeardownOutstanding(t, nc, reply)
+
+	got, err := AccountTeardownOutstanding(context.Background(), nc, "123456789012")
+
+	require.NoError(t, err)
+	require.Len(t, got, 1)
+	assert.Equal(t, "i-1", got[0].InstanceID)
+}
+
+// A read that cannot be answered must never look like an empty list, because an
+// empty list lets the storage cleanup believe no QEMU still holds a volume.
+func TestAccountTeardownOutstanding_UnansweredReadIsAnError(t *testing.T) {
+	t.Parallel()
+
+	t.Run("no responder", func(t *testing.T) {
+		t.Parallel()
+		_, nc := startTestNATSServer(t)
+
+		got, err := AccountTeardownOutstanding(context.Background(), nc, "123456789012")
+
+		require.Error(t, err)
+		assert.Nil(t, got)
+		assert.ErrorIs(t, err, nats.ErrNoResponders)
+	})
+	t.Run("error reply", func(t *testing.T) {
+		t.Parallel()
+		_, nc := startTestNATSServer(t)
+		serveTeardownOutstanding(t, nc, awserrors.GenerateErrorPayload(awserrors.ErrorServerInternal))
+
+		got, err := AccountTeardownOutstanding(context.Background(), nc, "123456789012")
+
+		require.Error(t, err)
+		assert.Nil(t, got)
+	})
+	t.Run("unreadable reply", func(t *testing.T) {
+		t.Parallel()
+		_, nc := startTestNATSServer(t)
+		serveTeardownOutstanding(t, nc, []byte("not json"))
+
+		got, err := AccountTeardownOutstanding(context.Background(), nc, "123456789012")
+
+		require.Error(t, err)
+		assert.Nil(t, got)
+	})
+}
+
+// Not parallel: it shortens the package timeout.
+func TestAccountTeardownOutstanding_TimeoutIsAnError(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+	_, err := nc.Subscribe(AccountTeardownOutstandingSubject, func(*nats.Msg) {})
+	require.NoError(t, err)
+	require.NoError(t, nc.Flush())
+	previous := accountTeardownOutstandingTimeout
+	accountTeardownOutstandingTimeout = 200 * time.Millisecond
+	t.Cleanup(func() { accountTeardownOutstandingTimeout = previous })
+
+	got, err := AccountTeardownOutstanding(context.Background(), nc, "123456789012")
+
+	require.Error(t, err)
+	assert.Nil(t, got)
+}

spinifex/domains/ec2/awsapi/instance/ModifyInstanceAttribute.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/nats-io/nats.go"
 )
@@ -65,6 +66,20 @@ func ModifyInstanceAttribute(ctx context.Context, input *ec2.ModifyInstanceAttri
 
 	slog.InfoContext(ctx, "ModifyInstanceAttribute: Processing request", "instance_id", *input.InstanceId)
 
+	// Termination protection on a running instance lives on its owner, so it
+	// goes there first. A nil value is not a value and keeps the queue path.
+	if input.DisableApiTermination != nil && input.DisableApiTermination.Value != nil {
+		cmd := ec2v1.EC2InstanceCommand{
+			ID:                        *input.InstanceId,
+			Attributes:                ec2v1.EC2CommandAttributes{SetDisableApiTermination: true},
+			DisableApiTerminationData: &ec2v1.DisableApiTerminationData{Value: *input.DisableApiTermination.Value},
+		}
+		if _, err := routeInstanceMutation[ec2.ModifyInstanceAttributeOutput](ctx, natsConn, *input.InstanceId, accountID, cmd, "ec2.ModifyInstanceAttribute", input); err != nil {
+			return ec2.ModifyInstanceAttributeOutput{}, err
+		}
+		return ec2.ModifyInstanceAttributeOutput{}, nil
+	}
+
 	jsonData, err := json.Marshal(input)
 	if err != nil {
 		slog.ErrorContext(ctx, "ModifyInstanceAttribute: Failed to marshal request", "instance_id", *input.InstanceId, "err", err)

spinifex/domains/ec2/awsapi/instance/ModifyInstanceAttribute_test.go

@@ -3,10 +3,13 @@ package instance
 import (
 	"context"
 	"encoding/json"
+	"sync"
+	"sync/atomic"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
@@ -226,3 +229,53 @@ func TestModifyInstanceAttribute_ValidationFailure(t *testing.T) {
 	require.Error(t, err)
 	assert.Equal(t, awserrors.ErrorInvalidParameterValue, err.Error())
 }
+
+func TestModifyInstanceAttribute_DisableApiTerminationRouting(t *testing.T) {
+	const id = "i-0123456789abcdef0"
+	_, nc := startTestNATSServer(t)
+
+	var gotValues []bool
+	var mu sync.Mutex
+	_, err := nc.Subscribe(ec2v1.InstanceCommandSubject(id), func(msg *nats.Msg) {
+		var cmd ec2v1.EC2InstanceCommand
+		if json.Unmarshal(msg.Data, &cmd) == nil && cmd.Attributes.SetDisableApiTermination && cmd.DisableApiTerminationData != nil {
+			mu.Lock()
+			gotValues = append(gotValues, cmd.DisableApiTerminationData.Value)
+			mu.Unlock()
+		}
+		reply := nats.NewMsg(msg.Reply)
+		reply.Data = []byte(`{}`)
+		reply.Header.Set(ec2v1.HandledHeader, "1")
+		_ = msg.RespondMsg(reply)
+	})
+	require.NoError(t, err)
+	var queue atomic.Int32
+	_, err = nc.Subscribe("ec2.ModifyInstanceAttribute", func(msg *nats.Msg) {
+		queue.Add(1)
+		_ = msg.Respond([]byte(`{}`))
+	})
+	require.NoError(t, err)
+
+	for _, v := range []bool{true, false} {
+		_, err := ModifyInstanceAttribute(context.Background(), &ec2.ModifyInstanceAttributeInput{
+			InstanceId:            aws.String(id),
+			DisableApiTermination: &ec2.AttributeBooleanValue{Value: aws.Bool(v)},
+		}, nc, "acct")
+		require.NoError(t, err)
+	}
+	mu.Lock()
+	assert.Equal(t, []bool{true, false}, gotValues, "explicit true and false both reach the owner")
+	mu.Unlock()
+	assert.Zero(t, queue.Load())
+
+	// A nil value is not a value: queue path, no owner command.
+	_, err = ModifyInstanceAttribute(context.Background(), &ec2.ModifyInstanceAttributeInput{
+		InstanceId:            aws.String(id),
+		DisableApiTermination: &ec2.AttributeBooleanValue{},
+	}, nc, "acct")
+	require.NoError(t, err)
+	assert.Equal(t, int32(1), queue.Load())
+	mu.Lock()
+	assert.Len(t, gotValues, 2, "no owner command for a nil value")
+	mu.Unlock()
+}

spinifex/domains/ec2/awsapi/instance/ModifyInstanceMetadataOptions.go

@@ -3,11 +3,11 @@ package instance
 import (
 	"context"
 	"errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/messaging/nats"
 	"strings"
-	"time"
 
+	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/nats-io/nats.go"
 )
@@ -22,5 +22,16 @@ func ModifyInstanceMetadataOptions(ctx context.Context, input *ec2.ModifyInstanc
 	if !strings.HasPrefix(*input.InstanceId, "i-") {
 		return nil, errors.New(awserrors.ErrorInvalidInstanceIDMalformed)
 	}
-	return natsmsg.NATSRequest[ec2.ModifyInstanceMetadataOptionsOutput](ctx, natsConn, "ec2.ModifyInstanceMetadataOptions", input, 30*time.Second, accountID)
+	cmd := ec2v1.EC2InstanceCommand{
+		ID:         *input.InstanceId,
+		Attributes: ec2v1.EC2CommandAttributes{ModifyInstanceMetadataOptions: true},
+		InstanceMetadataOptionsData: &ec2v1.InstanceMetadataOptionsData{
+			HttpTokens:              aws.StringValue(input.HttpTokens),
+			HttpEndpoint:            aws.StringValue(input.HttpEndpoint),
+			HttpProtocolIpv6:        aws.StringValue(input.HttpProtocolIpv6),
+			InstanceMetadataTags:    aws.StringValue(input.InstanceMetadataTags),
+			HttpPutResponseHopLimit: input.HttpPutResponseHopLimit,
+		},
+	}
+	return routeInstanceMutation[ec2.ModifyInstanceMetadataOptionsOutput](ctx, natsConn, *input.InstanceId, accountID, cmd, "ec2.ModifyInstanceMetadataOptions", input)
 }

spinifex/domains/ec2/awsapi/instance/ModifyInstanceMetadataOptions_test.go

@@ -2,11 +2,16 @@ package instance
 
 import (
 	"context"
+	"encoding/json"
+	"sync/atomic"
 	"testing"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/mulgadc/spinifex/contracts/ec2/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -30,3 +35,179 @@ func TestModifyInstanceMetadataOptions_RejectsMalformedID(t *testing.T) {
 		})
 	}
 }
+
+const metaTestID = "i-0123456789abcdef0"
+
+func metaInput(hop int64) *ec2.ModifyInstanceMetadataOptionsInput {
+	return &ec2.ModifyInstanceMetadataOptionsInput{
+		InstanceId:              aws.String(metaTestID),
+		HttpPutResponseHopLimit: aws.Int64(hop),
+	}
+}
+
+// ownerReply answers one ec2.cmd request, optionally carrying the handled marker.
+func ownerReply(t *testing.T, nc *nats.Conn, handled bool, body []byte, calls *atomic.Int32) {
+	t.Helper()
+	_, err := nc.Subscribe(ec2v1.InstanceCommandSubject(metaTestID), func(msg *nats.Msg) {
+		calls.Add(1)
+		reply := nats.NewMsg(msg.Reply)
+		reply.Data = body
+		if handled {
+			reply.Header.Set(ec2v1.HandledHeader, "1")
+		}
+		_ = msg.RespondMsg(reply)
+	})
+	require.NoError(t, err)
+}
+
+func queueReply(t *testing.T, nc *nats.Conn, body []byte, calls *atomic.Int32) {
+	t.Helper()
+	_, err := nc.QueueSubscribe("ec2.ModifyInstanceMetadataOptions", "spinifex-workers", func(msg *nats.Msg) {
+		calls.Add(1)
+		_ = msg.Respond(body)
+	})
+	require.NoError(t, err)
+}
+
+func metaOutputJSON(t *testing.T, hop int64) []byte {
+	t.Helper()
+	b, err := json.Marshal(&ec2.ModifyInstanceMetadataOptionsOutput{
+		InstanceId:              aws.String(metaTestID),
+		InstanceMetadataOptions: &ec2.InstanceMetadataOptionsResponse{HttpPutResponseHopLimit: aws.Int64(hop)},
+	})
+	require.NoError(t, err)
+	return b
+}
+
+func TestModifyInstanceMetadataOptions_OwnerAnswerReturnedAsIs(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+	var owner, queue atomic.Int32
+	ownerReply(t, nc, true, metaOutputJSON(t, 3), &owner)
+	queueReply(t, nc, []byte(`{}`), &queue)
+
+	out, err := ModifyInstanceMetadataOptions(context.Background(), metaInput(3), nc, "acct")
+	require.NoError(t, err)
+	assert.Equal(t, int64(3), aws.Int64Value(out.InstanceMetadataOptions.HttpPutResponseHopLimit))
+	assert.Equal(t, int32(1), owner.Load())
+	assert.Zero(t, queue.Load())
+}
+
+func TestModifyInstanceMetadataOptions_OwnerErrorWithHeaderIsNotRetried(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+	var owner, queue atomic.Int32
+	ownerReply(t, nc, true, awserrors.GenerateErrorPayload(awserrors.ErrorServerInternal), &owner)
+	queueReply(t, nc, metaOutputJSON(t, 3), &queue)
+
+	_, err := ModifyInstanceMetadataOptions(context.Background(), metaInput(3), nc, "acct")
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+	assert.Zero(t, queue.Load())
+}
+
+func TestModifyInstanceMetadataOptions_OwnerValidationErrorRelayed(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+	var owner, queue atomic.Int32
+	ownerReply(t, nc, true, awserrors.GenerateErrorPayload(awserrors.ErrorUnsupportedOperation), &owner)
+	queueReply(t, nc, metaOutputJSON(t, 3), &queue)
+
+	_, err := ModifyInstanceMetadataOptions(context.Background(), metaInput(3), nc, "acct")
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorUnsupportedOperation, err.Error())
+	assert.Zero(t, queue.Load())
+}
+
+func TestModifyInstanceMetadataOptions_HeaderlessOtherErrorReturnedAsIs(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+	var owner, queue atomic.Int32
+	ownerReply(t, nc, false, awserrors.GenerateErrorPayload(awserrors.ErrorInvalidInstanceIDNotFound), &owner)
+	queueReply(t, nc, metaOutputJSON(t, 3), &queue)
+
+	_, err := ModifyInstanceMetadataOptions(context.Background(), metaInput(3), nc, "acct")
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
+	assert.Zero(t, queue.Load())
+}
+
+func TestS9_N1OwnerRejectionFallsBackOnceToQueueGroup(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+	var owner, queue atomic.Int32
+	ownerReply(t, nc, false, awserrors.GenerateErrorPayload(awserrors.ErrorServerInternal), &owner)
+	queueReply(t, nc, metaOutputJSON(t, 3), &queue)
+
+	out, err := ModifyInstanceMetadataOptions(context.Background(), metaInput(3), nc, "acct")
+	require.NoError(t, err)
+	assert.Equal(t, int64(3), aws.Int64Value(out.InstanceMetadataOptions.HttpPutResponseHopLimit))
+	assert.Equal(t, int32(1), owner.Load(), "exactly one owner request")
+	assert.Equal(t, int32(1), queue.Load(), "exactly one queue-group request")
+}
+
+func TestModifyInstanceMetadataOptions_NoOwnerUsesQueueGroupAfterThreeAttempts(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+	var sleeps atomic.Int32
+	prev := cmdRetrySleep
+	cmdRetrySleep = func(time.Duration) { sleeps.Add(1) }
+	t.Cleanup(func() { cmdRetrySleep = prev })
+
+	var queue atomic.Int32
+	queueReply(t, nc, metaOutputJSON(t, 2), &queue)
+
+	out, err := ModifyInstanceMetadataOptions(context.Background(), metaInput(2), nc, "acct")
+	require.NoError(t, err)
+	assert.Equal(t, int64(2), aws.Int64Value(out.InstanceMetadataOptions.HttpPutResponseHopLimit))
+	assert.Equal(t, int32(2), sleeps.Load(), "three attempts means two backoffs")
+	assert.Equal(t, int32(1), queue.Load())
+}
+
+func TestModifyInstanceMetadataOptions_AbsentIsNotFound(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+	var queue atomic.Int32
+	queueReply(t, nc, awserrors.GenerateErrorPayload(awserrors.ErrorInvalidInstanceIDNotFound), &queue)
+
+	_, err := ModifyInstanceMetadataOptions(context.Background(), metaInput(2), nc, "acct")
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorInvalidInstanceIDNotFound, err.Error())
+}
+
+func TestModifyInstanceMetadataOptions_OwnerTimeoutIsServerInternalWithoutFallback(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+	prev := instanceCmdTimeout
+	instanceCmdTimeout = 50 * time.Millisecond
+	t.Cleanup(func() { instanceCmdTimeout = prev })
+
+	// The owner applies the change, then withholds its reply.
+	var applied, queue atomic.Int32
+	_, err := nc.Subscribe(ec2v1.InstanceCommandSubject(metaTestID), func(*nats.Msg) { applied.Add(1) })
+	require.NoError(t, err)
+	queueReply(t, nc, metaOutputJSON(t, 3), &queue)
+
+	_, err = ModifyInstanceMetadataOptions(context.Background(), metaInput(3), nc, "acct")
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+	assert.Equal(t, int32(1), applied.Load(), "the owner saw the command, so the outcome is unknown")
+	assert.Zero(t, queue.Load(), "a timeout never falls back to the queue group")
+}
+
+func TestModifyInstanceMetadataOptions_TimeoutAfterApplyIsUnknownNotRolledBack(t *testing.T) {
+	_, nc := startTestNATSServer(t)
+	prev := instanceCmdTimeout
+	instanceCmdTimeout = 50 * time.Millisecond
+	t.Cleanup(func() { instanceCmdTimeout = prev })
+
+	// The owner persists the change, then the reply is lost.
+	var persisted atomic.Int64
+	_, err := nc.Subscribe(ec2v1.InstanceCommandSubject(metaTestID), func(msg *nats.Msg) {
+		var cmd ec2v1.EC2InstanceCommand
+		if json.Unmarshal(msg.Data, &cmd) == nil && cmd.InstanceMetadataOptionsData != nil {
+			persisted.Store(aws.Int64Value(cmd.InstanceMetadataOptionsData.HttpPutResponseHopLimit))
+		}
+	})
+	require.NoError(t, err)
+	var queue atomic.Int32
+	queueReply(t, nc, metaOutputJSON(t, 1), &queue)
+
+	_, err = ModifyInstanceMetadataOptions(context.Background(), metaInput(3), nc, "acct")
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+	assert.Zero(t, queue.Load(), "no fallback that could race a live owner")
+	assert.Equal(t, int64(3), persisted.Load(), "the gateway error does not undo what the owner stored")
+}

spinifex/domains/ec2/awsapi/instance/instanceid.go

@@ -99,3 +99,46 @@ func classifyUnownedInstance(ctx context.Context, natsConn *nats.Conn, instanceI
 	}
 	return bucketStateAbsent, allOK
 }
+
+// routeInstanceMutation sends cmd to the instance's owner first and falls back
+// to the queue-group subject only when that is safe: no daemon runs the
+// instance (ErrNoResponders after every retry), or the owner is N-1 and
+// rejected the command without writing (ServerInternal with no HandledHeader).
+// A timeout or other transport error is an unknown outcome and never falls back.
+func routeInstanceMutation[Out any](ctx context.Context, natsConn *nats.Conn, instanceID, accountID string, cmd ec2v1.EC2InstanceCommand, queueSubject string, queueInput any) (*Out, error) {
+	queue := func() (*Out, error) {
+		return natsmsg.NATSRequest[Out](ctx, natsConn, queueSubject, queueInput, 30*time.Second, accountID)
+	}
+
+	data, err := json.Marshal(cmd)
+	if err != nil {
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	msg, err := requestInstanceCmd(ctx, natsConn, instanceID, accountID, data)
+	if err != nil {
+		if errors.Is(err, nats.ErrNoResponders) {
+			return queue()
+		}
+		slog.ErrorContext(ctx, "Owner request failed, outcome unknown", "instance_id", instanceID, "subject", queueSubject, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+
+	responseError, parseErr := awserrors.ValidateErrorPayload(msg.Data)
+	if parseErr != nil {
+		handled := msg.Header.Get(ec2v1.HandledHeader) == "1"
+		if !handled && *responseError.Code == awserrors.ErrorServerInternal {
+			slog.WarnContext(ctx, "Owner did not recognise the command, using the queue group", "instance_id", instanceID, "subject", queueSubject)
+			return queue()
+		}
+		if responseError.Message != nil && *responseError.Message != "" {
+			return nil, awserrors.Errorf(*responseError.Code, "%s", *responseError.Message)
+		}
+		return nil, errors.New(*responseError.Code)
+	}
+
+	var out Out
+	if err := json.Unmarshal(msg.Data, &out); err != nil {
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	return &out, nil
+}

spinifex/domains/ec2/ebs/metadata/metadata.go

@@ -110,6 +110,23 @@ type AMI struct {
 	Tags            map[string]string `json:"tags,omitempty"`
 	SourceDigest    *ImageDigest      `json:"source_digest,omitempty"`
 	State           string            `json:"state,omitempty"`
+	// UEFIData names the image's own UEFI variable store object; nil launches
+	// from the node's VARS template.
+	UEFIData *UEFIDataRef `json:"uefi_data,omitempty"`
+}
+
+// UEFIDataRef names an image's UEFI variable store object and the firmware it
+// was written under, so a launch can check the firmware without reading it.
+type UEFIDataRef struct {
+	Key      string       `json:"key"`
+	Firmware UEFIFirmware `json:"firmware"`
+}
+
+// UEFIFirmware is the firmware a variable store was written under.
+type UEFIFirmware struct {
+	Arch       string `json:"arch"`
+	VarsSize   int64  `json:"vars_size"`
+	CodeSHA256 string `json:"code_sha256"`
 }
 
 // ImageDigest identifies the artifact an AMI was imported from, not the

spinifex/domains/ec2/ebs/metadata/metadata_test.go

@@ -73,6 +73,16 @@ func TestAMIWithoutSourceDigestStillDecodes(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, "ami-1", got.ImageID)
 	assert.Nil(t, got.SourceDigest)
+	assert.Nil(t, got.UEFIData, "a document with no uefi_data launches from the template")
+}
+
+// The UEFI store reference rides at schema version 2 under its JSON names.
+func TestAMIUEFIDataDecodesAtSchemaVersion2(t *testing.T) {
+	got, err := UnmarshalAMI([]byte(`{"schema_version":2,"image_id":"ami-1","uefi_data":` +
+		`{"key":"uefi/ami-1/k","firmware":{"arch":"x86_64","vars_size":540672,"code_sha256":"abc"}}}`))
+	require.NoError(t, err)
+	require.NotNil(t, got.UEFIData)
+	assert.Equal(t, UEFIDataRef{Key: "uefi/ami-1/k", Firmware: UEFIFirmware{Arch: "x86_64", VarsSize: 540672, CodeSHA256: "abc"}}, *got.UEFIData)
 }
 
 func TestMetadataRejectsUnknownSchemaVersion(t *testing.T) {

spinifex/domains/ec2/eigw/service_impl.go

@@ -19,7 +19,7 @@ import (
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -56,7 +56,7 @@ func NewEgressOnlyIGWServiceImplWithNATS(ctx context.Context, cfg *config.Config
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	eigwKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketEgressOnlyIGW, 10)
+	eigwKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketEgressOnlyIGW, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketEgressOnlyIGW, err)
 	}
@@ -65,7 +65,7 @@ func NewEgressOnlyIGWServiceImplWithNATS(ctx context.Context, cfg *config.Config
 	}
 
 	// Get or create VPC KV bucket for cross-resource ownership validation
-	vpcKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
+	vpcKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get VPC KV bucket: %w", err)
 	}
@@ -90,7 +90,7 @@ func (s *EgressOnlyIGWServiceImpl) CreateEgressOnlyInternetGateway(ctx context.C
 		slog.ErrorContext(ctx, "VPC KV unavailable, cannot verify VPC ownership")
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-	if _, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, *input.VpcId)); err != nil {
+	if _, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, *input.VpcId)); err != nil {
 		slog.WarnContext(ctx, "CreateEgressOnlyInternetGateway: VPC not found for account", "vpcId", *input.VpcId, "accountID", accountID)
 		return nil, awserrors.IDNotFound(awserrors.ErrorInvalidVpcIDNotFound, "vpc", *input.VpcId)
 	}
@@ -109,7 +109,7 @@ func (s *EgressOnlyIGWServiceImpl) CreateEgressOnlyInternetGateway(ctx context.C
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal Egress-only IGW record: %w", err)
 	}
-	if _, err := s.eigwKV.Put(ctx, kvutil.AccountKey(accountID, eigwID), data); err != nil {
+	if _, err := s.eigwKV.Put(ctx, kvstore.AccountKey(accountID, eigwID), data); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -127,7 +127,7 @@ func (s *EgressOnlyIGWServiceImpl) DeleteEgressOnlyInternetGateway(ctx context.C
 	}
 
 	eigwID := *input.EgressOnlyInternetGatewayId
-	key := kvutil.AccountKey(accountID, eigwID)
+	key := kvstore.AccountKey(accountID, eigwID)
 
 	// Verify the EIGW exists before deleting
 	if _, err := s.eigwKV.Get(ctx, key); err != nil {
@@ -178,7 +178,7 @@ func (s *EgressOnlyIGWServiceImpl) DescribeEgressOnlyInternetGateways(ctx contex
 	}
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey {
+		if key == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(key, prefix) {

spinifex/domains/ec2/eigw/service_impl_test.go

@@ -9,7 +9,7 @@ import (
 	"github.com/mulgadc/spinifex/internal/testkit"
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -23,7 +23,7 @@ func setupTestEIGWService(t *testing.T) *EgressOnlyIGWServiceImpl {
 	// Create VPC KV bucket and register test VPCs so fail-closed ownership checks pass
 	vpcEntries := map[string][]byte{}
 	for _, vpcID := range []string{"vpc-test123", "vpc-tagged", "vpc-tagged2"} {
-		vpcEntries[kvutil.AccountKey(testAccountID, vpcID)] = []byte(`{"vpc_id":"` + vpcID + `","state":"available"}`)
+		vpcEntries[kvstore.AccountKey(testAccountID, vpcID)] = []byte(`{"vpc_id":"` + vpcID + `","state":"available"}`)
 	}
 	testutil.SeedKV(t, js, ec2vpc.KVBucketVPCs, vpcEntries)
 
@@ -312,7 +312,7 @@ func TestCreateEgressOnlyInternetGateway_CrossAccountVPCRejected(t *testing.T) {
 	// Create VPC KV bucket with a VPC owned by testAccountID
 	vpcID := "vpc-alpha456"
 	testutil.SeedKV(t, js, ec2vpc.KVBucketVPCs, map[string][]byte{
-		kvutil.AccountKey(testAccountID, vpcID): []byte(`{"vpc_id":"vpc-alpha456","state":"available"}`),
+		kvstore.AccountKey(testAccountID, vpcID): []byte(`{"vpc_id":"vpc-alpha456","state":"available"}`),
 	})
 
 	var err error

spinifex/domains/ec2/eip/allocate_idempotency.go

@@ -12,7 +12,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/spinifex/foundation/lifecycle/idempotency"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -36,7 +36,7 @@ func (s *EIPServiceImpl) allocateOnce(ctx context.Context, accountID string, all
 	if key == "" || s.idemKV == nil {
 		return alloc()
 	}
-	kvKey := kvutil.AccountKey(accountID, idempotencyKeyHash(key))
+	kvKey := kvstore.AccountKey(accountID, idempotencyKeyHash(key))
 
 	if out := s.cachedAllocation(ctx, kvKey); out != nil {
 		slog.WarnContext(ctx, "AllocateAddress: returning first result for a retried request",

spinifex/domains/ec2/eip/lease_rebind.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/mulgadc/spinifex/spinifex/domains/network/projection"
 	"github.com/mulgadc/spinifex/spinifex/domains/network/topology"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
 
@@ -80,7 +80,7 @@ func (s *EIPServiceImpl) AllocationExists(ctx context.Context, allocationID stri
 	}
 	suffix := "." + allocationID
 	for _, k := range keys {
-		if k != kvutil.VersionKey && strings.HasSuffix(k, suffix) {
+		if k != kvstore.VersionKey && strings.HasSuffix(k, suffix) {
 			return true, nil
 		}
 	}
@@ -98,7 +98,7 @@ func (s *EIPServiceImpl) findByAllocationIDAnyAccount(ctx context.Context, alloc
 
 	suffix := "." + allocationID
 	for _, k := range keys {
-		if k == kvutil.VersionKey || !strings.HasSuffix(k, suffix) {
+		if k == kvstore.VersionKey || !strings.HasSuffix(k, suffix) {
 			continue
 		}
 		entry, err := s.eipKV.Get(ctx, k)

spinifex/domains/ec2/eip/service_impl.go

@@ -20,7 +20,7 @@ import (
 	awsfilters "github.com/mulgadc/spinifex/spinifex/foundation/aws/filters"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -66,7 +66,7 @@ func NewEIPServiceImpl(ctx context.Context, natsConn *nats.Conn, externalIPAM *e
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	eipKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketEIPs, 10)
+	eipKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketEIPs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketEIPs, err)
 	}
@@ -76,7 +76,7 @@ func NewEIPServiceImpl(ctx context.Context, natsConn *nats.Conn, externalIPAM *e
 
 	// TTL'd, so a failure to create it is not worth refusing to serve EIPs over:
 	// a nil bucket only costs post-completion dedupe.
-	idemKV, err := kvutil.GetOrCreateBucketWithTTL(ctx, js, KVBucketEIPIdempotency, 1, eipIdempotencyTTL)
+	idemKV, err := kvstore.GetOrCreateBucketWithTTL(ctx, js, KVBucketEIPIdempotency, 1, eipIdempotencyTTL)
 	if err != nil {
 		slog.Warn("EIP service: idempotency bucket unavailable; a retried AllocateAddress may allocate twice",
 			"bucket", KVBucketEIPIdempotency, "err", err)
@@ -146,7 +146,7 @@ func (s *EIPServiceImpl) allocateAddress(ctx context.Context, input *ec2.Allocat
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal EIP record: %w", err)
 	}
-	if _, err := s.eipKV.Put(ctx, kvutil.AccountKey(accountID, allocID), data); err != nil {
+	if _, err := s.eipKV.Put(ctx, kvstore.AccountKey(accountID, allocID), data); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -167,7 +167,7 @@ func (s *EIPServiceImpl) ReleaseAddress(ctx context.Context, input *ec2.ReleaseA
 	}
 
 	allocID := *input.AllocationId
-	key := kvutil.AccountKey(accountID, allocID)
+	key := kvstore.AccountKey(accountID, allocID)
 
 	entry, err := s.eipKV.Get(ctx, key)
 	if err != nil {
@@ -206,7 +206,7 @@ func (s *EIPServiceImpl) AssociateAddress(ctx context.Context, input *ec2.Associ
 	}
 
 	allocID := *input.AllocationId
-	key := kvutil.AccountKey(accountID, allocID)
+	key := kvstore.AccountKey(accountID, allocID)
 
 	entry, err := s.eipKV.Get(ctx, key)
 	if err != nil {
@@ -536,7 +536,7 @@ func (s *EIPServiceImpl) findByENI(ctx context.Context, accountID, eniID string)
 		if err := ctx.Err(); err != nil {
 			return nil, err
 		}
-		if k == kvutil.VersionKey || !strings.HasPrefix(k, prefix) {
+		if k == kvstore.VersionKey || !strings.HasPrefix(k, prefix) {
 			continue
 		}
 		entry, err := s.eipKV.Get(ctx, k)
@@ -590,7 +590,7 @@ func (s *EIPServiceImpl) DescribeAddresses(ctx context.Context, input *ec2.Descr
 
 	var addresses []*ec2.Address
 	for _, k := range keys {
-		if k == kvutil.VersionKey {
+		if k == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(k, prefix) {
@@ -699,7 +699,7 @@ func (s *EIPServiceImpl) DescribeAddressesAttribute(ctx context.Context, input *
 			if id == nil {
 				continue
 			}
-			key := kvutil.AccountKey(accountID, *id)
+			key := kvstore.AccountKey(accountID, *id)
 			entry, err := s.eipKV.Get(ctx, key)
 			if err != nil {
 				continue // not found, skip
@@ -722,7 +722,7 @@ func (s *EIPServiceImpl) DescribeAddressesAttribute(ctx context.Context, input *
 			return nil, errors.New(awserrors.ErrorServerInternal)
 		}
 		for _, k := range keys {
-			if k == kvutil.VersionKey {
+			if k == kvstore.VersionKey {
 				continue
 			}
 			if !strings.HasPrefix(k, prefix) {
@@ -819,7 +819,7 @@ func (s *EIPServiceImpl) findByAssociationID(ctx context.Context, accountID, ass
 	}
 
 	for _, k := range keys {
-		if k == kvutil.VersionKey {
+		if k == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(k, prefix) {
@@ -857,7 +857,7 @@ func (s *EIPServiceImpl) AssociatedPublicIPForInstance(ctx context.Context, acco
 		return "", false
 	}
 	for _, k := range keys {
-		if k == kvutil.VersionKey || !strings.HasPrefix(k, prefix) {
+		if k == kvstore.VersionKey || !strings.HasPrefix(k, prefix) {
 			continue
 		}
 		entry, err := s.eipKV.Get(ctx, k)

spinifex/domains/ec2/igw/service_impl.go

@@ -22,7 +22,7 @@ import (
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/paging"
 	awstags "github.com/mulgadc/spinifex/spinifex/foundation/aws/tags"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/migrate"
 	"github.com/nats-io/nats.go"
 	"github.com/nats-io/nats.go/jetstream"
@@ -93,7 +93,7 @@ func NewIGWServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 		return nil, fmt.Errorf("failed to get JetStream context: %w", err)
 	}
 
-	igwKV, err := kvutil.GetOrCreateBucket(ctx, js, KVBucketIGW, 10)
+	igwKV, err := kvstore.GetOrCreateBucket(ctx, js, KVBucketIGW, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create KV bucket %s: %w", KVBucketIGW, err)
 	}
@@ -102,13 +102,13 @@ func NewIGWServiceImplWithNATS(ctx context.Context, cfg *config.Config, natsConn
 	}
 
 	// Get or create VPC KV bucket for cross-resource ownership validation
-	vpcKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
+	vpcKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2vpc.KVBucketVPCs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get VPC KV bucket: %w", err)
 	}
 
 	// Read EIP associations so detach can refuse while addresses are mapped.
-	eipKV, err := kvutil.GetOrCreateBucket(ctx, js, ec2eip.KVBucketEIPs, 10)
+	eipKV, err := kvstore.GetOrCreateBucket(ctx, js, ec2eip.KVBucketEIPs, 10)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get EIP KV bucket: %w", err)
 	}
@@ -142,7 +142,7 @@ func (s *IGWServiceImpl) createIGW(ctx context.Context, input *ec2.CreateInterne
 	if err != nil {
 		return nil, fmt.Errorf("failed to marshal IGW record: %w", err)
 	}
-	if _, err := s.igwKV.Put(ctx, kvutil.AccountKey(accountID, igwID), data); err != nil {
+	if _, err := s.igwKV.Put(ctx, kvstore.AccountKey(accountID, igwID), data); err != nil {
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -160,7 +160,7 @@ func (s *IGWServiceImpl) DeleteInternetGateway(ctx context.Context, input *ec2.D
 	}
 
 	igwID := *input.InternetGatewayId
-	key := kvutil.AccountKey(accountID, igwID)
+	key := kvstore.AccountKey(accountID, igwID)
 
 	entry, err := s.igwKV.Get(ctx, key)
 	if err != nil {
@@ -236,7 +236,7 @@ func (s *IGWServiceImpl) DescribeInternetGateways(ctx context.Context, input *ec
 	foundIDs := make(map[string]bool)
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey {
+		if key == kvstore.VersionKey {
 			continue
 		}
 		if !strings.HasPrefix(key, prefix) {
@@ -326,7 +326,7 @@ func (s *IGWServiceImpl) AttachInternetGateway(ctx context.Context, input *ec2.A
 
 	igwID := *input.InternetGatewayId
 	vpcID := *input.VpcId
-	key := kvutil.AccountKey(accountID, igwID)
+	key := kvstore.AccountKey(accountID, igwID)
 
 	entry, err := s.igwKV.Get(ctx, key)
 	if err != nil {
@@ -347,7 +347,7 @@ func (s *IGWServiceImpl) AttachInternetGateway(ctx context.Context, input *ec2.A
 		slog.ErrorContext(ctx, "VPC KV unavailable, cannot verify VPC ownership")
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
-	if _, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, vpcID)); err != nil {
+	if _, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, vpcID)); err != nil {
 		slog.WarnContext(ctx, "AttachInternetGateway: VPC not found for account", "vpcId", vpcID, "accountID", accountID)
 		return nil, awserrors.IDNotFound(awserrors.ErrorInvalidVpcIDNotFound, "vpc", vpcID)
 	}
@@ -388,7 +388,7 @@ func (s *IGWServiceImpl) DetachInternetGateway(ctx context.Context, input *ec2.D
 
 	igwID := *input.InternetGatewayId
 	vpcID := *input.VpcId
-	key := kvutil.AccountKey(accountID, igwID)
+	key := kvstore.AccountKey(accountID, igwID)
 
 	entry, err := s.igwKV.Get(ctx, key)
 	if err != nil {
@@ -449,7 +449,7 @@ func (s *IGWServiceImpl) DetachInternetGateway(ctx context.Context, input *ec2.D
 // associated into vpcID, whether to an interface or a NAT gateway. The listing
 // is filtered server-side to the account's keys.
 func (s *IGWServiceImpl) vpcHasMappedAddress(ctx context.Context, accountID, vpcID string) (bool, error) {
-	lister, err := s.eipKV.ListKeysFiltered(ctx, kvutil.AccountKey(accountID, "*"))
+	lister, err := s.eipKV.ListKeysFiltered(ctx, kvstore.AccountKey(accountID, "*"))
 	if err != nil {
 		return false, err
 	}
@@ -493,11 +493,11 @@ func (s *IGWServiceImpl) CreateAttachedInternetGateway(ctx context.Context, acco
 	if s.vpcKV == nil {
 		return false, errors.New(awserrors.ErrorServerInternal)
 	}
-	if _, err := s.vpcKV.Get(ctx, kvutil.AccountKey(accountID, vpcID)); err != nil {
+	if _, err := s.vpcKV.Get(ctx, kvstore.AccountKey(accountID, vpcID)); err != nil {
 		return false, awserrors.IDNotFound(awserrors.ErrorInvalidVpcIDNotFound, "vpc", vpcID)
 	}
 
-	key := kvutil.AccountKey(accountID, igwID)
+	key := kvstore.AccountKey(accountID, igwID)
 	data, err := json.Marshal(IGWRecord{
 		InternetGatewayId: igwID,
 		VpcId:             vpcID,
@@ -558,7 +558,7 @@ func (s *IGWServiceImpl) AttachmentIntent(ctx context.Context, accountID, vpcID
 	}
 
 	for _, key := range keys {
-		if key == kvutil.VersionKey || !strings.HasPrefix(key, prefix) {
+		if key == kvstore.VersionKey || !strings.HasPrefix(key, prefix) {
 			continue
 		}
 		entry, err := s.igwKV.Get(ctx, key)
@@ -599,7 +599,7 @@ func (s *IGWServiceImpl) AttachmentIntent(ctx context.Context, accountID, vpcID
 // Writing only on the pending transition keeps a pass from waking the drift loop.
 func MarkAttached(ctx context.Context, kv jetstream.KeyValue, recordKey, vpcID string) error {
 	confirmed := false
-	record, err := kvutil.Update(ctx, kv, recordKey, kvutil.CASConfig{}, func(r *IGWRecord) (bool, error) {
+	record, err := kvstore.Update(ctx, kv, recordKey, kvstore.CASConfig{}, func(r *IGWRecord) (bool, error) {
 		// Reset per attempt: a retried CAS may see a record another writer has
 		// already moved out of pending.
 		confirmed = r.VpcId == vpcID && r.AttachState == AttachStatePending

spinifex/domains/ec2/igw/service_impl_test.go

@@ -13,7 +13,7 @@ import (
 	ec2eip "github.com/mulgadc/spinifex/spinifex/domains/ec2/eip"
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
 	"github.com/nats-io/nats.go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -55,7 +55,7 @@ func setupTestIGWService(t *testing.T) (*IGWServiceImpl, *nats.Conn) {
 	// Create VPC KV bucket and register test VPCs so fail-closed ownership checks pass
 	vpcEntries := map[string][]byte{}
 	for _, vpcID := range []string{"vpc-test123", "vpc-other", "vpc-lifecycle", "vpc-event-test"} {
-		vpcEntries[kvutil.AccountKey(testAccountID, vpcID)] = []byte(`{"vpc_id":"` + vpcID + `","state":"available"}`)
+		vpcEntries[kvstore.AccountKey(testAccountID, vpcID)] = []byte(`{"vpc_id":"` + vpcID + `","state":"available"}`)
 	}
 	testutil.SeedKV(t, js, ec2vpc.KVBucketVPCs, vpcEntries)
 
@@ -68,7 +68,7 @@ func setupTestIGWService(t *testing.T) (*IGWServiceImpl, *nats.Conn) {
 // gateway come up. Without it an attach stays pending and reports no attachment.
 func confirmAttach(t *testing.T, svc *IGWServiceImpl, igwID, vpcID string) {
 	t.Helper()
-	require.NoError(t, MarkAttached(context.Background(), svc.igwKV, kvutil.AccountKey(testAccountID, igwID), vpcID))
+	require.NoError(t, MarkAttached(context.Background(), svc.igwKV, kvstore.AccountKey(testAccountID, igwID), vpcID))
 }
 
 func createTestIGW(t *testing.T, svc *IGWServiceImpl) string {
@@ -331,7 +331,7 @@ func TestDescribeInternetGateways_LegacyRecordReportsAttachment(t *testing.T) {
 	svc, _ := setupTestIGWService(t)
 	igwID := createTestIGW(t, svc)
 
-	key := kvutil.AccountKey(testAccountID, igwID)
+	key := kvstore.AccountKey(testAccountID, igwID)
 	entry, err := svc.igwKV.Get(context.Background(), key)
 	require.NoError(t, err)
 	var record IGWRecord
@@ -398,7 +398,7 @@ func TestMarkAttached_NoRewriteWhenAlreadyAttached(t *testing.T) {
 	}, testAccountID)
 	require.NoError(t, err)
 
-	key := kvutil.AccountKey(testAccountID, igwID)
+	key := kvstore.AccountKey(testAccountID, igwID)
 	confirmAttach(t, svc, igwID, "vpc-test123")
 	first, err := svc.igwKV.Get(context.Background(), key)
 	require.NoError(t, err)
@@ -605,7 +605,7 @@ func TestDetachInternetGateway_MappedPublicAddress(t *testing.T) {
 	putEIP := func(rec ec2eip.EIPRecord) {
 		data, err := json.Marshal(rec)
 		require.NoError(t, err)
-		_, err = svc.eipKV.Put(context.Background(), kvutil.AccountKey(testAccountID, rec.AllocationId), data)
+		_, err = svc.eipKV.Put(context.Background(), kvstore.AccountKey(testAccountID, rec.AllocationId), data)
 		require.NoError(t, err)
 	}
 	putEIP(ec2eip.EIPRecord{AllocationId: "eipalloc-other", AssociationId: "eipassoc-other", VpcId: "vpc-other", State: "associated"})
@@ -614,7 +614,7 @@ func TestDetachInternetGateway_MappedPublicAddress(t *testing.T) {
 	// Another account's address naming the same VPC ID is not this account's dependency.
 	foreign, err := json.Marshal(ec2eip.EIPRecord{AllocationId: "eipalloc-foreign", AssociationId: "eipassoc-foreign", VpcId: "vpc-test123", State: "associated"})
 	require.NoError(t, err)
-	_, err = svc.eipKV.Put(context.Background(), kvutil.AccountKey("999999999999", "eipalloc-foreign"), foreign)
+	_, err = svc.eipKV.Put(context.Background(), kvstore.AccountKey("999999999999", "eipalloc-foreign"), foreign)
 	require.NoError(t, err)
 
 	detach := func() error {
@@ -828,7 +828,7 @@ func TestAttachInternetGateway_CrossAccountVPCRejected(t *testing.T) {
 	require.NoError(t, err)
 
 	vpcID := "vpc-alpha123"
-	_, err = vpcKV.Put(t.Context(), kvutil.AccountKey(testAccountID, vpcID), []byte(`{"vpc_id":"vpc-alpha123","state":"available"}`))
+	_, err = vpcKV.Put(t.Context(), kvstore.AccountKey(testAccountID, vpcID), []byte(`{"vpc_id":"vpc-alpha123","state":"available"}`))
 	require.NoError(t, err)
 
 	// Refresh service to pick up the VPC KV bucket

spinifex/domains/ec2/image/deregister_legacy_config_test.go

@@ -1,7 +1,9 @@
 package image
 
 import (
+	"bytes"
 	"context"
+	"io"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -12,17 +14,22 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-// TestDeregisterImage_NoLegacyConfig covers every AMI created since the
-// metadata store landed: they have no {imageID}/config.json, so deleting one
-// answers NoSuchKey. Treating that as a failure returned InternalError after
-// the AMI document was already gone, and a retrying caller then saw
-// InvalidAMIID.NotFound for an image it had just been told it could not delete.
+// TestDeregisterImage_NoLegacyConfig: deregistering deletes the AMI document
+// and nothing under {imageID}/, where a system AMI's image volume keeps its
+// own config.json under the same ID.
 func TestDeregisterImage_NoLegacyConfig(t *testing.T) {
 	mem := objectstore.NewMemoryObjectStore()
 	svc := NewImageServiceImplWithStore(mem, testBucket)
 
 	amiID := "ami-nolegacy001"
 	createTestAMIConfigWithOwner(t, mem, amiID, "no-legacy-config", testAccountID)
+	volumeConfigKey := amiID + "/config.json"
+	_, err := mem.PutObject(context.Background(), &awss3.PutObjectInput{
+		Bucket: aws.String(testBucket),
+		Key:    aws.String(volumeConfigKey),
+		Body:   bytes.NewReader([]byte(`{"volume":"state"}`)),
+	})
+	require.NoError(t, err)
 
 	out, err := svc.DeregisterImage(context.Background(), &ec2.DeregisterImageInput{ImageId: aws.String(amiID)}, testAccountID)
 	require.NoError(t, err)
@@ -33,7 +40,16 @@ func TestDeregisterImage_NoLegacyConfig(t *testing.T) {
 		Key:    aws.String(amiDocumentKey(t, amiID)),
 	})
 	require.Error(t, getErr)
-	assert.True(t, objectstore.IsNoSuchKeyError(getErr), "the AMI document must still be deleted")
+	assert.True(t, objectstore.IsNoSuchKeyError(getErr), "the AMI document must be deleted")
+
+	kept, err := mem.GetObject(context.Background(), &awss3.GetObjectInput{
+		Bucket: aws.String(testBucket),
+		Key:    aws.String(volumeConfigKey),
+	})
+	require.NoError(t, err, "the object under {imageID}/ must be left alone")
+	body, err := io.ReadAll(kept.Body)
+	require.NoError(t, err)
+	assert.JSONEq(t, `{"volume":"state"}`, string(body))
 }
 
 // racingDeleteStore lets a concurrent deleter win every DeleteObject, so the

spinifex/domains/ec2/image/images_promote.go

@@ -38,7 +38,7 @@ type PromoteImageResult struct {
 //
 // Guards:
 //   - ImageID must have "ami-" prefix
-//   - config.json must exist and parse cleanly
+//   - the AMI document must exist and parse cleanly
 //   - AMI must currently be account-owned; already-system AMIs are rejected
 //   - the snapshot document must be readable, so it can be re-keyed
 func PromoteSystemImage(store objectstore.ObjectStore, bucket string, opts PromoteImageOpts) (*PromoteImageResult, error) {

spinifex/domains/ec2/image/service_impl.go

@@ -13,11 +13,11 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/mulgadc/bluebottle/pkg/safecast"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	ebsencryption "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/encryption"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
 	ec2platform "github.com/mulgadc/spinifex/spinifex/domains/ec2/platform"
 	ec2snapshot "github.com/mulgadc/spinifex/spinifex/domains/ec2/snapshot"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
@@ -39,6 +39,9 @@ type CreateImageParams struct {
 	RootVolumeID  string
 	SourceImageID string
 	IsRunning     bool // logged only: the provider's CreateSnapshot drains a live volume itself
+	// UEFI is set for an instance that boots through UEFI, whose variable
+	// store the image carries.
+	UEFI bool
 }
 
 // ImageServiceImpl handles AMI image operations with S3 storage.
@@ -49,6 +52,9 @@ type ImageServiceImpl struct {
 	natsConn   *nats.Conn
 	metadata   *ebsmetadata.Store
 	provider   ebsprovider.EBSProvider
+
+	uefiRecords UEFIStoreRecords
+	nodeID      string
 }
 
 // SetEBSProvider injects the provider boundary used by the control plane.
@@ -122,7 +128,7 @@ var describeImagesValidFilters = map[string]bool{
 	"root-device-type":    true,
 }
 
-// DescribeImages lists available AMI images by reading config.json files from S3.
+// DescribeImages lists available AMI images from their AMI metadata documents.
 func (s *ImageServiceImpl) DescribeImages(ctx context.Context, input *ec2.DescribeImagesInput, accountID string) (*ec2.DescribeImagesOutput, error) {
 	if input == nil {
 		input = &ec2.DescribeImagesInput{}
@@ -444,12 +450,23 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 		"rootVolumeId", params.RootVolumeID, "amiId", amiID, "snapshotId", snapshotID,
 		"isRunning", params.IsRunning)
 
-	// Step 1: Snapshot root volume
+	// Step 1: Copy the instance's UEFI variable store to the image's own key,
+	// before the snapshot, so a refusal here leaves no snapshot behind.
+	var uefiData *ebsmetadata.UEFIDataRef
+	if params.UEFI {
+		var err error
+		uefiData, err = s.copyInstanceUEFIStore(context.Background(), *input.InstanceId, amiID)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Step 2: Snapshot root volume
 	if err := s.snapshotVolume(params.RootVolumeID, snapshotID, accountID); err != nil {
 		return nil, err
 	}
 
-	// Step 2: Read source volume config for size
+	// Step 3: Read source volume config for size
 	volMeta, err := s.getVolumeMetadata(context.Background(), accountID, params.RootVolumeID)
 	if err != nil {
 		slog.Error("CreateImageFromInstance: failed to read volume metadata", "volumeId", params.RootVolumeID, "err", err)
@@ -457,7 +474,7 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 	}
 	volumeSizeGiB := volMeta.CapacityGiB
 
-	// Step 3: Read source AMI config for architecture, platform, etc.
+	// Step 4: Read source AMI config for architecture, platform, etc.
 	sourceAMI := ebsmetadata.AMI{
 		Architecture:    "x86_64",
 		PlatformDetails: "Linux/UNIX",
@@ -472,13 +489,13 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 		sourceAMI = srcCfg
 	}
 
-	// Step 4: Store snapshot metadata
+	// Step 5: Store snapshot metadata
 	if err := s.putSnapshotMetadata(context.Background(), snapshotID, params.RootVolumeID, volumeSizeGiB, accountID); err != nil {
 		slog.Error("CreateImageFromInstance: failed to write snapshot metadata", "snapshotId", snapshotID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
-	// Step 5: Build and store AMI config
+	// Step 6: Build and store AMI config
 	meta := ebsmetadata.AMI{
 		ImageID:         amiID,
 		Name:            name,
@@ -497,10 +514,12 @@ func (s *ImageServiceImpl) CreateImageFromInstance(params CreateImageParams, acc
 		Distro:       sourceAMI.Distro,
 		DistroFamily: sourceAMI.DistroFamily,
 		// Snapshot succeeded before this point, so the image is complete.
-		State: "available",
+		State:    "available",
+		UEFIData: uefiData,
 	}
 
 	if err := s.putAMIConfig(context.Background(), amiID, meta); err != nil {
+		// A UEFI object left under the image's prefix is in the orphan report.
 		slog.Error("CreateImageFromInstance: failed to store AMI config", "amiId", amiID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
@@ -856,6 +875,17 @@ func (s *ImageServiceImpl) CopyImage(ctx context.Context, input *ec2.CopyImageIn
 
 	tags := mergeCopyImageTags(srcMeta.Tags, input.TagSpecifications, aws.BoolValue(input.CopyImageTags))
 
+	uefiData, err := s.copyImageUEFIStore(ctx, srcMeta, newImageID)
+	if err != nil {
+		slog.ErrorContext(ctx, "CopyImage: failed to copy the UEFI variable store",
+			"sourceImageId", sourceImageID, "amiId", newImageID, "orphanSnapshotId", newSnapshotID, "err", err)
+		if delErr := s.metadata.DeleteSnapshot(ctx, accountID, newSnapshotID); delErr != nil {
+			slog.ErrorContext(ctx, "CopyImage: failed to roll back orphaned snapshot metadata",
+				"snapshotId", newSnapshotID, "err", delErr)
+		}
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+
 	meta := ebsmetadata.AMI{
 		ImageID:         newImageID,
 		Name:            name,
@@ -876,12 +906,18 @@ func (s *ImageServiceImpl) CopyImage(ctx context.Context, input *ec2.CopyImageIn
 		SourceDigest:    srcMeta.SourceDigest,
 		// Zero-copy: the new config shares the source's already-durable
 		// snapshot, so the image is complete as soon as this is written.
-		State: "available",
+		State:    "available",
+		UEFIData: uefiData,
 	}
 
 	if err := s.putAMIConfig(ctx, newImageID, meta); err != nil {
+		var orphanUEFIKey string
+		if uefiData != nil {
+			orphanUEFIKey = uefiData.Key
+		}
+		// The copied UEFI object stays under the new image's prefix; the log names it.
 		slog.ErrorContext(ctx, "CopyImage: failed to write AMI config",
-			"amiId", newImageID, "orphanSnapshotId", newSnapshotID, "err", err)
+			"amiId", newImageID, "orphanSnapshotId", newSnapshotID, "orphanUEFIKey", orphanUEFIKey, "err", err)
 		// Best-effort rollback of the orphaned snapshot metadata.
 		if delErr := s.metadata.DeleteSnapshot(ctx, accountID, newSnapshotID); delErr != nil {
 			slog.ErrorContext(ctx, "CopyImage: failed to roll back orphaned snapshot metadata",
@@ -1125,10 +1161,9 @@ func pickRootSnapshotBDM(mappings []*ec2.BlockDeviceMapping, rootDeviceName *str
 	return nil
 }
 
-// DeregisterImage hard-deletes the AMI document. The pre-provider
-// {imageID}/config.json goes too, or the metadata store's legacy read fallback
-// would resurrect the image. Backing snapshot is untouched, so operators run
-// delete-snapshot separately to reclaim block storage.
+// DeregisterImage hard-deletes the image's UEFI store prefix, then the AMI
+// document, so a retry after a failed delete finishes the job. The backing
+// snapshot is untouched, as on AWS.
 func (s *ImageServiceImpl) DeregisterImage(ctx context.Context, input *ec2.DeregisterImageInput, accountID string) (*ec2.DeregisterImageOutput, error) {
 	if input == nil || input.ImageId == nil || *input.ImageId == "" {
 		return nil, errors.New(awserrors.ErrorMissingParameter)
@@ -1140,17 +1175,14 @@ func (s *ImageServiceImpl) DeregisterImage(ctx context.Context, input *ec2.Dereg
 		return nil, err
 	}
 
-	if err := s.metadata.DeleteAMI(ctx, imageID); err != nil && !objectstore.IsNoSuchKeyError(err) {
-		slog.ErrorContext(ctx, "DeregisterImage: failed to delete AMI document", "imageId", imageID, "err", err)
+	// Only the image's own prefix: a copy or a source instance has its own.
+	if n, err := uefistore.DeleteObjects(ctx, s.store, s.bucketName, imageID); err != nil {
+		slog.ErrorContext(ctx, "DeregisterImage: failed to delete the UEFI variable store", "imageId", imageID, "deleted", n, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
-	configKey := fmt.Sprintf("%s/config.json", imageID)
-	if _, err := s.store.DeleteObject(ctx, &s3.DeleteObjectInput{
-		Bucket: aws.String(s.bucketName),
-		Key:    aws.String(configKey),
-	}); err != nil && !objectstore.IsNoSuchKeyError(err) {
-		slog.ErrorContext(ctx, "DeregisterImage: failed to delete legacy AMI config", "imageId", imageID, "err", err)
+	if err := s.metadata.DeleteAMI(ctx, imageID); err != nil && !objectstore.IsNoSuchKeyError(err) {
+		slog.ErrorContext(ctx, "DeregisterImage: failed to delete AMI document", "imageId", imageID, "err", err)
 		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 

spinifex/domains/ec2/image/uefi.go

@@ -0,0 +1,101 @@
+package image
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+)
+
+// uefiPointerAttempts bounds how often CreateImage re-reads a running
+// instance's pointer after the object it named was pruned behind a newer one.
+const uefiPointerAttempts = 3
+
+// UEFIStoreRecords is the store-record read CreateImage makes of its source
+// instance, answered by the stream leader so it sees the last landed pointer.
+type UEFIStoreRecords interface {
+	LeaderGet(ctx context.Context, instanceID string) (*uefistore.UEFIStoreRecord, uint64, error)
+}
+
+var _ UEFIStoreRecords = (*uefistore.Store)(nil)
+
+// SetUEFIStores wires the store records CreateImage reads a UEFI instance's
+// pointer from, and the node ID it writes image store objects as.
+func (s *ImageServiceImpl) SetUEFIStores(records UEFIStoreRecords, nodeID string) {
+	s.uefiRecords = records
+	s.nodeID = nodeID
+}
+
+// copyInstanceUEFIStore copies the object instanceID's store record points at
+// to a fresh key under imageID and returns its reference, or nil when the
+// instance has no record or no pointer. The error is an AWS error code.
+func (s *ImageServiceImpl) copyInstanceUEFIStore(ctx context.Context, instanceID, imageID string) (*ebsmetadata.UEFIDataRef, error) {
+	if s.uefiRecords == nil || s.nodeID == "" {
+		slog.ErrorContext(ctx, "CreateImage: no UEFI store records are wired on this node", "instanceId", instanceID)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	// Once a live pointer is seen, a record that then vanishes is a terminate
+	// racing this request, not an instance without a store.
+	sawPointer := false
+	for range uefiPointerAttempts {
+		rec, _, err := s.uefiRecords.LeaderGet(ctx, instanceID)
+		switch {
+		case errors.Is(err, kvstore.ErrNotFound) && sawPointer:
+			return nil, errors.New(awserrors.ErrorIncorrectInstanceState)
+		case errors.Is(err, kvstore.ErrNotFound):
+			return nil, nil
+		case err != nil:
+			slog.ErrorContext(ctx, "CreateImage: failed to read the UEFI store record", "instanceId", instanceID, "err", err)
+			return nil, errors.New(awserrors.ErrorServerInternal)
+		case rec.State != uefistore.StateLive:
+			return nil, errors.New(awserrors.ErrorIncorrectInstanceState)
+		case rec.Pointer == nil:
+			return nil, nil
+		}
+		sawPointer = true
+		ptr := rec.Pointer
+		key, err := uefistore.CopyObject(ctx, s.store, s.bucketName, ptr.Key, uefistore.ObjectMeta{
+			OwnerID: imageID, WriterNode: s.nodeID, Firmware: ptr.Firmware, Source: uefistore.SourceInstance(instanceID),
+		})
+		if errors.Is(err, uefistore.ErrObjectNotFound) {
+			// A write-back moved the pointer and a prune removed this one.
+			continue
+		}
+		if err != nil {
+			slog.ErrorContext(ctx, "CreateImage: failed to copy the UEFI variable store",
+				"instanceId", instanceID, "imageId", imageID, "key", ptr.Key, "err", err)
+			return nil, errors.New(awserrors.ErrorServerInternal)
+		}
+		slog.InfoContext(ctx, "CreateImage: copied the UEFI variable store",
+			"instanceId", instanceID, "imageId", imageID, "sourceKey", ptr.Key, "key", key)
+		return uefistore.AMIRef(key, ptr.Firmware), nil
+	}
+	slog.ErrorContext(ctx, "CreateImage: the UEFI store pointer kept naming a missing object",
+		"instanceId", instanceID, "attempts", uefiPointerAttempts)
+	return nil, errors.New(awserrors.ErrorServerInternal)
+}
+
+// copyImageUEFIStore copies the store src names to a fresh key under imageID,
+// so deregistering either image leaves the other's intact. It returns nil
+// when src names none.
+func (s *ImageServiceImpl) copyImageUEFIStore(ctx context.Context, src ebsmetadata.AMI, imageID string) (*ebsmetadata.UEFIDataRef, error) {
+	store, err := uefistore.ImageVarStoreFromAMI(src)
+	if err != nil || store == nil {
+		return nil, err
+	}
+	if s.nodeID == "" {
+		return nil, errors.New("no node ID is wired to write UEFI store objects as")
+	}
+	key, err := uefistore.CopyObject(ctx, s.store, s.bucketName, store.Key, uefistore.ObjectMeta{
+		OwnerID: imageID, WriterNode: s.nodeID, Firmware: store.Firmware, Source: uefistore.SourceImage(src.ImageID),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("copy UEFI store %s to %s: %w", store.Key, imageID, err)
+	}
+	return uefistore.AMIRef(key, store.Firmware), nil
+}

spinifex/domains/ec2/image/uefi_test.go

@@ -0,0 +1,417 @@
+package image
+
+import (
+	"bytes"
+	"compress/gzip"
+	"context"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	awss3 "github.com/aws/aws-sdk-go/service/s3"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	uefiTestInstance = "i-0123456789abcdef0"
+	uefiTestVarsSize = 4096
+)
+
+var uefiTestFirmware = uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: uefiTestVarsSize, CodeSHA256: "code-sha"}
+
+func uefiRegion(tag string) []byte {
+	b := make([]byte, uefiTestVarsSize)
+	copy(b, tag)
+	return b
+}
+
+// uefiImageFixture is an image service wired to a real store-record bucket,
+// with a UEFI instance whose root volume CreateImage can snapshot.
+type uefiImageFixture struct {
+	svc       *ImageServiceImpl
+	snapshots *countingSnapshots
+	objects   *objectstore.MemoryObjectStore
+	records   *uefistore.Store
+	volume    ebsmetadata.Volume
+}
+
+func newUEFIImageFixture(t *testing.T) *uefiImageFixture {
+	t.Helper()
+	svc, objects, provider := setupProviderSnapshotImageService(t)
+	_, _, js := testutil.StartTestJetStream(t)
+	records := uefistore.NewStore(js)
+	require.NoError(t, records.Open(t.Context()))
+	svc.SetUEFIStores(records, "node-a")
+	volume := seedProviderVolume(t, svc, provider, "vol-uefi001", 8)
+	snapshots := &countingSnapshots{EBSProvider: provider}
+	svc.SetEBSProvider(snapshots)
+	return &uefiImageFixture{svc: svc, snapshots: snapshots, objects: objects, records: records, volume: volume}
+}
+
+// countingSnapshots counts the snapshots CreateImage asks the provider for.
+type countingSnapshots struct {
+	ebsprovider.EBSProvider
+
+	created int
+}
+
+func (c *countingSnapshots) CreateSnapshot(ctx context.Context, req ebsprovider.CreateSnapshotRequest) (*ebsprovider.Snapshot, error) {
+	c.created++
+	return c.EBSProvider.CreateSnapshot(ctx, req)
+}
+
+// assertNoSnapshot checks a refused CreateImage took no snapshot and wrote no
+// snapshot document.
+func (f *uefiImageFixture) assertNoSnapshot(t *testing.T) {
+	t.Helper()
+	assert.Zero(t, f.snapshots.created, "a refused CreateImage takes no snapshot")
+	snaps, err := f.svc.MetadataStore().ListAllSnapshots(t.Context())
+	require.NoError(t, err)
+	assert.Empty(t, snaps, "a refused CreateImage writes no snapshot document")
+}
+
+// putInstanceStore puts vars at a fresh key under the instance and points a
+// live store record at it.
+func (f *uefiImageFixture) putInstanceStore(t *testing.T, vars []byte) string {
+	t.Helper()
+	key := f.putObject(t, uefiTestInstance, vars)
+	_, err := f.records.Create(t.Context(), &uefistore.UEFIStoreRecord{
+		InstanceID: uefiTestInstance, AccountID: testAccountID, State: uefistore.StateLive, Generation: 1,
+		Pointer: &uefistore.UEFIVarStoreRef{Key: key, Firmware: uefiTestFirmware, WriterNode: "node-a", WrittenAt: time.Now().UTC()},
+	})
+	require.NoError(t, err)
+	return key
+}
+
+func (f *uefiImageFixture) putObject(t *testing.T, owner string, vars []byte) string {
+	t.Helper()
+	key, err := uefistore.NewObjectKey(owner)
+	require.NoError(t, err)
+	require.NoError(t, uefistore.PutObject(t.Context(), f.objects, testBucket, key, vars, uefistore.ObjectMeta{
+		OwnerID: owner, WriterNode: "node-a", Firmware: uefiTestFirmware, Source: uefistore.SourceWriteBack,
+	}))
+	return key
+}
+
+func (f *uefiImageFixture) createImage(t *testing.T, uefi bool) (string, error) {
+	t.Helper()
+	out, err := f.svc.CreateImageFromInstance(CreateImageParams{
+		Input:        &ec2.CreateImageInput{InstanceId: aws.String(uefiTestInstance), Name: aws.String("uefi-image")},
+		RootVolumeID: f.volume.VolumeID,
+		UEFI:         uefi,
+	}, testAccountID)
+	if err != nil {
+		return "", err
+	}
+	return aws.StringValue(out.ImageId), nil
+}
+
+func (f *uefiImageFixture) ami(t *testing.T, imageID string) ebsmetadata.AMI {
+	t.Helper()
+	ami, err := f.svc.MetadataStore().GetAMI(t.Context(), imageID)
+	require.NoError(t, err)
+	return ami
+}
+
+func (f *uefiImageFixture) region(t *testing.T, key string) []byte {
+	t.Helper()
+	out, err := f.objects.GetObject(t.Context(), &awss3.GetObjectInput{Bucket: aws.String(testBucket), Key: aws.String(key)})
+	require.NoError(t, err)
+	defer out.Body.Close()
+	zr, err := gzip.NewReader(out.Body)
+	require.NoError(t, err)
+	b, err := io.ReadAll(zr)
+	require.NoError(t, err)
+	return b
+}
+
+func (f *uefiImageFixture) keys(t *testing.T, owner string) []string {
+	t.Helper()
+	keys, err := uefistore.ListObjects(t.Context(), f.objects, testBucket, owner)
+	require.NoError(t, err)
+	return keys
+}
+
+func (f *uefiImageFixture) copyImage(t *testing.T, src, name string) string {
+	t.Helper()
+	out, err := f.svc.CopyImage(t.Context(), validCopyImageServiceInput(src, name), testAccountID)
+	require.NoError(t, err)
+	return aws.StringValue(out.ImageId)
+}
+
+func (f *uefiImageFixture) deregister(t *testing.T, imageID string) error {
+	t.Helper()
+	_, err := f.svc.DeregisterImage(t.Context(), &ec2.DeregisterImageInput{ImageId: aws.String(imageID)}, testAccountID)
+	return err
+}
+
+// CreateImage copies the object the instance's record points at to the
+// image's own prefix, and the AMI document names it with its firmware.
+func TestCreateImage_CopiesTheInstancesUEFIStore(t *testing.T) {
+	f := newUEFIImageFixture(t)
+	instanceKey := f.putInstanceStore(t, uefiRegion("guest variables"))
+
+	imageID, err := f.createImage(t, true)
+	require.NoError(t, err)
+
+	ami := f.ami(t, imageID)
+	require.NotNil(t, ami.UEFIData)
+	assert.Equal(t, []string{ami.UEFIData.Key}, f.keys(t, imageID), "the image has one object under its own prefix")
+	assert.Equal(t, uefiRegion("guest variables"), f.region(t, ami.UEFIData.Key))
+	assert.Equal(t, uefistore.AMIRef(ami.UEFIData.Key, uefiTestFirmware), ami.UEFIData)
+	assert.Equal(t, []string{instanceKey}, f.keys(t, uefiTestInstance), "the instance's store is left as it was")
+
+	store, err := uefistore.ImageVarStoreFromAMI(ami)
+	require.NoError(t, err)
+	assert.Equal(t, &uefistore.ImageVarStore{ImageID: imageID, Key: ami.UEFIData.Key, Firmware: uefiTestFirmware}, store,
+		"the document decodes into the store a launch seeds from")
+}
+
+// A UEFI instance with no store record or no pointer, and a BIOS instance,
+// give an image that launches from the template.
+func TestCreateImage_AnInstanceWithoutAPointerContributesNothing(t *testing.T) {
+	t.Run("no record", func(t *testing.T) {
+		f := newUEFIImageFixture(t)
+		imageID, err := f.createImage(t, true)
+		require.NoError(t, err)
+		assert.Nil(t, f.ami(t, imageID).UEFIData)
+		assert.Empty(t, f.keys(t, imageID))
+	})
+	t.Run("no pointer", func(t *testing.T) {
+		f := newUEFIImageFixture(t)
+		_, err := f.records.Create(t.Context(), &uefistore.UEFIStoreRecord{
+			InstanceID: uefiTestInstance, AccountID: testAccountID, State: uefistore.StateLive, Generation: 1,
+		})
+		require.NoError(t, err)
+		imageID, err := f.createImage(t, true)
+		require.NoError(t, err)
+		assert.Nil(t, f.ami(t, imageID).UEFIData)
+	})
+	t.Run("BIOS instance", func(t *testing.T) {
+		f := newUEFIImageFixture(t)
+		f.putInstanceStore(t, uefiRegion("never read"))
+		f.svc.SetUEFIStores(nil, "")
+		imageID, err := f.createImage(t, false)
+		require.NoError(t, err)
+		assert.Nil(t, f.ami(t, imageID).UEFIData)
+	})
+}
+
+// A UEFI instance's image is refused rather than made without its store when
+// this node cannot read store records, and a terminating store is refused.
+func TestCreateImage_RefusesWhenTheStoreCannotBeCarried(t *testing.T) {
+	t.Run("no records wired", func(t *testing.T) {
+		f := newUEFIImageFixture(t)
+		f.svc.SetUEFIStores(nil, "")
+		_, err := f.createImage(t, true)
+		require.EqualError(t, err, awserrors.ErrorServerInternal)
+		f.assertNoSnapshot(t)
+	})
+	t.Run("deleting", func(t *testing.T) {
+		f := newUEFIImageFixture(t)
+		f.putInstanceStore(t, uefiRegion("terminating"))
+		_, err := f.records.MarkDeleting(t.Context(), uefiTestInstance)
+		require.NoError(t, err)
+		_, err = f.createImage(t, true)
+		require.EqualError(t, err, awserrors.ErrorIncorrectInstanceState)
+		f.assertNoSnapshot(t)
+	})
+	t.Run("record unreadable", func(t *testing.T) {
+		f := newUEFIImageFixture(t)
+		f.svc.SetUEFIStores(failingRecords{}, "node-a")
+		_, err := f.createImage(t, true)
+		require.EqualError(t, err, awserrors.ErrorServerInternal)
+		f.assertNoSnapshot(t)
+	})
+	t.Run("object unreadable", func(t *testing.T) {
+		f := newUEFIImageFixture(t)
+		key := f.putInstanceStore(t, uefiRegion("x"))
+		_, err := f.objects.PutObject(t.Context(), &awss3.PutObjectInput{
+			Bucket: aws.String(testBucket), Key: aws.String(key), Body: bytes.NewReader([]byte("not gzip")),
+		})
+		require.NoError(t, err)
+		_, err = f.createImage(t, true)
+		require.EqualError(t, err, awserrors.ErrorServerInternal)
+		f.assertNoSnapshot(t)
+	})
+}
+
+// failingRecords cannot read any store record.
+type failingRecords struct{}
+
+func (failingRecords) LeaderGet(context.Context, string) (*uefistore.UEFIStoreRecord, uint64, error) {
+	return nil, 0, assert.AnError
+}
+
+// vanishingRecord shows a live pointer whose object is gone, then no record,
+// as a terminate's reaper leaves it.
+type vanishingRecord struct {
+	stale *uefistore.UEFIStoreRecord
+	reads int
+}
+
+func (v *vanishingRecord) LeaderGet(context.Context, string) (*uefistore.UEFIStoreRecord, uint64, error) {
+	v.reads++
+	if v.reads == 1 {
+		return v.stale, 1, nil
+	}
+	return nil, 0, kvstore.ErrNotFound
+}
+
+// A record that vanishes after CreateImage saw its live pointer is a terminate
+// racing the request, so the image is refused rather than made without its store.
+func TestCreateImage_StoreDeletedMidRequestIsIncorrectInstanceState(t *testing.T) {
+	f := newUEFIImageFixture(t)
+	f.putInstanceStore(t, uefiRegion("x"))
+	stale, _, err := f.records.Get(t.Context(), uefiTestInstance)
+	require.NoError(t, err)
+	stale.Pointer.Key = uefistore.KeyPrefix(uefiTestInstance) + "reaped"
+	f.svc.SetUEFIStores(&vanishingRecord{stale: stale}, "node-a")
+
+	_, err = f.createImage(t, true)
+
+	require.EqualError(t, err, awserrors.ErrorIncorrectInstanceState)
+	f.assertNoSnapshot(t)
+	amis, err := f.svc.MetadataStore().ListAMIs(t.Context())
+	require.NoError(t, err)
+	assert.Empty(t, amis)
+}
+
+// laggingPointer answers the first read with a pointer whose object a prune
+// has removed, and every later read with the record as stored.
+type laggingPointer struct {
+	*uefistore.Store
+
+	stale *uefistore.UEFIStoreRecord
+	reads int
+}
+
+func (l *laggingPointer) LeaderGet(ctx context.Context, id string) (*uefistore.UEFIStoreRecord, uint64, error) {
+	l.reads++
+	if l.reads == 1 {
+		return l.stale, 1, nil
+	}
+	return l.Store.LeaderGet(ctx, id)
+}
+
+// A running instance's write-back can move the pointer and prune the old
+// object between CreateImage's read and its get, so the pointer is read again.
+func TestCreateImage_RereadsAPointerWhoseObjectWasPruned(t *testing.T) {
+	f := newUEFIImageFixture(t)
+	f.putInstanceStore(t, uefiRegion("newest"))
+	stale, _, err := f.records.Get(t.Context(), uefiTestInstance)
+	require.NoError(t, err)
+	stale.Pointer.Key = uefistore.KeyPrefix(uefiTestInstance) + "pruned"
+	lagging := &laggingPointer{Store: f.records, stale: stale}
+	f.svc.SetUEFIStores(lagging, "node-a")
+
+	imageID, err := f.createImage(t, true)
+	require.NoError(t, err)
+
+	assert.Equal(t, 2, lagging.reads)
+	assert.Equal(t, uefiRegion("newest"), f.region(t, f.ami(t, imageID).UEFIData.Key))
+}
+
+// CopyImage gives the copy its own object holding the source's region.
+func TestCopyImage_GivesTheCopyItsOwnUEFIStore(t *testing.T) {
+	f := newUEFIImageFixture(t)
+	f.putInstanceStore(t, uefiRegion("source variables"))
+	src, err := f.createImage(t, true)
+	require.NoError(t, err)
+
+	cp := f.copyImage(t, src, "uefi-copy")
+
+	srcRef, cpRef := f.ami(t, src).UEFIData, f.ami(t, cp).UEFIData
+	require.NotNil(t, cpRef)
+	assert.Equal(t, []string{cpRef.Key}, f.keys(t, cp))
+	assert.Equal(t, srcRef.Firmware, cpRef.Firmware)
+	assert.Equal(t, uefiRegion("source variables"), f.region(t, cpRef.Key))
+	assert.Equal(t, []string{srcRef.Key}, f.keys(t, src), "the source keeps its own object")
+}
+
+// A source AMI with no store gives a copy with none.
+func TestCopyImage_SourceWithoutAUEFIStoreCopiesNone(t *testing.T) {
+	f := newUEFIImageFixture(t)
+	seedCopyableAMI(t, f.objects, "ami-src001", "source-ami", testAccountID, "snap-src001", "vol-src001", 8)
+
+	cp := f.copyImage(t, "ami-src001", "copy")
+
+	assert.Nil(t, f.ami(t, cp).UEFIData)
+	assert.Empty(t, f.keys(t, cp))
+}
+
+// A source store that cannot be copied fails the copy and leaves no AMI.
+func TestCopyImage_UnreadableSourceStoreFails(t *testing.T) {
+	f := newUEFIImageFixture(t)
+	seedCopyableAMI(t, f.objects, "ami-src001", "source-ami", testAccountID, "snap-src001", "vol-src001", 8)
+	src := f.ami(t, "ami-src001")
+	src.UEFIData = uefistore.AMIRef(uefistore.KeyPrefix("ami-src001")+"gone", uefiTestFirmware)
+	require.NoError(t, f.svc.MetadataStore().PutAMI(t.Context(), src))
+
+	_, err := f.svc.CopyImage(t.Context(), validCopyImageServiceInput("ami-src001", "copy"), testAccountID)
+
+	require.EqualError(t, err, awserrors.ErrorServerInternal)
+	amis, err := f.svc.MetadataStore().ListAMIs(t.Context())
+	require.NoError(t, err)
+	assert.Len(t, amis, 1, "no copy is registered without its store")
+}
+
+// TestS7_DeregisteringASourceImageLeavesItsCopysStore enforces ADR-0003:S7
+// for DeregisterImage, which account termination also calls: it deletes only
+// the deregistered image's prefix, and the copy's store is its own.
+func TestS7_DeregisteringASourceImageLeavesItsCopysStore(t *testing.T) {
+	f := newUEFIImageFixture(t)
+	instanceKey := f.putInstanceStore(t, uefiRegion("source variables"))
+	src, err := f.createImage(t, true)
+	require.NoError(t, err)
+	cp := f.copyImage(t, src, "uefi-copy")
+
+	require.NoError(t, f.deregister(t, src))
+
+	assert.Empty(t, f.keys(t, src), "the deregistered image's store is gone")
+	assert.Equal(t, uefiRegion("source variables"), f.region(t, f.ami(t, cp).UEFIData.Key),
+		"ADR-0003:S7: deregistering the source must not delete the store its copy boots from")
+	assert.Equal(t, []string{instanceKey}, f.keys(t, uefiTestInstance), "the instance's store is left as it was")
+}
+
+// failingUEFIDelete fails every delete under the UEFI root.
+type failingUEFIDelete struct{ *objectstore.MemoryObjectStore }
+
+func (s failingUEFIDelete) DeleteObject(ctx context.Context, in *awss3.DeleteObjectInput) (*awss3.DeleteObjectOutput, error) {
+	if strings.HasPrefix(aws.StringValue(in.Key), uefistore.ObjectRoot) {
+		return nil, assert.AnError
+	}
+	return s.MemoryObjectStore.DeleteObject(ctx, in)
+}
+
+// DeregisterImage deletes the store before the document, so a failed delete
+// keeps the image for a retry to finish.
+func TestDeregisterImage_FailedUEFIStoreDeleteKeepsTheImage(t *testing.T) {
+	f := newUEFIImageFixture(t)
+	f.putInstanceStore(t, uefiRegion("x"))
+	imageID, err := f.createImage(t, true)
+	require.NoError(t, err)
+	f.svc.store = failingUEFIDelete{f.objects}
+
+	require.EqualError(t, f.deregister(t, imageID), awserrors.ErrorServerInternal)
+	assert.NotNil(t, f.ami(t, imageID).UEFIData, "the document stays for the retry")
+
+	f.svc.store = f.objects
+	require.NoError(t, f.deregister(t, imageID))
+	assert.Empty(t, f.keys(t, imageID))
+	_, err = f.svc.MetadataStore().GetAMI(t.Context(), imageID)
+	assert.True(t, objectstore.IsNoSuchKeyError(err), "the retry removes the document: %v", err)
+}
+
+var _ UEFIStoreRecords = (*laggingPointer)(nil)

spinifex/domains/ec2/instance/launch_claimed_unsealed_test.go

@@ -10,7 +10,6 @@ import (
 	"testing"
 	"time"
 
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
 	"github.com/stretchr/testify/assert"
@@ -37,7 +36,7 @@ func refusedLaunch(t *testing.T, checker *fakeUnsealedChecker) error {
 
 	// An instance type this node does not offer is the simplest capacity refusal.
 	instance := &vm.VM{ID: "i-held", AccountID: "acc", InstanceType: "no.such.type"}
-	instance.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-root", Boot: true}, {Name: "vol-data"}}
+	instance.EBSRequests.Requests = []vm.EBSRequest{{Name: "vol-root", Boot: true}, {Name: "vol-data"}}
 	return svc.launchClaimedInstance(t.Context(), instance, func() {})
 }
 
@@ -87,7 +86,7 @@ func TestLaunchClaimedInstance_UnansweredCheckIsBoundedAndFailsSafe(t *testing.T
 	svc, _ := claimedRelaunchService(t, func(*vm.VM) error { return nil })
 	svc.SetUnsealedWritesChecker(blockingUnsealedChecker{}, "node-a")
 	instance := &vm.VM{ID: "i-wedged", AccountID: "acc", InstanceType: "no.such.type"}
-	instance.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-root", Boot: true}}
+	instance.EBSRequests.Requests = []vm.EBSRequest{{Name: "vol-root", Boot: true}}
 
 	start := time.Now()
 	err := svc.launchClaimedInstance(t.Context(), instance, func() {})

spinifex/domains/ec2/instance/provider_efi_volume_test.go

@@ -1,126 +0,0 @@
-package instance
-
-import (
-	"bytes"
-	"context"
-	"testing"
-
-	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
-	"github.com/mulgadc/spinifex/spinifex/providers/ebs"
-	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
-	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// varsTemplateBytes stands in for a firmware VARS template. The real one is
-// 540,672 bytes, but nothing here depends on the size beyond it being exact.
-func varsTemplateBytes() []byte { return bytes.Repeat([]byte{0xAB}, 4096) }
-
-func providerEFIService(t *testing.T, provider ebsprovider.EBSProvider) *InstanceServiceImpl {
-	t.Helper()
-	return providerRootVolumeService(t, provider, rootVolumeAMILoader(), objectstore.NewMemoryObjectStore())
-}
-
-// TestCreateEFIVolume_Provider_SeedsFromTemplate is the core of the EFI
-// decouple: the variable store is allocated through the provider with the
-// template as seed bytes, so no control-plane engine is built for it.
-func TestCreateEFIVolume_Provider_SeedsFromTemplate(t *testing.T) {
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true})
-	svc := providerEFIService(t, provider)
-	template := varsTemplateBytes()
-
-	require.NoError(t, svc.createEFIVolumeViaProvider(context.Background(), "vol-efiseed001-efi", template))
-
-	volume, err := provider.GetVolume(context.Background(), ebsprovider.GetVolumeRequest{
-		Versioned: ebsprovider.NewVersioned(), VolumeID: "vol-efiseed001-efi",
-	})
-	require.NoError(t, err)
-	assert.Equal(t, int64(len(template)), volume.CapacityBytes,
-		"the store must be sized byte-exactly to the VARS template")
-
-	seed, ok := provider.SeedData("vol-efiseed001-efi")
-	require.True(t, ok)
-	assert.Equal(t, template, seed, "the template must reach the provider as seed bytes")
-}
-
-// A provider that rounds capacity to a GiB boundary produces a VARS region
-// pflash refuses, which would surface as an unbootable guest rather than a
-// failed launch. The mismatch has to fail the launch instead.
-func TestCreateEFIVolume_Provider_CapacityMismatchFails(t *testing.T) {
-	provider := &roundingProvider{
-		EBSProvider: ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true}),
-	}
-	svc := providerEFIService(t, provider)
-
-	err := svc.createEFIVolumeViaProvider(context.Background(), "vol-efiround01-efi", varsTemplateBytes())
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-// A relaunch reissues the same create. Reseeding would overwrite the BootOrder
-// the guest wrote on its first boot, so the second call must be a no-op.
-func TestCreateEFIVolume_Provider_RepeatDoesNotReseed(t *testing.T) {
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true})
-	svc := providerEFIService(t, provider)
-	ctx := context.Background()
-	template := varsTemplateBytes()
-
-	require.NoError(t, svc.createEFIVolumeViaProvider(ctx, "vol-efirepeat1-efi", template))
-	require.NoError(t, svc.createEFIVolumeViaProvider(ctx, "vol-efirepeat1-efi", template))
-
-	seed, ok := provider.SeedData("vol-efirepeat1-efi")
-	require.True(t, ok)
-	assert.Equal(t, template, seed)
-}
-
-// A seed above the wire bound must fail the launch with an error naming the
-// limit, not reach NATS and fail as an oversized publish.
-func TestCreateEFIVolume_Provider_OversizedTemplateFails(t *testing.T) {
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true})
-	svc := providerEFIService(t, provider)
-
-	err := svc.createEFIVolumeViaProvider(context.Background(), "vol-efitoobig1-efi", make([]byte, ebsprovider.MaxSeedBytes+1))
-	require.Error(t, err)
-	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
-}
-
-// TestPrepareEFIVolume_Provider_RegistersEBSRequest covers the whole function
-// against this host's real firmware, including the name the VM launches with.
-func TestPrepareEFIVolume_Provider_RegistersEBSRequest(t *testing.T) {
-	if _, _, _, err := vm.FirmwarePaths("x86_64"); err != nil {
-		t.Skipf("no UEFI firmware on this host: %v", err)
-	}
-
-	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{VolumeSeeding: true})
-	svc := providerEFIService(t, provider)
-	instance := &vm.VM{}
-
-	require.NoError(t, svc.prepareEFIVolume(context.Background(), "vol-efiprepare001", instance, "x86_64"))
-
-	instance.EBSRequests.Mu.Lock()
-	defer instance.EBSRequests.Mu.Unlock()
-	require.Len(t, instance.EBSRequests.Requests, 1)
-	assert.Equal(t, "vol-efiprepare001-efi", instance.EBSRequests.Requests[0].Name)
-	assert.True(t, instance.EBSRequests.Requests[0].EFI)
-	assert.False(t, instance.EBSRequests.Requests[0].Boot)
-
-	seed, ok := provider.SeedData("vol-efiprepare001-efi")
-	require.True(t, ok)
-	assert.NotEmpty(t, seed, "the store must be seeded from this host's VARS template")
-}
-
-// roundingProvider reports a GiB-rounded capacity the way a provider with no
-// sub-GiB volume granularity would.
-type roundingProvider struct {
-	ebsprovider.EBSProvider
-}
-
-func (p *roundingProvider) CreateVolume(ctx context.Context, req ebsprovider.CreateVolumeRequest) (*ebsprovider.Volume, error) {
-	volume, err := p.EBSProvider.CreateVolume(ctx, req)
-	if err != nil {
-		return nil, err
-	}
-	volume.CapacityBytes = 1 << 30
-	return volume, nil
-}

spinifex/domains/ec2/instance/service_impl.go

@@ -4,13 +4,11 @@ import (
 	"cmp"
 	"context"
 	"encoding/base64"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/tagmirror"
 	"log/slog"
 	"maps"
-	"os"
 	"slices"
 	"sort"
 	"strconv"
@@ -21,12 +19,12 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/bluebottle/pkg/safecast"
 	"github.com/mulgadc/spinifex/contracts/ec2/v1"
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/dns"
 	ebsencryption "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/encryption"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
 	ebspolicy "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/policy"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instancetypes"
 	ec2platform "github.com/mulgadc/spinifex/spinifex/domains/ec2/platform"
 	ec2vpc "github.com/mulgadc/spinifex/spinifex/domains/ec2/vpc"
@@ -1265,6 +1263,10 @@ func (s *InstanceServiceImpl) StartInstance(ctx context.Context, instance *vm.VM
 				slog.WarnContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
 				return refusal
 			}
+			if errors.Is(err, vm.ErrUEFIClaimHeld) {
+				slog.WarnContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
+				return errInsufficientCapacity
+			}
 			slog.ErrorContext(ctx, "StartInstance: vmMgr.Start failed", "instanceId", command.ID, "err", err)
 			return errors.New(awserrors.ErrorServerInternal)
 		}
@@ -1612,18 +1614,20 @@ func (s *InstanceServiceImpl) GenerateVolumes(ctx context.Context, input *ec2.Ru
 		return nil, err
 	}
 
-	// Step 2: Create EFI variable store (only when the AMI is UEFI; BIOS
-	// guests must not allocate an orphan VARS volume).
-	if instance.BootMode == "uefi" || instance.BootMode == "uefi-preferred" {
-		arch := instanceArchitecture(s.instanceTypes[*input.InstanceType])
-		err = s.prepareEFIVolume(ctx, imageId, instance, arch)
-		if err != nil {
-			return nil, err
-		}
+	// Step 2: Seed the UEFI variable store, which is not a volume. A failure
+	// terminates the instance, and terminate deletes whatever store it left.
+	image, err := s.imageUEFIVarStore(ctx, instance, aws.StringValue(input.ImageId))
+	if err != nil {
+		slog.ErrorContext(ctx, "Failed to read the image's UEFI variable store", "instanceId", instance.ID,
+			"imageId", aws.StringValue(input.ImageId), "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
+	}
+	if err := s.vmMgr.SeedUEFIVarStore(ctx, instance, image); err != nil {
+		slog.ErrorContext(ctx, "Failed to seed the UEFI variable store", "instanceId", instance.ID, "err", err)
+		return nil, errors.New(awserrors.ErrorServerInternal)
 	}
 
-	// The EFI store is internal, so it is not reported; every volume the
-	// request named is.
+	// Every volume the request named is reported.
 	volumeInfos := []VolumeInfo{
 		{
 			VolumeId:            imageId,
@@ -1643,6 +1647,23 @@ func (s *InstanceServiceImpl) GenerateVolumes(ctx context.Context, input *ec2.Ru
 	return volumeInfos, nil
 }
 
+// imageUEFIVarStore returns the store the launching AMI names for a UEFI
+// instance to seed from, or nil when the instance boots without UEFI or the
+// AMI names none.
+func (s *InstanceServiceImpl) imageUEFIVarStore(ctx context.Context, instance *vm.VM, imageID string) (*uefistore.ImageVarStore, error) {
+	if !vm.UsesUEFIVarStore(instance) || !strings.HasPrefix(imageID, "ami-") {
+		return nil, nil
+	}
+	if s.amiLoader == nil {
+		return nil, errors.New("AMI loader not initialized")
+	}
+	ami, err := s.amiLoader.GetAMIConfig(ctx, imageID)
+	if err != nil {
+		return nil, err
+	}
+	return uefistore.ImageVarStoreFromAMI(ami)
+}
+
 // prepareDataVolumes creates an empty volume for every EBS mapping that is not
 // the root and records each as a non-boot request the launcher attaches. A
 // failure unwinds the volumes this call created, so a refused launch leaves none.
@@ -1766,7 +1787,7 @@ func (s *InstanceServiceImpl) prepareRootVolume(ctx context.Context, input *ec2.
 // metadata, which is what DescribeVolumes reports as Attachments[].Device.
 func appendRootEBSRequest(instance *vm.VM, volumeID, deviceName string, deleteOnTermination bool) {
 	instance.EBSRequests.Mu.Lock()
-	instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{
+	instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, vm.EBSRequest{
 		Name:                volumeID,
 		DeviceName:          deviceName,
 		Boot:                true,
@@ -1780,7 +1801,7 @@ func appendRootEBSRequest(instance *vm.VM, volumeID, deviceName string, deleteOn
 // Attachments[].Device.
 func appendDataEBSRequest(instance *vm.VM, volumeID, deviceName string, deleteOnTermination bool) {
 	instance.EBSRequests.Mu.Lock()
-	instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{
+	instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, vm.EBSRequest{
 		Name:                volumeID,
 		DeviceName:          deviceName,
 		DeleteOnTermination: deleteOnTermination,
@@ -1885,81 +1906,6 @@ func rootVolumeIOPS(requested int) int {
 	return requested
 }
 
-// prepareEFIVolume creates the per-VM EFI variable store, sized exactly to the
-// firmware VARS template (pflash requires byte-exact size) and seeded from it.
-// arch is "x86_64" | "arm64".
-func (s *InstanceServiceImpl) prepareEFIVolume(ctx context.Context, volumeID string, instance *vm.VM, arch string) error {
-	codePath, varsTemplate, varsSize, err := vm.FirmwarePaths(arch)
-	if err != nil {
-		slog.ErrorContext(ctx, "UEFI firmware not installed on this host", "arch", arch, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	template, err := os.ReadFile(varsTemplate)
-	if err != nil {
-		slog.ErrorContext(ctx, "Failed to read VARS template", "path", varsTemplate, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	if int64(len(template)) != varsSize {
-		slog.ErrorContext(ctx, "VARS template size mismatch between stat and read", "path", varsTemplate, "statSize", varsSize, "readSize", len(template))
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	slog.InfoContext(ctx, "Preparing EFI variable store", "arch", arch, "firmwarePath", codePath, "varsTemplate", varsTemplate, "size", varsSize)
-
-	efiVolumeName := fmt.Sprintf("%s-efi", volumeID)
-	if s.ebsProvider == nil {
-		slog.ErrorContext(ctx, "no EBS provider configured", "op", "prepareEFIVolume")
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	if err := s.createEFIVolumeViaProvider(ctx, efiVolumeName, template); err != nil {
-		return err
-	}
-
-	instance.EBSRequests.Mu.Lock()
-	instance.EBSRequests.Requests = append(instance.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{
-		Name: efiVolumeName,
-		Boot: false,
-		EFI:  true,
-	})
-	instance.EBSRequests.Mu.Unlock()
-
-	return nil
-}
-
-// createEFIVolumeViaProvider allocates the EFI variable store through the
-// provider, seeding it with this node's firmware VARS template. The bytes
-// travel with the request so the region matches the firmware that will run.
-func (s *InstanceServiceImpl) createEFIVolumeViaProvider(ctx context.Context, efiVolumeName string, template []byte) error {
-	slog.InfoContext(ctx, "Creating EFI variable store via EBS provider", "name", efiVolumeName, "seedBytes", len(template))
-
-	// The store is internal to the launch, never attachable and filtered out of
-	// DescribeVolumes, so it gets no ebsmetadata document the way a root volume
-	// does. Its lifecycle is the root volume's DeleteVolume.
-	created, err := s.ebsProvider.CreateVolume(ctx, ebsprovider.CreateVolumeRequest{
-		Versioned:        ebsprovider.NewVersioned(),
-		VolumeID:         efiVolumeName,
-		CapacityRange:    ebsprovider.CapacityRange{RequiredBytes: int64(len(template))},
-		AvailabilityZone: s.config.AZ,
-		SeedData:         template,
-	})
-	if err != nil {
-		slog.ErrorContext(ctx, "Provider EFI volume creation failed", "name", efiVolumeName, "err", err)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	if created == nil {
-		slog.ErrorContext(ctx, "Provider returned no volume for the EFI variable store", "name", efiVolumeName)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-
-	// pflash rejects a VARS region that is not byte-exact, so a provider that
-	// rounded the capacity up to a GiB boundary yields a guest that cannot boot.
-	if created.CapacityBytes != int64(len(template)) {
-		slog.ErrorContext(ctx, "Provider EFI volume capacity does not match the VARS template",
-			"name", efiVolumeName, "wantBytes", len(template), "gotBytes", created.CapacityBytes)
-		return errors.New(awserrors.ErrorServerInternal)
-	}
-	return nil
-}
-
 // checkImageArchitecture refuses an AMI whose architecture the instance type
 // does not support, as AWS does. An unrecorded architecture on either side is
 // not a mismatch.
@@ -2760,7 +2706,7 @@ func (s *InstanceServiceImpl) capacityRefusal(ctx context.Context, instance *vm.
 func (s *InstanceServiceImpl) launchClaimedInstance(ctx context.Context, instance *vm.VM, undo func()) error {
 	// Before anything is stamped or claimed, so the record handed back is the
 	// one that was claimed.
-	if err := vm.CheckUEFIVarStore(instance); err != nil {
+	if err := s.vmMgr.CheckUEFIVarStore(ctx, instance); err != nil {
 		slog.ErrorContext(ctx, "launchClaimedInstance: refusing to launch a UEFI instance without exactly one variable store",
 			"instanceId", instance.ID, "err", err)
 		undo()
@@ -2893,6 +2839,10 @@ func (s *InstanceServiceImpl) launchClaimedInstance(ctx context.Context, instanc
 		if refusal := volumeHeldElsewhereError(err); refusal != nil {
 			return refusal
 		}
+		// A live node holds the variable store, so another node may take it.
+		if errors.Is(err, vm.ErrUEFIClaimHeld) {
+			return s.capacityRefusal(ctx, instance)
+		}
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 
@@ -3069,19 +3019,11 @@ func (s *InstanceServiceImpl) deleteInstanceVolumes(ctx context.Context, instanc
 	}
 
 	for _, ebsRequest := range instance.EBSRequests.Requests {
-		// Internal volumes (EFI) are always cleaned up via ebs.delete.
+		// No release creates an EFI request and start refuses one, so nothing
+		// serves it; its objects go with the root volume or the orphan report.
 		if ebsRequest.EFI {
-			ebsDeleteData, err := json.Marshal(viperblocklegacyv1.EBSDeleteRequest{Volume: ebsRequest.Name})
-			if err != nil {
-				slog.ErrorContext(ctx, "TerminateStoppedInstance: failed to marshal ebs.delete request", "name", ebsRequest.Name, "err", err)
-				continue
-			}
-			deleteMsg, err := s.natsConn.Request(viperblocklegacyv1.DeleteSubject, ebsDeleteData, 30*time.Second)
-			if err != nil {
-				slog.WarnContext(ctx, "TerminateStoppedInstance: ebs.delete failed for internal volume", "name", ebsRequest.Name, "err", err)
-			} else {
-				slog.InfoContext(ctx, "TerminateStoppedInstance: ebs.delete sent for internal volume", "name", ebsRequest.Name, "data", string(deleteMsg.Data))
-			}
+			slog.WarnContext(ctx, "TerminateStoppedInstance: skipping leftover EFI volume request",
+				"name", ebsRequest.Name, "instanceId", instanceID)
 			continue
 		}
 

spinifex/domains/ec2/instance/service_impl_test.go

@@ -15,7 +15,6 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mulgadc/spinifex/contracts/ec2/v1"
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/internal/testkit"
 	"github.com/mulgadc/spinifex/spinifex/bootstrap/config"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
@@ -1855,8 +1854,8 @@ func (f *fakePublicIPReleaser) ReleaseIP(_ context.Context, pool, ip, ownerENIID
 }
 
 // embeddedNATS spins up an in-process NATS server scoped to the test and
-// returns a connected client. Used for service tests that exercise the
-// ebs.delete path inside TerminateStoppedInstance.
+// returns a connected client, for service tests that observe what
+// TerminateStoppedInstance sends over NATS.
 func embeddedNATS(t *testing.T) *nats.Conn {
 	t.Helper()
 	_, nc := testutil.StartTestNATS(t)
@@ -1927,7 +1926,7 @@ func TestTerminateStoppedInstance_TerminationProtected(t *testing.T) {
 			DisableApiTermination: aws.Bool(true),
 		},
 	}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+	v.EBSRequests.Requests = []vm.EBSRequest{
 		{Name: "vol-001", DeleteOnTermination: true},
 	}
 	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
@@ -2038,7 +2037,7 @@ func TestTerminateStoppedInstance_RetriesStoppedDelete(t *testing.T) {
 func TestTerminateStoppedInstance_UserVolumeDeleted(t *testing.T) {
 	id := "i-vol"
 	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc"}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+	v.EBSRequests.Requests = []vm.EBSRequest{
 		{Name: "vol-user-001", DeleteOnTermination: true},
 		{Name: "vol-keep-001", DeleteOnTermination: false},
 	}
@@ -2059,7 +2058,7 @@ func TestTerminateStoppedInstance_FailedSealOwesTheStoppingNodeAnUnpublish(t *te
 	id := "i-seal-failed"
 	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc", LastNode: "node-a",
 		UnpublishOwed: []string{"vol-root-001", "vol-keep-001"}}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+	v.EBSRequests.Requests = []vm.EBSRequest{
 		{Name: "vol-root-001", Boot: true, DeleteOnTermination: true},
 		{Name: "vol-keep-001", DeleteOnTermination: false},
 		{Name: "vol-sealed-001", DeleteOnTermination: false},
@@ -2081,7 +2080,7 @@ func TestTerminateStoppedInstance_FailedSealOwesTheStoppingNodeAnUnpublish(t *te
 func TestTerminateStoppedInstance_NoLastNodeOwesNothing(t *testing.T) {
 	id := "i-seal-failed-nonode"
 	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc", UnpublishOwed: []string{"vol-keep-001"}}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-keep-001"}}
+	v.EBSRequests.Requests = []vm.EBSRequest{{Name: "vol-keep-001"}}
 	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
 	vd := &fakeVolumeDeleter{}
 	svc := &InstanceServiceImpl{stoppedStore: store, volumeDeleter: vd}
@@ -2100,7 +2099,7 @@ func TestTerminateStoppedInstance_NoLastNodeOwesNothing(t *testing.T) {
 func TestTerminateStoppedInstance_StampsTeardownVolumesDone(t *testing.T) {
 	id := "i-teardown-done"
 	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc"}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+	v.EBSRequests.Requests = []vm.EBSRequest{
 		{Name: "vol-root-001", Boot: true, DeleteOnTermination: true},
 	}
 	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
@@ -2124,7 +2123,7 @@ func TestTerminateStoppedInstance_StampsTeardownVolumesDone(t *testing.T) {
 func TestTerminateStoppedInstance_StampsTeardownVolumesFailedOnDeleteError(t *testing.T) {
 	id := "i-teardown-failed"
 	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc"}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+	v.EBSRequests.Requests = []vm.EBSRequest{
 		{Name: "vol-root-002", Boot: true, DeleteOnTermination: true},
 	}
 	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
@@ -2149,7 +2148,7 @@ func TestTerminateStoppedInstance_StampsTeardownVolumesFailedOnDeleteError(t *te
 func TestTerminateStoppedInstance_NonDoTBootVolumeDetachedNotDeleted(t *testing.T) {
 	id := "i-nondot-boot"
 	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc"}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+	v.EBSRequests.Requests = []vm.EBSRequest{
 		{Name: "vol-root-nondot", Boot: true, DeleteOnTermination: false},
 	}
 	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
@@ -2173,7 +2172,7 @@ func TestTerminateStoppedInstance_NonDoTBootVolumeDetachedNotDeleted(t *testing.
 func TestTerminateStoppedInstance_NonDoTDataVolumeDetachedNotDeleted(t *testing.T) {
 	id := "i-nondot-data"
 	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc"}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+	v.EBSRequests.Requests = []vm.EBSRequest{
 		{Name: "vol-data-nondot", DeleteOnTermination: false},
 	}
 	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
@@ -2195,7 +2194,7 @@ func TestTerminateStoppedInstance_NonDoTDataVolumeDetachedNotDeleted(t *testing.
 func TestTerminateStoppedInstance_DetachOfAlreadyGoneVolumeIsNotAFailure(t *testing.T) {
 	id := "i-nondot-gone"
 	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc"}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+	v.EBSRequests.Requests = []vm.EBSRequest{
 		{Name: "vol-data-gone", DeleteOnTermination: false},
 	}
 	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
@@ -2213,7 +2212,7 @@ func TestTerminateStoppedInstance_DetachOfAlreadyGoneVolumeIsNotAFailure(t *test
 func TestTerminateStoppedInstance_NoVolumeDeleterSkipsGracefully(t *testing.T) {
 	id := "i-no-vd"
 	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc"}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+	v.EBSRequests.Requests = []vm.EBSRequest{
 		{Name: "vol-user-001", DeleteOnTermination: true},
 	}
 	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
@@ -2224,30 +2223,48 @@ func TestTerminateStoppedInstance_NoVolumeDeleterSkipsGracefully(t *testing.T) {
 	require.NotNil(t, store.WroteTerminated[id])
 }
 
-func TestTerminateStoppedInstance_InternalVolumesViaNATS(t *testing.T) {
-	id := "i-int-vol"
-	v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc"}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
-		{Name: "vol-efi-001", EFI: true},
+// TestTerminateStoppedInstance_SkipsLeftoverEFIRequest pins that an EFI request
+// left on a record sends nothing and touches no volume, so the teardown mark is
+// decided by the instance's other volumes alone.
+func TestTerminateStoppedInstance_SkipsLeftoverEFIRequest(t *testing.T) {
+	for _, tc := range []struct {
+		name      string
+		deleteErr error
+		want      vm.TeardownState
+	}{
+		{"other volume deleted", nil, vm.TeardownDone},
+		{"other volume failed", errors.New("backend unavailable"), vm.TeardownFailed},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			id := "i-int-vol"
+			v := &vm.VM{ID: id, Status: vm.StateStopped, AccountID: "acc"}
+			v.EBSRequests.Requests = []vm.EBSRequest{
+				{Name: "vol-root-001-efi", EFI: true, DeleteOnTermination: true},
+				{Name: "vol-root-001", Boot: true, DeleteOnTermination: true},
+			}
+			store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
+
+			nc := embeddedNATS(t)
+			deleteSub, err := nc.SubscribeSync("ebs.delete")
+			require.NoError(t, err)
+			defer deleteSub.Unsubscribe()
+
+			vd := &fakeVolumeDeleter{err: tc.deleteErr}
+			svc := &InstanceServiceImpl{stoppedStore: store, natsConn: nc, volumeDeleter: vd}
+
+			_, err = svc.TerminateStoppedInstance(context.Background(), &TerminateStoppedInstanceInput{InstanceID: id}, "acc")
+			require.NoError(t, err)
+
+			require.NoError(t, nc.Flush())
+			pending, _, err := deleteSub.Pending()
+			require.NoError(t, err)
+			assert.Zero(t, pending, "an EFI request must send nothing on ebs.delete")
+			assert.Equal(t, []string{"vol-root-001"}, vd.calls, "an EFI request must reach neither volume-service call")
+			assert.Empty(t, vd.detached)
+			require.NotNil(t, store.WroteTerminated[id])
+			assert.Equal(t, string(tc.want), store.WroteTerminated[id].Teardown[vm.TeardownVolumes])
+		})
 	}
-	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: v}}
-
-	nc := embeddedNATS(t)
-	var ebsDeleted []string
-	sub, err := nc.Subscribe("ebs.delete", func(msg *nats.Msg) {
-		var req viperblocklegacyv1.EBSDeleteRequest
-		_ = json.Unmarshal(msg.Data, &req)
-		ebsDeleted = append(ebsDeleted, req.Volume)
-		_ = msg.Respond([]byte(`{"Success":true}`))
-	})
-	require.NoError(t, err)
-	defer sub.Unsubscribe()
-
-	svc := &InstanceServiceImpl{stoppedStore: store, natsConn: nc}
-
-	_, err = svc.TerminateStoppedInstance(context.Background(), &TerminateStoppedInstanceInput{InstanceID: id}, "acc")
-	require.NoError(t, err)
-	assert.ElementsMatch(t, []string{"vol-efi-001"}, ebsDeleted)
 }
 
 func TestTerminateStoppedInstance_PublicIPReleased(t *testing.T) {
@@ -2815,10 +2832,10 @@ type raceVolumeMounter struct{}
 func (raceVolumeMounter) Mount(context.Context, *vm.VM) error           { return nil }
 func (raceVolumeMounter) Unmount(context.Context, *vm.VM) error         { return nil }
 func (raceVolumeMounter) Abandon(context.Context, *vm.VM, string) error { return nil }
-func (raceVolumeMounter) MountOne(context.Context, string, *viperblocklegacyv1.EBSRequest) error {
+func (raceVolumeMounter) MountOne(context.Context, string, *vm.EBSRequest) error {
 	return nil
 }
-func (raceVolumeMounter) UnmountOne(context.Context, string, viperblocklegacyv1.EBSRequest) error {
+func (raceVolumeMounter) UnmountOne(context.Context, string, vm.EBSRequest) error {
 	return nil
 }
 

spinifex/domains/ec2/instance/start_after_detach_test.go

@@ -7,7 +7,6 @@ import (
 	"sync"
 	"testing"
 
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -31,7 +30,7 @@ func (r *mountRecorder) Mount(_ context.Context, v *vm.VM) error {
 	return errors.New("stop after recording")
 }
 
-func (r *mountRecorder) MountOne(context.Context, string, *viperblocklegacyv1.EBSRequest) error {
+func (r *mountRecorder) MountOne(context.Context, string, *vm.EBSRequest) error {
 	return nil
 }
 

spinifex/domains/ec2/instance/start_public_ip_test.go

@@ -12,7 +12,6 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
 	vmmock "github.com/mulgadc/spinifex/spinifex/runtime/compute/vm/mock"
@@ -28,11 +27,11 @@ type refusingMounter struct{}
 
 func (refusingMounter) Mount(context.Context, *vm.VM) error   { return errors.New("no volumes here") }
 func (refusingMounter) Unmount(context.Context, *vm.VM) error { return nil }
-func (refusingMounter) MountOne(context.Context, string, *viperblocklegacyv1.EBSRequest) error {
+func (refusingMounter) MountOne(context.Context, string, *vm.EBSRequest) error {
 	return nil
 }
 func (refusingMounter) Abandon(context.Context, *vm.VM, string) error { return nil }
-func (refusingMounter) UnmountOne(context.Context, string, viperblocklegacyv1.EBSRequest) error {
+func (refusingMounter) UnmountOne(context.Context, string, vm.EBSRequest) error {
 	return nil
 }
 

spinifex/domains/ec2/instance/start_volumes.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
@@ -28,7 +27,7 @@ func (s *InstanceServiceImpl) reconcileStartVolumes(ctx context.Context, instanc
 	requests := slices.Clone(instance.EBSRequests.Requests)
 	instance.EBSRequests.Mu.Unlock()
 
-	if !slices.ContainsFunc(requests, func(r viperblocklegacyv1.EBSRequest) bool { return r.Boot }) {
+	if !slices.ContainsFunc(requests, func(r vm.EBSRequest) bool { return r.Boot }) {
 		return noRootVolumeError(instance)
 	}
 
@@ -65,7 +64,7 @@ func (s *InstanceServiceImpl) reconcileStartVolumes(ctx context.Context, instanc
 	}
 
 	instance.EBSRequests.Mu.Lock()
-	instance.EBSRequests.Requests = slices.DeleteFunc(instance.EBSRequests.Requests, func(r viperblocklegacyv1.EBSRequest) bool {
+	instance.EBSRequests.Requests = slices.DeleteFunc(instance.EBSRequests.Requests, func(r vm.EBSRequest) bool {
 		return slices.Contains(drop, r.Name)
 	})
 	instance.EBSRequests.Mu.Unlock()

spinifex/domains/ec2/instance/start_volumes_test.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
@@ -36,7 +35,7 @@ func startingInstance(id string, names ...string) *vm.VM {
 		if boot {
 			device = "/dev/sda1"
 		}
-		v.EBSRequests.Requests = append(v.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{Name: n, Boot: boot, DeviceName: device})
+		v.EBSRequests.Requests = append(v.EBSRequests.Requests, vm.EBSRequest{Name: n, Boot: boot, DeviceName: device})
 		v.Instance.BlockDeviceMappings = append(v.Instance.BlockDeviceMappings, &ec2.InstanceBlockDeviceMapping{
 			DeviceName: aws.String(device), Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String(n)},
 		})
@@ -136,7 +135,7 @@ func TestReconcileStartVolumes_DirectBootNeedsNoRootVolume(t *testing.T) {
 
 	sqs := &vm.VM{ID: "i-sqs", AccountID: startVolumesAccount, Status: vm.StateStopped, DirectBoot: true,
 		Instance: &ec2.Instance{InstanceId: aws.String("i-sqs")}}
-	sqs.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{{Name: "vol-store-gone", DeviceName: "/dev/vdb"}}
+	sqs.EBSRequests.Requests = []vm.EBSRequest{{Name: "vol-store-gone", DeviceName: "/dev/vdb"}}
 	require.NoError(t, svc.reconcileStartVolumes(context.Background(), sqs))
 	assert.Equal(t, []string{"vol-store-gone"}, requestedNames(sqs), "a system store volume is never pruned")
 }

spinifex/domains/ec2/instance/stopped_detach.go

@@ -8,18 +8,17 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	ebsmetadata "github.com/mulgadc/spinifex/spinifex/domains/ec2/ebs/metadata"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
 )
 
-func findVolumeRequest(v *vm.VM, volumeID string) (viperblocklegacyv1.EBSRequest, bool) {
+func findVolumeRequest(v *vm.VM, volumeID string) (vm.EBSRequest, bool) {
 	v.EBSRequests.Mu.Lock()
 	defer v.EBSRequests.Mu.Unlock()
-	i := slices.IndexFunc(v.EBSRequests.Requests, func(r viperblocklegacyv1.EBSRequest) bool { return r.Name == volumeID })
+	i := slices.IndexFunc(v.EBSRequests.Requests, func(r vm.EBSRequest) bool { return r.Name == volumeID })
 	if i < 0 {
-		return viperblocklegacyv1.EBSRequest{}, false
+		return vm.EBSRequest{}, false
 	}
 	return v.EBSRequests.Requests[i], true
 }
@@ -29,7 +28,7 @@ func findVolumeRequest(v *vm.VM, volumeID string) (viperblocklegacyv1.EBSRequest
 func removeVolumeFromRecord(v *vm.VM, volumeID string) bool {
 	v.EBSRequests.Mu.Lock()
 	defer v.EBSRequests.Mu.Unlock()
-	i := slices.IndexFunc(v.EBSRequests.Requests, func(r viperblocklegacyv1.EBSRequest) bool { return r.Name == volumeID })
+	i := slices.IndexFunc(v.EBSRequests.Requests, func(r vm.EBSRequest) bool { return r.Name == volumeID })
 	if i < 0 || v.EBSRequests.Requests[i].EFI {
 		return false
 	}
@@ -93,7 +92,7 @@ func stoppedAttachDevice(v *vm.VM, requested string) (string, bool, error) {
 func addVolumeToRecord(v *vm.VM, volumeID, device string, boot bool) {
 	v.EBSRequests.Mu.Lock()
 	defer v.EBSRequests.Mu.Unlock()
-	v.EBSRequests.Requests = append(v.EBSRequests.Requests, viperblocklegacyv1.EBSRequest{
+	v.EBSRequests.Requests = append(v.EBSRequests.Requests, vm.EBSRequest{
 		Name:       volumeID,
 		DeviceName: device,
 		Boot:       boot,

spinifex/domains/ec2/instance/stopped_detach_test.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/ec2"
-	viperblocklegacyv1 "github.com/mulgadc/spinifex/contracts/viperblockd/legacy/v1"
 	"github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
 	awsidentifiers "github.com/mulgadc/spinifex/spinifex/foundation/aws/identifiers"
 	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
@@ -27,7 +26,7 @@ func stoppedWithVolumes(owner, id string) *vm.VM {
 			},
 		},
 	}
-	v.EBSRequests.Requests = []viperblocklegacyv1.EBSRequest{
+	v.EBSRequests.Requests = []vm.EBSRequest{
 		{Name: "vol-root", Boot: true, DeviceName: "/dev/sda1"},
 		{Name: "vol-data", DeviceName: "/dev/sdf"},
 		{Name: "vol-other", DeviceName: "/dev/sdg"},
@@ -127,7 +126,7 @@ func TestDetachVolumeFromStopped_RefusesTheInternalEFIStore(t *testing.T) {
 	id := "i-stopped-efi"
 	stored := stoppedWithVolumes(owner, id)
 	stored.EBSRequests.Requests = append(stored.EBSRequests.Requests,
-		viperblocklegacyv1.EBSRequest{Name: "vol-efi", EFI: true})
+		vm.EBSRequest{Name: "vol-efi", EFI: true})
 	store := &vmmock.StateStore{Stopped: map[string]*vm.VM{id: stored}}
 	svc := &InstanceServiceImpl{vmMgr: mgrWith(map[string]*vm.VM{}), stoppedStore: store}
 

spinifex/domains/ec2/instance/teardown_outstanding.go

@@ -1,12 +1,33 @@
 package instance
 
+import "context"
+
 // OutstandingInstance is an instance of an account that is not yet fully torn
 // down, with the state its record carries.
 type OutstandingInstance struct {
-	InstanceID string
+	InstanceID string `json:"instance_id"`
 	// State is the record's lifecycle state, such as running or terminated.
-	State string
+	State string `json:"state"`
 	// Teardown is the per-dependent progress (done, pending or failed) the
 	// record carries. It is empty when no dependent has been recorded.
-	Teardown map[string]string
+	Teardown map[string]string `json:"teardown,omitempty"`
+}
+
+// TeardownOutstandingInput is the empty request of the outstanding-teardown
+// read; the account comes from the message header.
+type TeardownOutstandingInput struct{}
+
+// TeardownOutstandingOutput carries every instance not yet fully torn down.
+type TeardownOutstandingOutput struct {
+	Instances []OutstandingInstance `json:"instances"`
+}
+
+// AccountTeardownOutstandingRPC serves AccountTeardownOutstanding over the
+// message bus, so a process without the instance record store can read it.
+func (s *InstanceServiceImpl) AccountTeardownOutstandingRPC(_ context.Context, _ *TeardownOutstandingInput, accountID string) (*TeardownOutstandingOutput, error) {
+	outstanding, err := s.AccountTeardownOutstanding(accountID)
+	if err != nil {
+		return nil, err
+	}
+	return &TeardownOutstandingOutput{Instances: outstanding}, nil
 }

spinifex/domains/ec2/instance/uefi_seed_test.go

@@ -0,0 +1,171 @@
+package instance
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	testutil "github.com/mulgadc/spinifex/internal/testkit"
+	"github.com/mulgadc/spinifex/spinifex/domains/ec2/instance/uefistore"
+	awserrors "github.com/mulgadc/spinifex/spinifex/foundation/aws/errors"
+	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
+	ebsprovider "github.com/mulgadc/spinifex/spinifex/providers/ebs"
+	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
+	"github.com/mulgadc/spinifex/spinifex/runtime/compute/vm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const testSeedInstance = "i-0a1b2c3d4e5f60718"
+
+type seedInstanceTypes map[string]vm.InstanceTypeSpec
+
+func (r seedInstanceTypes) Resolve(name string) (vm.InstanceTypeSpec, bool) {
+	spec, ok := r[name]
+	return spec, ok
+}
+
+// uefiSeedingService is an instance service whose VM manager seeds stores into
+// a real JetStream's store records and claims and an in-memory object store,
+// under a stand-in x86_64 firmware.
+func uefiSeedingService(t *testing.T) (*InstanceServiceImpl, *ebsprovider.MemoryProvider, *uefistore.Store, *objectstore.MemoryObjectStore) {
+	t.Helper()
+	fwDir := t.TempDir()
+	code := filepath.Join(fwDir, "OVMF_CODE.fd")
+	vars := filepath.Join(fwDir, "OVMF_VARS.fd")
+	require.NoError(t, os.WriteFile(code, []byte("code"), 0o600))
+	require.NoError(t, os.WriteFile(vars, make([]byte, 4096), 0o600))
+	orig, had := vm.FirmwarePathCandidates["x86_64"]
+	vm.FirmwarePathCandidates["x86_64"] = []vm.FirmwareCandidate{{Code: code, VarsTemplate: vars}}
+	t.Cleanup(func() {
+		if had {
+			vm.FirmwarePathCandidates["x86_64"] = orig
+		} else {
+			delete(vm.FirmwarePathCandidates, "x86_64")
+		}
+	})
+
+	_, _, js := testutil.StartTestJetStream(t)
+	records := uefistore.NewStore(js)
+	require.NoError(t, records.Open(t.Context()))
+
+	provider := ebsprovider.NewMemoryProvider(ebsprovider.Capabilities{CrashConsistentSnapshot: true, VolumeEnumeration: true})
+	seedProviderSnapshot(t, provider, "vol-origin", "snap-source")
+	objects := objectstore.NewMemoryObjectStore()
+	svc := providerRootVolumeService(t, provider, rootVolumeAMILoader(), objects)
+	svc.vmMgr = vm.NewManagerWithDeps(vm.Deps{
+		NodeID:        "node-a",
+		InstanceTypes: seedInstanceTypes{"t3.nano": {Architecture: "x86_64", VCPUs: 1, MemoryMiB: 512}},
+		UEFIVarStore: vm.UEFIVarStoreDeps{
+			Objects: objects, Bucket: testRootBucket, DataDir: t.TempDir(), Records: records,
+			Claims: uefistore.NewClaims(kvstore.OpenSyncBucket(js), "node-a"),
+		},
+	})
+	return svc, provider, records, objects
+}
+
+func uefiLaunch() (*ec2.RunInstancesInput, *vm.VM) {
+	input := &ec2.RunInstancesInput{ImageId: aws.String("ami-1"), InstanceType: aws.String("t3.nano")}
+	return input, &vm.VM{ID: testSeedInstance, AccountID: testRootAccount, InstanceType: "t3.nano", BootMode: "uefi"}
+}
+
+// A UEFI launch seeds its store as an object and a store record, and creates
+// no <root>-efi volume and no EFI request.
+func TestGenerateVolumes_UEFILaunchSeedsAStoreAndCreatesNoEFIVolume(t *testing.T) {
+	svc, provider, records, objects := uefiSeedingService(t)
+	input, instance := uefiLaunch()
+
+	infos, err := svc.GenerateVolumes(t.Context(), input, instance)
+	require.NoError(t, err)
+
+	require.Len(t, infos, 1, "only the root volume is reported")
+	instance.EBSRequests.Mu.Lock()
+	for _, req := range instance.EBSRequests.Requests {
+		assert.False(t, req.EFI, "no EFI request is appended")
+		assert.False(t, strings.HasSuffix(req.Name, "-efi"), req.Name)
+	}
+	instance.EBSRequests.Mu.Unlock()
+	listed, err := provider.ListVolumes(t.Context(), ebsprovider.ListVolumesRequest{Versioned: ebsprovider.NewVersioned()})
+	require.NoError(t, err)
+	for _, v := range listed.Volumes {
+		assert.False(t, strings.HasSuffix(v.ID, "-efi"), "no EFI volume is created: %s", v.ID)
+	}
+
+	rec, _, err := records.Get(t.Context(), testSeedInstance)
+	require.NoError(t, err)
+	assert.Equal(t, uefistore.StateLive, rec.State)
+	require.NotNil(t, rec.Pointer)
+	keys, err := uefistore.ListObjects(t.Context(), objects, testRootBucket, testSeedInstance)
+	require.NoError(t, err)
+	assert.Equal(t, []string{rec.Pointer.Key}, keys, "the pointer names the one template object")
+}
+
+// A seed that cannot be written fails the launch as InternalError, and no EFI
+// volume is created in its place.
+func TestGenerateVolumes_FailedSeedIsInternalError(t *testing.T) {
+	svc, provider, _, _ := uefiSeedingService(t)
+	svc.vmMgr = vm.NewManager()
+	input, instance := uefiLaunch()
+
+	_, err := svc.GenerateVolumes(t.Context(), input, instance)
+
+	require.Error(t, err)
+	assert.Equal(t, awserrors.ErrorServerInternal, err.Error())
+	listed, err := provider.ListVolumes(t.Context(), ebsprovider.ListVolumesRequest{Versioned: ebsprovider.NewVersioned()})
+	require.NoError(t, err)
+	for _, v := range listed.Volumes {
+		assert.False(t, strings.HasSuffix(v.ID, "-efi"), v.ID)
+	}
+}
+
+// putImageStore gives ami-1 a store object holding vars and names it in the
+// AMI document the launch reads.
+func putImageStore(t *testing.T, svc *InstanceServiceImpl, objects *objectstore.MemoryObjectStore, key string, vars []byte) {
+	t.Helper()
+	if strings.HasPrefix(key, uefistore.KeyPrefix("ami-1")) {
+		require.NoError(t, uefistore.PutObject(t.Context(), objects, testRootBucket, key, vars, uefistore.ObjectMeta{
+			OwnerID: "ami-1", WriterNode: "node-b", Source: uefistore.SourceInstance("i-0000000000000000a"),
+			Firmware: uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: int64(len(vars)), CodeSHA256: "other"},
+		}))
+	}
+	loader := svc.amiLoader.(*fakeAMILoader)
+	ami := loader.byID["ami-1"]
+	ami.UEFIData = uefistore.AMIRef(key, uefistore.FirmwareIdentity{Arch: "x86_64", VarsSize: int64(len(vars)), CodeSHA256: "other"})
+	loader.byID["ami-1"] = ami
+}
+
+// An AMI that names a store seeds the new instance's own object from it.
+func TestGenerateVolumes_UEFILaunchSeedsFromTheAMIsStore(t *testing.T) {
+	svc, _, records, objects := uefiSeedingService(t)
+	vars := make([]byte, 4096)
+	copy(vars, "image variables")
+	putImageStore(t, svc, objects, uefistore.KeyPrefix("ami-1")+"k", vars)
+	input, instance := uefiLaunch()
+
+	_, err := svc.GenerateVolumes(t.Context(), input, instance)
+	require.NoError(t, err)
+
+	rec, _, err := records.Get(t.Context(), testSeedInstance)
+	require.NoError(t, err)
+	require.NotNil(t, rec.Pointer)
+	got, _, err := uefistore.ReadObject(t.Context(), objects, testRootBucket, rec.Pointer.Key, 4096)
+	require.NoError(t, err)
+	assert.Equal(t, vars, got)
+}
+
+// An AMI document whose store reference is outside the image's own prefix is
+// refused, and the launch records no store.
+func TestGenerateVolumes_AMIStoreOutsideItsPrefixIsInternalError(t *testing.T) {
+	svc, _, records, objects := uefiSeedingService(t)
+	putImageStore(t, svc, objects, uefistore.KeyPrefix("i-0000000000000000a")+"k", make([]byte, 4096))
+	input, instance := uefiLaunch()
+
+	_, err := svc.GenerateVolumes(t.Context(), input, instance)
+
+	require.EqualError(t, err, awserrors.ErrorServerInternal)
+	_, _, err = records.Get(t.Context(), testSeedInstance)
+	require.ErrorIs(t, err, kvstore.ErrNotFound)
+}

spinifex/domains/ec2/instance/uefistore/cleanup.go

@@ -10,7 +10,6 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvstore"
-	"github.com/mulgadc/spinifex/spinifex/foundation/state/kvutil"
 	"github.com/mulgadc/spinifex/spinifex/providers/objectstore"
 	"github.com/nats-io/nats.go/jetstream"
 )
@@ -26,9 +25,9 @@ func (s *Store) Exists(ctx context.Context, instanceID string) (bool, error) {
 	return s.kv.Exists(ctx, instanceID)
 }
 
-// MarkDeleting records that instanceID's store is to be deleted, and drops any
-// write-back still owed. It reports false with no error when the instance has
-// no store record. Only a terminate or an operator's forget may call it.
+// MarkDeleting records that instanceID's store is to be deleted. It reports
+// false with no error when the instance has no store record. Only a decided
+// terminate may call it.
 func (s *Store) MarkDeleting(ctx context.Context, instanceID string) (bool, error) {
 	for range markAttempts {
 		rec, rev, err := s.Get(ctx, instanceID)
@@ -42,7 +41,6 @@ func (s *Store) MarkDeleting(ctx context.Context, instanceID string) (bool, erro
 			return true, nil
 		}
 		rec.State = StateDeleting
-		rec.WriteBack = nil
 		_, err = s.CompareAndSet(ctx, rec, rev)
 		if err == nil {
 			return true, nil
@@ -72,7 +70,7 @@ func (s *Store) InstanceIDs(ctx context.Context) ([]string, error) {
 
 	var ids []string
 	for key := range lister.Keys() {
-		if key != kvutil.VersionKey {
+		if key != kvstore.VersionKey {
 			ids = append(ids, key)
 		}
 	}

spinifex/domains/ec2/instance/uefistore/cleanup_test.go

@@ -37,7 +37,7 @@ func keysUnder(t *testing.T, objects objectstore.ObjectStore, prefix string) []s
 }
 
 // Deletion is final: a writer holding the revision of a deleting record, such
-// as a write-back retry, must not bring the store back to live.
+// as a late pointer move, must not bring the store back to live.
 func TestCompareAndSet_RefusesDeletingToLive(t *testing.T) {
 	s, _ := openStore(t)
 	rev, err := s.Create(t.Context(), liveRecord())
@@ -47,8 +47,7 @@ func TestCompareAndSet_RefusesDeletingToLive(t *testing.T) {
 	rev, err = s.CompareAndSet(t.Context(), deleting, rev)
 	require.NoError(t, err)
 
-	revived := liveRecord()
-	revived.WriteBack = writeBack()
+	revived := pointedRecord()
 	_, err = s.CompareAndSet(t.Context(), revived, rev)
 	require.ErrorIs(t, err, uefistore.ErrDeleting)
 
@@ -58,24 +57,9 @@ func TestCompareAndSet_RefusesDeletingToLive(t *testing.T) {
 	assert.Equal(t, uefistore.StateDeleting, got.State)
 }
 
-// A write-back owed on a deleting store would put an object behind the delete.
-func TestCompareAndSet_RefusesDeletingWithAWriteBack(t *testing.T) {
+func TestMarkDeleting_MovesLiveToDeletingAndKeepsWhatItNames(t *testing.T) {
 	s, _ := openStore(t)
-	rev, err := s.Create(t.Context(), liveRecord())
-	require.NoError(t, err)
-	rec := liveRecord()
-	rec.State = uefistore.StateDeleting
-	rec.WriteBack = writeBack()
-
-	_, err = s.CompareAndSet(t.Context(), rec, rev)
-	require.ErrorIs(t, err, uefistore.ErrInvalidRecord)
-}
-
-func TestMarkDeleting_MovesLiveToDeletingAndDropsTheWriteBack(t *testing.T) {
-	s, _ := openStore(t)
-	rec := liveRecord()
-	rec.WriteBack = writeBack()
-	_, err := s.Create(t.Context(), rec)
+	_, err := s.Create(t.Context(), pointedRecord())
 	require.NoError(t, err)
 
 	has, err := s.MarkDeleting(t.Context(), instanceID)
@@ -85,8 +69,8 @@ func TestMarkDeleting_MovesLiveToDeletingAndDropsTheWriteBack(t *testing.T) {
 	got, rev, err := s.Get(t.Context(), instanceID)
 	require.NoError(t, err)
 	assert.Equal(t, uefistore.StateDeleting, got.State)
-	assert.Nil(t, got.WriteBack)
 	assert.Equal(t, "111122223333", got.AccountID)
+	assert.Equal(t, pointer("b"), got.Pointer, "the pointer stays readable until the prefix is gone")
 
 	has, err = s.MarkDeleting(t.Context(), instanceID)
 	require.NoError(t, err)